Merge pull request 'TMTC CFDP integration' (#113) from mueller/cfdp-integration into main

Reviewed-on: #113
Reviewed-by: Jakob Meier <meierj@irs.uni-stuttgart.de>

.gitignore

@@ -8,4 +8,4 @@ log
 
 /scex_conf.json
 /tmtc_conf.json
-/seqcnt.txt
+/seqcnt*.txt

.run/CFDP Help.run.xml

@@ -0,0 +1,24 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="CFDP Help" type="PythonConfigurationType" factoryName="Python" folderName="CFDP">
+    <module name="tmtc" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="PARENT_ENVS" value="true" />
+    <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
+    </envs>
+    <option name="SDK_HOME" value="" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="IS_MODULE_SDK" value="true" />
+    <option name="ADD_CONTENT_ROOTS" value="true" />
+    <option name="ADD_SOURCE_ROOTS" value="true" />
+    <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+    <option name="SCRIPT_NAME" value="$PROJECT_DIR$/tmtcc.py" />
+    <option name="PARAMETERS" value="cfdp -h" />
+    <option name="SHOW_COMMAND_LINE" value="false" />
+    <option name="EMULATE_TERMINAL" value="false" />
+    <option name="MODULE_MODE" value="false" />
+    <option name="REDIRECT_INPUT" value="false" />
+    <option name="INPUT_FILE" value="" />
+    <method v="2" />
+  </configuration>
+</component>

.run/CFDP Test File Large.run.xml

@@ -0,0 +1,24 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="CFDP Test File Large" type="PythonConfigurationType" factoryName="Python" folderName="CFDP">
+    <module name="tmtc" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="PARENT_ENVS" value="true" />
+    <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
+    </envs>
+    <option name="SDK_HOME" value="" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="IS_MODULE_SDK" value="true" />
+    <option name="ADD_CONTENT_ROOTS" value="true" />
+    <option name="ADD_SOURCE_ROOTS" value="true" />
+    <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+    <option name="SCRIPT_NAME" value="$PROJECT_DIR$/tmtcc.py" />
+    <option name="PARAMETERS" value="cfdp filetest/obsw_update.bin /tmp/obsw_update.bin -d 0.3" />
+    <option name="SHOW_COMMAND_LINE" value="false" />
+    <option name="EMULATE_TERMINAL" value="false" />
+    <option name="MODULE_MODE" value="false" />
+    <option name="REDIRECT_INPUT" value="false" />
+    <option name="INPUT_FILE" value="" />
+    <method v="2" />
+  </configuration>
+</component>

.run/CFDP Test File Small.run.xml

@@ -0,0 +1,24 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="CFDP Test File Small" type="PythonConfigurationType" factoryName="Python" folderName="CFDP">
+    <module name="tmtc" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="PARENT_ENVS" value="true" />
+    <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
+    </envs>
+    <option name="SDK_HOME" value="" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="IS_MODULE_SDK" value="true" />
+    <option name="ADD_CONTENT_ROOTS" value="true" />
+    <option name="ADD_SOURCE_ROOTS" value="true" />
+    <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+    <option name="SCRIPT_NAME" value="$PROJECT_DIR$/tmtcc.py" />
+    <option name="PARAMETERS" value="cfdp filetest/cfdp_test.txt /tmp/cfdp_test.txt -d 0.2" />
+    <option name="SHOW_COMMAND_LINE" value="false" />
+    <option name="EMULATE_TERMINAL" value="false" />
+    <option name="MODULE_MODE" value="false" />
+    <option name="REDIRECT_INPUT" value="false" />
+    <option name="INPUT_FILE" value="" />
+    <method v="2" />
+  </configuration>
+</component>

CHANGELOG.md

@@ -12,6 +12,8 @@ list yields a list of all related PRs for each release.
 
 # [v1.13.0]
 
+- CFDP integration
+  PR: https://egit.irs.uni-stuttgart.de/eive/eive-tmtc/pulls/113
 - Major Update for `tmtccmd` and `spacepackets` dependencies which improves user API significantly.
   PR: https://egit.irs.uni-stuttgart.de/eive/eive-tmtc/pulls/102
 - Add commands to request MGM HK or enable/disable periodic HK for it

config/definitions.py

@@ -6,9 +6,14 @@
 
 import enum
 
+from spacepackets.util import UnsignedByteField
 
 PUS_APID = 0x65
 SPACE_PACKET_IDS = (0x08 << 8 | PUS_APID,)
+CFDP_APID = 0x66
+
+CFDP_LOCAL_ENTITY_ID = UnsignedByteField(byte_len=2, val=1)
+CFDP_REMOTE_ENTITY_ID = UnsignedByteField(byte_len=2, val=CFDP_APID)
 
 
 class CustomServiceList(str, enum.Enum):

config/hook.py

@@ -25,7 +25,7 @@ class EiveHookObject(TmTcCfgHookBase):
 
         cfg = create_com_interface_cfg_default(
             com_if_key=com_if_key,
-            json_cfg_path=self.json_cfg_path,
+            json_cfg_path=self.cfg_path,
             space_packet_ids=SPACE_PACKET_IDS,
         )
         return create_com_interface_default(cfg)

deps/spacepackets

@@ -1 +1 @@
-Subproject commit da2859688bc36ac258f0d75edbc2768761f34f31
+Subproject commit 84f1af27d4b15dde47b9944e0e533dff6fd8b1a4

deps/tmtccmd

@@ -1 +1 @@
-Subproject commit 5583005cd57fce94ac718fd5ad35f639623fcac6
+Subproject commit 96ca55b17a823ce10307e8aa6d8cd906323d1ae5

filetest/cfdp_test.txt

@@ -0,0 +1 @@
+hello world

tmtcc.py

@@ -3,7 +3,31 @@ import logging
 import sys
 import time
 import traceback
+from pathlib import Path
+from typing import cast
 
+from spacepackets import SpacePacketHeader, SpacePacket
+from spacepackets.cfdp import (
+    ConditionCode,
+    ChecksumType,
+    TransmissionMode,
+    PduHolder,
+    DirectiveType,
+)
+from tmtccmd.cfdp import CfdpUserBase, TransactionId
+from tmtccmd.cfdp.defs import CfdpRequestType
+from tmtccmd.cfdp.handler import CfdpInCcsdsHandler
+from tmtccmd.cfdp.mib import (
+    DefaultFaultHandlerBase,
+    LocalEntityCfg,
+    IndicationCfg,
+    RemoteEntityCfg,
+)
+from tmtccmd.cfdp.user import (
+    TransactionFinishedParams,
+    MetadataRecvParams,
+    FileSegmentRecvdParams,
+)
 from tmtccmd.tc.handler import SendCbParams
 
 try:
@@ -41,16 +65,29 @@ from tmtccmd.tm import SpecificApidHandlerBase, GenericApidHandlerBase, CcsdsTmH
 from tmtccmd.core import BackendRequest
 from tmtccmd.logging import get_current_time_string
 from tmtccmd.tc import (
-    ProcedureHelper,
+    ProcedureWrapper,
     FeedWrapper,
     TcProcedureType,
     TcQueueEntryType,
     DefaultPusQueueHelper,
 )
-from tmtccmd.config import default_json_path, SetupWrapper
+from tmtccmd.config import (
-from tmtccmd.config.args import SetupParams, ArgParserWrapper
+    default_json_path,
+    SetupWrapper,
+    params_to_procedure_conversion,
+)
+from tmtccmd.config.args import (
+    SetupParams,
+    PreArgsParsingWrapper,
+    ProcedureParamsWrapper,
+)
 from config import __version__
-from config.definitions import PUS_APID
+from config.definitions import (
+    PUS_APID,
+    CFDP_APID,
+    CFDP_LOCAL_ENTITY_ID,
+    CFDP_REMOTE_ENTITY_ID,
+)
 from config.hook import EiveHookObject
 from pus_tm.factory_hook import pus_factory_hook
 from pus_tc.procedure_packer import handle_default_procedure
@@ -63,6 +100,61 @@ ROTATING_TIMED_LOGGER_INTERVAL_WHEN = TimedLogWhen.PER_MINUTE
 ROTATING_TIMED_LOGGER_INTERVAL = 30
 
 
+class EiveCfdpFaultHandler(DefaultFaultHandlerBase):
+    def notice_of_suspension_cb(self, cond: ConditionCode):
+        pass
+
+    def notice_of_cancellation_cb(self, cond: ConditionCode):
+        pass
+
+    def abandoned_cb(self, cond: ConditionCode):
+        pass
+
+    def ignore_cb(self, cond: ConditionCode):
+        pass
+
+
+class EiveCfdpUser(CfdpUserBase):
+    def transaction_indication(self, transaction_id: TransactionId):
+        LOGGER.info(f"CFDP User: Start of File {transaction_id}")
+
+    def eof_sent_indication(self, transaction_id: TransactionId):
+        LOGGER.info(f"CFDP User: EOF sent for {transaction_id}")
+
+    def transaction_finished_indication(self, params: TransactionFinishedParams):
+        LOGGER.info(f"CFDP User: {params.transaction_id} finished")
+
+    def metadata_recv_indication(self, params: MetadataRecvParams):
+        pass
+
+    def file_segment_recv_indication(self, params: FileSegmentRecvdParams):
+        pass
+
+    def report_indication(self, transaction_id: TransactionId, status_report: any):
+        pass
+
+    def suspended_indication(
+        self, transaction_id: TransactionId, cond_code: ConditionCode
+    ):
+        pass
+
+    def resumed_indication(self, transaction_id: TransactionId, progress: int):
+        pass
+
+    def fault_indication(
+        self, transaction_id: TransactionId, cond_code: ConditionCode, progress: int
+    ):
+        pass
+
+    def abandoned_indication(
+        self, transaction_id: TransactionId, cond_code: ConditionCode, progress: int
+    ):
+        pass
+
+    def eof_recv_indication(self, transaction_id: TransactionId):
+        pass
+
+
 class PusHandler(SpecificApidHandlerBase):
     def __init__(
         self,
@@ -84,32 +176,52 @@ class UnknownApidHandler(GenericApidHandlerBase):
         LOGGER.warning(f"Packet with unknwon APID {apid} detected")
 
 
+class CfdpInCcsdsWrapper(SpecificApidHandlerBase):
+    def __init__(self, cfdp_in_ccsds_handler: CfdpInCcsdsHandler):
+        super().__init__(CFDP_APID, None)
+        self.handler = cfdp_in_ccsds_handler
+
+    def handle_tm(self, packet: bytes, _user_args: any):
+        ccsds_header_raw = packet[0:6]
+        sp_header = SpacePacketHeader.unpack(ccsds_header_raw)
+        pdu = packet[6:]
+        sp = SpacePacket(sp_header, sec_header=None, user_data=pdu)
+        self.handler.pass_packet(sp)
+
+
 class TcHandler(TcHandlerBase):
     def __init__(
         self,
         seq_count_provider: FileSeqCountProvider,
+        cfdp_in_ccsds_wrapper: CfdpInCcsdsWrapper,
         pus_verificator: PusVerificator,
         file_logger: logging.Logger,
         raw_logger: RawTmtcTimedLogWrapper,
         gui: bool,
     ):
         super().__init__()
+        self.cfdp_handler_started = False
+        self.cfdp_dest_id = CFDP_REMOTE_ENTITY_ID
         self.seq_count_provider = seq_count_provider
         self.pus_verificator = pus_verificator
         self.file_logger = file_logger
         self.raw_logger = raw_logger
         self.gui = gui
+        self.cfdp_done = False
         self.queue_helper = DefaultPusQueueHelper(
             queue_wrapper=None,
             pus_apid=PUS_APID,
             seq_cnt_provider=seq_count_provider,
             pus_verificator=pus_verificator,
         )
+        self.cfdp_in_ccsds_wrapper = cfdp_in_ccsds_wrapper
 
-    def feed_cb(self, info: ProcedureHelper, wrapper: FeedWrapper):
+    def feed_cb(self, info: ProcedureWrapper, wrapper: FeedWrapper):
         self.queue_helper.queue_wrapper = wrapper.queue_wrapper
         if info.proc_type == TcProcedureType.DEFAULT:
             handle_default_procedure(self, info.to_def_procedure(), self.queue_helper)
+        elif info.proc_type == TcProcedureType.CFDP:
+            self.handle_cfdp_procedure(info)
 
     def send_cb(self, send_params: SendCbParams):
         entry_helper = send_params.entry
@@ -130,17 +242,63 @@ class TcHandler(TcHandlerBase):
                     f"{get_current_time_string(True)}: {tc_info_string}"
                 )
                 send_params.com_if.send(raw_tc)
+            elif entry_helper.entry_type == TcQueueEntryType.CCSDS_TC:
+                cfdp_packet_in_ccsds = entry_helper.to_space_packet_entry()
+                send_params.com_if.send(cfdp_packet_in_ccsds.space_packet.pack())
         elif entry_helper.entry_type == TcQueueEntryType.LOG:
             log_entry = entry_helper.to_log_entry()
             LOGGER.info(log_entry.log_str)
             self.file_logger.info(log_entry.log_str)
 
-    def queue_finished_cb(self, info: ProcedureHelper):
+    def handle_cfdp_procedure(self, info: ProcedureWrapper):
-        if info is not None and info.proc_type == TcQueueEntryType.PUS_TC:
+        cfdp_procedure = info.to_cfdp_procedure()
-            def_proc = info.to_def_procedure()
+        if cfdp_procedure.cfdp_request_type == CfdpRequestType.PUT:
-            LOGGER.info(
+            if (
-                f"Finished queue for service {def_proc.service} and op code {def_proc.op_code}"
+                not self.cfdp_in_ccsds_wrapper.handler.put_request_pending()
-            )
+                and not self.cfdp_handler_started
+            ):
+                put_req = cfdp_procedure.request_wrapper.to_put_request()
+                put_req.cfg.destination_id = self.cfdp_dest_id
+                LOGGER.info(
+                    f"CFDP: Starting file put request with parameters:\n{put_req}"
+                )
+                self.cfdp_in_ccsds_wrapper.handler.cfdp_handler.put_request(put_req)
+                self.cfdp_handler_started = True
+
+                for source_pair, dest_pair in self.cfdp_in_ccsds_wrapper.handler:
+                    pdu, sp = source_pair
+                    pdu = cast(PduHolder, pdu)
+                    if pdu.is_file_directive:
+                        if pdu.pdu_directive_type == DirectiveType.METADATA_PDU:
+                            metadata = pdu.to_metadata_pdu()
+                            self.queue_helper.add_log_cmd(
+                                f"CFDP Source: Sending Metadata PDU for file with size "
+                                f"{metadata.file_size}"
+                            )
+                        elif pdu.pdu_directive_type == DirectiveType.EOF_PDU:
+                            self.queue_helper.add_log_cmd(
+                                f"CFDP Source: Sending EOF PDU"
+                            )
+                    else:
+                        fd_pdu = pdu.to_file_data_pdu()
+                        self.queue_helper.add_log_cmd(
+                            f"CFDP Source: Sending File Data PDU for segment at offset "
+                            f"{fd_pdu.offset} with length {len(fd_pdu.file_data)}"
+                        )
+                    self.queue_helper.add_ccsds_tc(sp)
+                    self.cfdp_in_ccsds_wrapper.handler.confirm_source_packet_sent()
+                self.cfdp_in_ccsds_wrapper.handler.source_handler.state_machine()
+
+    def queue_finished_cb(self, info: ProcedureWrapper):
+        if info is not None:
+            if info.proc_type == TcQueueEntryType.PUS_TC:
+                def_proc = info.to_def_procedure()
+                LOGGER.info(
+                    f"Finished queue for service {def_proc.service} and op code {def_proc.op_code}"
+                )
+            elif info.proc_type == TcProcedureType.CFDP:
+                LOGGER.info(f"Finished CFDP queue")
+                self.cfdp_done = True
 
 
 def setup_params() -> SetupWrapper:
@@ -148,28 +306,71 @@ def setup_params() -> SetupWrapper:
     print(f"-- spacepackets v{spacepackets.__version__} --")
     hook_obj = EiveHookObject(default_json_path())
     params = SetupParams()
-    parser_wrapper = ArgParserWrapper(hook_obj)
+    parser_wrapper = PreArgsParsingWrapper()
     parser_wrapper.create_default_parent_parser()
     parser_wrapper.create_default_parser()
     parser_wrapper.add_def_proc_and_cfdp_as_subparsers()
-    parser_wrapper.parse()
+    post_arg_parsing_wrapper = parser_wrapper.parse(hook_obj)
-    tmtccmd.init_printout(parser_wrapper.use_gui)
+    tmtccmd.init_printout(post_arg_parsing_wrapper.use_gui)
-    parser_wrapper.set_params(params)
+    use_prompts = not post_arg_parsing_wrapper.use_gui
+    proc_param_wrapper = ProcedureParamsWrapper()
+    if use_prompts:
+        post_arg_parsing_wrapper.set_params_with_prompts(params, proc_param_wrapper)
+    else:
+        post_arg_parsing_wrapper.set_params_without_prompts(params, proc_param_wrapper)
     params.apid = PUS_APID
-    setup_wrapper = SetupWrapper(hook_obj=hook_obj, setup_params=params)
+    setup_wrapper = SetupWrapper(
+        hook_obj=hook_obj, setup_params=params, proc_param_wrapper=proc_param_wrapper
+    )
     return setup_wrapper
 
 
-def setup_tmtc(
+def setup_cfdp_handler() -> CfdpInCcsdsWrapper:
+    fh_base = EiveCfdpFaultHandler()
+    cfdp_cfg = LocalEntityCfg(
+        local_entity_id=CFDP_LOCAL_ENTITY_ID,
+        indication_cfg=IndicationCfg(),
+        default_fault_handlers=fh_base,
+    )
+    remote_cfg = RemoteEntityCfg(
+        closure_requested=False,
+        entity_id=CFDP_REMOTE_ENTITY_ID,
+        max_file_segment_len=1024,
+        check_limit=None,
+        crc_on_transmission=False,
+        crc_type=ChecksumType.CRC_32,
+        default_transmission_mode=TransmissionMode.UNACKNOWLEDGED,
+    )
+    cfdp_seq_count_provider = FileSeqCountProvider(
+        max_bit_width=16, file_name=Path("seqcnt_cfdp_transaction.txt")
+    )
+    cfdp_ccsds_seq_count_provider = PusFileSeqCountProvider(
+        file_name=Path("seqcnt_cfdp_ccsds_.txt")
+    )
+    cfdp_user = EiveCfdpUser()
+    cfdp_in_ccsds_handler = CfdpInCcsdsHandler(
+        cfg=cfdp_cfg,
+        remote_cfgs=[remote_cfg],
+        ccsds_apid=CFDP_APID,
+        ccsds_seq_cnt_provider=cfdp_ccsds_seq_count_provider,
+        cfdp_seq_cnt_provider=cfdp_seq_count_provider,
+        user=cfdp_user,
+    )
+    return CfdpInCcsdsWrapper(cfdp_in_ccsds_handler)
+
+
+def setup_tmtc_handlers(
     verificator: PusVerificator,
     printer: FsfwTmTcPrinter,
     raw_logger: RawTmtcTimedLogWrapper,
     gui: bool,
 ) -> (CcsdsTmHandler, TcHandler):
+    cfdp_in_ccsds_wrapper = setup_cfdp_handler()
     verification_wrapper = VerificationWrapper(verificator, LOGGER, printer.file_logger)
     pus_handler = PusHandler(verification_wrapper, printer, raw_logger)
     ccsds_handler = CcsdsTmHandler(generic_handler=UnknownApidHandler(None))
     ccsds_handler.add_apid_handler(pus_handler)
+    ccsds_handler.add_apid_handler(cfdp_in_ccsds_wrapper)
     seq_count_provider = PusFileSeqCountProvider()
     tc_handler = TcHandler(
         seq_count_provider=seq_count_provider,
@@ -177,6 +378,7 @@ def setup_tmtc(
         file_logger=printer.file_logger,
         raw_logger=raw_logger,
         gui=gui,
+        cfdp_in_ccsds_wrapper=cfdp_in_ccsds_wrapper,
     )
     return ccsds_handler, tc_handler
 
@@ -186,8 +388,12 @@ def setup_backend(
     tc_handler: TcHandler,
     ccsds_handler: CcsdsTmHandler,
 ) -> BackendBase:
+    init_proc = params_to_procedure_conversion(setup_wrapper.proc_param_wrapper)
     tmtc_backend = tmtccmd.create_default_tmtc_backend(
-        setup_wrapper=setup_wrapper, tm_handler=ccsds_handler, tc_handler=tc_handler
+        setup_wrapper=setup_wrapper,
+        tm_handler=ccsds_handler,
+        tc_handler=tc_handler,
+        init_procedure=init_proc,
     )
     tmtccmd.start(tmtc_backend=tmtc_backend, hook_obj=setup_wrapper.hook_obj)
     return tmtc_backend
@@ -206,7 +412,7 @@ def main():
         interval=ROTATING_TIMED_LOGGER_INTERVAL,
     )
     pus_verificator = PusVerificator()
-    ccsds_handler, tc_handler = setup_tmtc(
+    ccsds_handler, tc_handler = setup_tmtc_handlers(
         pus_verificator, printer, raw_logger, setup_wrapper.params.use_gui
     )