Merge pull request 'Bump tmtccmd and fix CFDP' (#297) from bump-tmtccmd-fix-cfdp into main

Reviewed-on: #297
Reviewed-by: Marius Eggert <eggertm@irs.uni-stuttgart.de>

eive_tmtc/cfdp/handler.py

@@ -62,13 +62,7 @@ class CfdpHandler:
         )
 
     def put_request(self, request: PutRequest):
-        if not self.remote_cfg_table.get_cfg(request.destination_id):
+        self.source_handler.put_request(request)
-            raise ValueError(
-                f"No remote CFDP config found for entity ID {request.destination_id}"
-            )
-        self.source_handler.put_request(
-            request, self.remote_cfg_table.get_cfg(request.destination_id)  # type: ignore
-        )
 
     def pull_next_source_packet(self) -> Optional[PduHolder]:
         res = self.source_handler.state_machine()

eive_tmtc/pus_tc/cmd_demux.py

@@ -4,7 +4,7 @@
 import logging
 from typing import List, cast
 
-from tmtccmd import DefaultProcedureInfo
+from tmtccmd import TreeCommandingProcedure
 from tmtccmd.tmtc import DefaultPusQueueHelper
 from tmtccmd.util import ObjectIdU32
 
@@ -78,7 +78,7 @@ from eive_tmtc.utility.input_helper import InputHelper
 
 
 def handle_pus_procedure(
-    info: DefaultProcedureInfo,
+    info: TreeCommandingProcedure,
     queue_helper: DefaultPusQueueHelper,
 ):
     cmd_path = info.cmd_path

eive_tmtc/pus_tc/tc_handler.py

@@ -68,8 +68,8 @@ class TcHandler(TcHandlerBase):
 
     def feed_cb(self, info: ProcedureWrapper, wrapper: FeedWrapper):
         self.queue_helper.queue_wrapper = wrapper.queue_wrapper
-        if info.proc_type == TcProcedureType.DEFAULT:
+        if info.proc_type == TcProcedureType.TREE_COMMANDING:
-            handle_pus_procedure(info.to_def_procedure(), self.queue_helper)
+            handle_pus_procedure(info.to_tree_commanding_procedure(), self.queue_helper)
         elif info.proc_type == TcProcedureType.CFDP:
             self.handle_cfdp_procedure(info)
 
@@ -155,7 +155,7 @@ class TcHandler(TcHandlerBase):
     def queue_finished_cb(self, info: ProcedureWrapper):
         if info is not None:
             if info.proc_type == TcQueueEntryType.PUS_TC:
-                def_proc = info.to_def_procedure()
+                def_proc = info.to_tree_commanding_procedure()
                 _LOGGER.info(f"Finished queue for command {def_proc.cmd_path}")
             elif info.proc_type == TcProcedureType.CFDP:
                 _LOGGER.info("Finished CFDP queue")

eive_tmtc/pus_tm/action_reply_handler.py

@@ -1,5 +1,8 @@
 import logging
 import struct
+
+from spacepackets.ecss import PusTm
+from tmtccmd.pus.s8_fsfw_action_defs import CustomSubservice
 from eive_tmtc.config.object_ids import (
     ACU_HANDLER_ID,
     PDU_1_HANDLER_ID,
@@ -20,31 +23,34 @@ from eive_tmtc.tmtc.payload.ploc_supervisor import SupvActionId
 from eive_tmtc.tmtc.acs.star_tracker import handle_star_tracker_action_replies
 from eive_tmtc.tmtc.acs.acs_ctrl import handle_acs_ctrl_action_replies
 from eive_tmtc.tmtc.power.tm import handle_get_param_data_reply
-from tmtccmd.pus.s8_fsfw_action import Service8FsfwTm
-from tmtccmd.fsfw.tmtc_printer import FsfwTmTcPrinter
 from spacepackets.ccsds.time import CdsShortTimestamp
-from tmtccmd.util import ObjectIdDictT
+from tmtccmd.util import ObjectIdBase, ObjectIdDictT, ObjectIdU32
 
 _LOGGER = logging.getLogger(__name__)
 
 
-def handle_action_reply(
+_LOGGER = logging.getLogger(__name__)
-    raw_tm: bytes, printer: FsfwTmTcPrinter, obj_id_dict: ObjectIdDictT
+
+
+def handle_action_service_tm(
+    raw_tm: bytes, pw: PrintWrapper, obj_id_dict: ObjectIdDictT
 ):
     """Core Action reply handler
     :return:
     """
-    tm_packet = Service8FsfwTm.unpack(
+    tm_packet = PusTm.unpack(raw_tm, timestamp_len=CdsShortTimestamp.TIMESTAMP_SIZE)
-        raw_telemetry=raw_tm, time_reader=CdsShortTimestamp.empty()
+    if len(tm_packet.source_data) < 8:
+        _LOGGER.warning(
+            "received action service reply with source data smaller than 8 bytes"
         )
-    object_id = obj_id_dict.get(tm_packet.source_object_id_as_bytes)
+        return
-    pw = PrintWrapper(printer.file_logger)
+    object_id_raw = struct.unpack("!I", tm_packet.source_data[0:4])[0]
-    custom_data = tm_packet.custom_data
+    action_id = struct.unpack("!I", tm_packet.source_data[4:8])[0]
-    action_id = tm_packet.action_id
+    object_id = obj_id_dict.get(object_id_raw)
-    generic_print_str = printer.generic_action_packet_tm_print(
+    custom_data = tm_packet.source_data[8:]
-        packet=tm_packet, obj_id=object_id
+    if object_id is None:
-    )
+        object_id = ObjectIdU32(object_id_raw, "Unknown ID")
-    pw.dlog(generic_print_str)
+    if tm_packet.subservice == CustomSubservice.TM_DATA_REPLY:
         if object_id.as_bytes == IMTQ_HANDLER_ID:
             return handle_imtq_replies(action_id, pw, custom_data)
         elif object_id.as_bytes == PLOC_MPSOC_ID:
@@ -65,11 +71,26 @@ def handle_action_reply(
         ]:
             return handle_get_param_data_reply(object_id, action_id, pw, custom_data)
         else:
-        pw.dlog(f"No dedicated action reply handler found for reply from {object_id}")
+            # TODO: Could add a handler here depending on action ID and object ID.
-        pw.dlog(f"Raw Data: {tm_packet.custom_data.hex(sep=',')}")
+            handle_action_data_reply(tm_packet, object_id, action_id, pw)
+    else:
+        pw.dlog(
+            f"service 8 packet from {object_id} with action ID {action_id} "
+            f"and unknown subservice {tm_packet.subservice}"
+        )
 
 
-def handle_imtq_replies(action_id: int, pw: PrintWrapper, custom_data: bytearray):
+def handle_action_data_reply(
+    tm_packet: PusTm, named_obj_id: ObjectIdBase, action_id: int, printer: PrintWrapper
+):
+    print_string = (
+        f"service 8 data reply from {named_obj_id} with action ID {action_id} "
+        f"and data size {len(tm_packet.tm_data[8:])}"
+    )
+    printer.dlog(print_string)
+
+
+def handle_imtq_replies(action_id: int, pw: PrintWrapper, custom_data: bytes):
     if action_id == struct.unpack("!I", ImtqActionId.get_commanded_dipole)[0]:
         header_list = [
             "Commanded X-Dipole",
@@ -82,7 +103,7 @@ def handle_imtq_replies(action_id: int, pw: PrintWrapper, custom_data: bytearray
         pw.dlog(f"{content_list}")
 
 
-def handle_supervisor_replies(action_id: int, pw: PrintWrapper, custom_data: bytearray):
+def handle_supervisor_replies(action_id: int, pw: PrintWrapper, custom_data: bytes):
     if action_id == SupvActionId.DUMP_MRAM:
         header_list = ["MRAM Dump"]
         content_list = [custom_data[: len(custom_data)]]

eive_tmtc/pus_tm/event_handler.py

@@ -25,7 +25,7 @@ def handle_event_packet(  # noqa C901: Complexity okay here
 ):  # noqa C901: Complexity okay here
     if PRINT_RAW_EVENTS_B64_STR:
         print(f"PUS Event TM Base64: {base64.b64encode(raw_tm)}")
-    tm = Service5Tm.unpack(data=raw_tm, time_reader=CdsShortTimestamp.empty())
+    tm = Service5Tm.unpack(data=raw_tm, timestamp_len=CdsShortTimestamp.TIMESTAMP_SIZE)
     event_dict = get_event_dict()
     event_def = tm.event_definition
     info = event_dict.get(event_def.event_id)
@@ -40,10 +40,10 @@ def handle_event_packet(  # noqa C901: Complexity okay here
         obj_name = event_def.reporter_id.hex(sep=",")
     else:
         obj_name = obj_id_obj.name
-    assert tm.time_provider is not None
+    timestamp = CdsShortTimestamp.unpack(tm.timestamp)
     generic_event_string = (
         f"Object {obj_name} generated Event {info.name} (ID: {event_def.event_id:#04x})"
-        f" at {tm.time_provider.as_date_time()}"
+        f" at {timestamp.as_date_time()}"
     )
     _LOGGER.info(generic_event_string)
     pw.file_logger.info(

eive_tmtc/pus_tm/hk.py

@@ -2,20 +2,17 @@ import uuid
 import dataclasses
 import datetime
 import sqlite3
-from tmtccmd.pus.tm.s3_fsfw_hk import Service3FsfwTm
+from spacepackets.ecss.tm import CdsShortTimestamp, PusTm
 
 
 @dataclasses.dataclass
 class HkTmInfo:
     packet_uuid: uuid.UUID
-    hk_packet: Service3FsfwTm
+    hk_packet: PusTm
+    set_id: int
     db_con: sqlite3.Connection
     hk_data: bytes
 
     @property
     def packet_datetime(self) -> datetime.datetime:
-        return self.hk_packet.pus_tm.time_provider.as_datetime()
+        return CdsShortTimestamp.unpack(self.hk_packet.timestamp).as_datetime()
-
-    @property
-    def set_id(self) -> int:
-        return self.hk_packet.set_id

eive_tmtc/pus_tm/hk_handler.py

@@ -2,10 +2,14 @@
 
 import dataclasses
 import logging
-import base64  # noqa
+import base64
 import sqlite3
+import struct
 from typing import List, cast
 from uuid import UUID
+
+from spacepackets.ccsds.time import CdsShortTimestamp
+from spacepackets.ecss import PusTm
 from eive_tmtc.config.definitions import PRINT_RAW_HK_B64_STR
 from eive_tmtc.pus_tm.hk import HkTmInfo
 
@@ -23,9 +27,6 @@ from eive_tmtc.tmtc.power.pwr_ctrl import handle_pwr_ctrl_hk_data
 from eive_tmtc.tmtc.com.syrlinks_handler import handle_syrlinks_hk_data
 from eive_tmtc.tmtc.tcs import handle_thermal_controller_hk_data
 from eive_tmtc.tmtc.tcs.tmp1075 import handle_tmp_1075_hk_data
-from tmtccmd.pus.tm.s3_fsfw_hk import (
-    Service3FsfwTm,
-)
 from tmtccmd.pus.tm.s3_hk_base import HkContentType
 from tmtccmd.util.obj_id import ObjectIdU32, ObjectIdDictT
 
@@ -68,17 +69,20 @@ def handle_hk_packet(
     hk_level: int,
     db_con: sqlite3.Connection,
 ):
-    tm_packet = Service3FsfwTm.unpack(raw_telemetry=raw_tm, custom_hk_handling=False)
+    tm_packet = PusTm.unpack(raw_tm, CdsShortTimestamp.TIMESTAMP_SIZE)
-    named_obj_id = cast(ObjectIdU32, obj_id_dict.get(tm_packet.object_id.as_bytes))
+    obj_id_raw = struct.unpack("!I", tm_packet.tm_data[0:4])[0]
+    named_obj_id = cast(ObjectIdU32, obj_id_dict.get(obj_id_raw))
     if named_obj_id is None:
-        named_obj_id = tm_packet.object_id
+        named_obj_id = ObjectIdU32(obj_id_raw, "Unknown ID")
     if tm_packet.subservice == 25 or tm_packet.subservice == 26:
+        set_id = struct.unpack("!I", tm_packet.tm_data[4:8])[0]
         hk_data = tm_packet.tm_data[8:]
         if named_obj_id.as_bytes in hk_filter.object_ids:
             if PRINT_RAW_HK_B64_STR:
                 print(f"PUS TM Base64: {base64.b64encode(raw_tm)}")
             handle_regular_hk_print(
                 printer=printer,
+                set_id=set_id,
                 packet_uuid=packet_uuid,
                 object_id=named_obj_id,
                 hk_packet=tm_packet,
@@ -89,11 +93,12 @@ def handle_hk_packet(
         try:
             if hk_level >= 1:
                 printer.generic_hk_tm_print(
-                    HkContentType.HK, named_obj_id, tm_packet.set_id, hk_data
+                    HkContentType.HK, named_obj_id, set_id, hk_data
                 )
             if hk_level >= 1:
                 handle_regular_hk_print(
                     printer=printer,
+                    set_id=set_id,
                     packet_uuid=packet_uuid,
                     object_id=named_obj_id,
                     hk_packet=tm_packet,
@@ -109,7 +114,8 @@ def handle_hk_packet(
 
 
 def handle_regular_hk_print(  # noqa C901: Complexity okay here
-    hk_packet: Service3FsfwTm,
+    hk_packet: PusTm,
+    set_id: int,
     packet_uuid: UUID,
     hk_data: bytes,
     db: sqlite3.Connection,
@@ -117,12 +123,15 @@ def handle_regular_hk_print(  # noqa C901: Complexity okay here
     printer: FsfwTmTcPrinter,
 ):
     objb = object_id.as_bytes
-    set_id = hk_packet.set_id
     hk_info = HkTmInfo(
-        packet_uuid=packet_uuid, hk_packet=hk_packet, db_con=db, hk_data=hk_data
+        packet_uuid=packet_uuid,
+        hk_packet=hk_packet,
+        db_con=db,
+        hk_data=hk_data,
+        set_id=set_id,
     )
-    assert hk_packet.pus_tm.time_provider is not None
+    timestamp = CdsShortTimestamp.unpack(hk_packet.timestamp)
-    packet_dt = hk_packet.pus_tm.time_provider.as_date_time()
+    packet_dt = timestamp.as_datetime()
     pw = PrintWrapper(printer.file_logger)
     """This function is called when a Service 3 Housekeeping packet is received."""
     if objb in [obj_ids.RW1_ID, obj_ids.RW2_ID, obj_ids.RW3_ID, obj_ids.RW4_ID]:
@@ -139,7 +148,7 @@ def handle_regular_hk_print(  # noqa C901: Complexity okay here
             pw=pw,
             set_id=set_id,
             hk_data=hk_data,
-            packet_time=packet_dt,
+            packet_time=timestamp.as_datetime(),
         )
     elif objb == obj_ids.PCDU_HANDLER_ID:
         return handle_pcdu_hk(pw=pw, set_id=set_id, hk_data=hk_data)

eive_tmtc/pus_tm/pus_handler.py

@@ -22,7 +22,7 @@ from eive_tmtc.config.definitions import TM_DB_PATH, PUS_APID
 
 from eive_tmtc.config.object_ids import get_object_ids
 
-from .action_reply_handler import handle_action_reply
+from .action_reply_handler import handle_action_service_tm
 from .defs import PrintWrapper
 from .event_handler import handle_event_packet
 from .hk_handler import HkFilter, handle_hk_packet
@@ -63,15 +63,22 @@ class PusHandler(SpecificApidHandlerBase):
             _LOGGER.warning("Detected packet shorter than 8 bytes!")
             return
         try:
-            tm_packet = PusTelemetry.unpack(packet, CdsShortTimestamp.empty())
+            tm_packet = PusTelemetry.unpack(packet, CdsShortTimestamp.TIMESTAMP_SIZE)
             # _LOGGER.info(f"Sequence count: {tm_packet.seq_count}")
         except ValueError as value_error:
             _LOGGER.warning(f"{value_error}")
             _LOGGER.warning("Could not generate PUS TM object from raw data")
             _LOGGER.warning(f"Raw Packet: [{packet.hex(sep=',')}], REPR: {packet!r}")
             return
+        timestamp = CdsShortTimestamp.unpack(tm_packet.timestamp)
         db_con = sqlite3.connect(TM_DB_PATH)
-        self._store_packet_in_db(db_con, packet, tm_packet, packet_uuid)
+        self._store_packet_in_db(
+            db_con=db_con,
+            packet=packet,
+            tm_packet=tm_packet,
+            timestamp=timestamp,
+            packet_uuid=packet_uuid,
+        )
         service = tm_packet.service
         dedicated_handler = True
         if service == 1:
@@ -89,12 +96,12 @@ class PusHandler(SpecificApidHandlerBase):
         elif service == 5:
             handle_event_packet(raw_tm=packet, pw=self.pw)
         elif service == 8:
-            handle_action_reply(
+            handle_action_service_tm(
-                raw_tm=packet, printer=self.printer, obj_id_dict=self.obj_id_dict
+                raw_tm=packet, pw=self.pw, obj_id_dict=self.obj_id_dict
             )
         elif service == 17:
             pus17_tm = Service17Tm.unpack(
-                data=packet, time_reader=CdsShortTimestamp.empty()
+                data=packet, timestamp_len=CdsShortTimestamp.TIMESTAMP_SIZE
             )
             if pus17_tm.subservice == 2:
                 self.verif_wrapper.dlog("Received Ping Reply TM[17,2]")
@@ -119,11 +126,11 @@ class PusHandler(SpecificApidHandlerBase):
         self,
         db_con: sqlite3.Connection,
         packet: bytes,
+        timestamp: CdsShortTimestamp,
         tm_packet: PusTelemetry,
         packet_uuid: uuid.UUID,
     ):
         cursor = db_con.cursor()
-        assert tm_packet.time_provider is not None
         cursor.execute(
             """
             CREATE TABLE IF NOT EXISTS pus_tm(
@@ -139,7 +146,7 @@ class PusHandler(SpecificApidHandlerBase):
             "INSERT INTO pus_tm VALUES(?, ?, ?, ?, ?, ?)",
             (
                 str(packet_uuid),
-                tm_packet.time_provider.as_datetime(),
+                timestamp.as_datetime(),
                 tm_packet.service,
                 tm_packet.subservice,
                 len(packet),
@@ -150,7 +157,7 @@ class PusHandler(SpecificApidHandlerBase):
 
     def _handle_param_packet(self, raw_data: bytes, tm_packet: PusTelemetry):
         param_packet = Service20FsfwTm.unpack(
-            raw_telemetry=raw_data, time_reader=CdsShortTimestamp.empty()
+            raw_telemetry=raw_data, timestamp_len=CdsShortTimestamp.TIMESTAMP_SIZE
         )
         if tm_packet.subservice == ParamSubservice.TM_DUMP_REPLY:
             param_wrapper = Service20ParamDumpWrapper(param_tm=param_packet)

eive_tmtc/pus_tm/verification_handler.py

@@ -17,7 +17,7 @@ def handle_service_1_fsfw_packet(wrapper: VerificationWrapper, raw_tm: bytes):
         )
     # Error code with length 2 is FSFW specific
     tm_packet = Service1Tm.unpack(
-        data=raw_tm, params=UnpackParams(CdsShortTimestamp.empty(), 1, 2)
+        data=raw_tm, params=UnpackParams(CdsShortTimestamp.TIMESTAMP_SIZE, 1, 2)
     )
     fsfw_wrapper = Service1FsfwWrapper(tm_packet)
     res = wrapper.verificator.add_tm(tm_packet)

eive_tmtc/tmtc/payload/ploc_mpsoc.py

@@ -717,7 +717,7 @@ class DirElement:
     size: int
 
 
-def handle_mpsoc_data_reply(action_id: int, pw: PrintWrapper, custom_data: bytearray):
+def handle_mpsoc_data_reply(action_id: int, pw: PrintWrapper, custom_data: bytes):
     if action_id == ActionId.TM_MEM_READ_RPT:
         header_list = [
             "PLOC Memory Address",

eive_tmtc/tmtc/power/tm.py

@@ -537,7 +537,7 @@ def handle_acu_hk_data(pw: PrintWrapper, set_id: int, hk_data: bytes):
 
 
 def handle_get_param_data_reply(
-    obj_id: ObjectIdBase, action_id: int, pw: PrintWrapper, custom_data: bytearray
+    obj_id: ObjectIdBase, action_id: int, pw: PrintWrapper, custom_data: bytes
 ):
     if action_id == GomspaceDeviceActionId.PARAM_GET:
         pw.dlog(f"Parameter Get Request received for object {obj_id}")

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "eive-tmtc"
 description = "TMTC Commander EIVE"
 readme = "README.md"
-version = "7.0.0"
+version = "7.1.0"
 requires-python = ">=3.10"
 license = {text = "Apache-2.0"}
 authors = [
@@ -29,9 +29,9 @@ classifiers = [
   "Topic :: Scientific/Engineering"
 ]
 dependencies = [
-  "tmtccmd == 8.0.0rc1",
+  "tmtccmd ~= 8.0",
   "cfdp-py~=0.1.0",
-  # "tmtccmd @ git+https://github.com/robamu-org/tmtccmd@main",
+  # "tmtccmd @ git+https://github.com/robamu-org/tmtccmd@prep_v8.0.0",
   "python-dateutil ~= 2.8",
 ]
 
@@ -45,7 +45,7 @@ include-package-data = true
 [tool.setuptools.packages]
 find = {}
 
-[tool.ruff]
+[tool.ruff.lint]
 ignore = ["E501"]
-[tool.ruff.extend-per-file-ignores]
+[tool.ruff.lint.extend-per-file-ignores]
 "__init__.py" = ["F401"]

tmtcc.py

@@ -97,8 +97,8 @@ def setup_params() -> Tuple[SetupWrapper, int]:
         hk_level = int(post_arg_parsing_wrapper.args_raw.hk)
     else:
         hk_level = 0
-    if params.app_params.print_tree:
+    if params.cmd_params.print_tree:
-        perform_tree_printout(params.app_params, hook_obj.get_command_definitions())
+        perform_tree_printout(params.cmd_params, hook_obj.get_command_definitions())
         sys.exit(0)
     params.apid = PUS_APID
     if params.com_if is None: