continued basic tm store impl

common/config/eive/eventSubsystemIds.h

@@ -36,6 +36,7 @@ enum : uint8_t {
   SCEX_HANDLER = 138,
   CONFIGHANDLER = 139,
   CORE = 140,
+  PERSISTENT_TM_STORE = 141,
   COMMON_SUBSYSTEM_ID_END
 
 };

fsfw

@@ -1 +1 @@
-Subproject commit 6af5274b688c2eb561a17d78e5417e6b843e4523
+Subproject commit 2646707d3fa1b568a6c99e8e0bd32585d729d162

mission/tmtc/PusTmFunnel.cpp

@@ -14,11 +14,13 @@ PusTmFunnel::PusTmFunnel(object_id_t objectId, TimeReaderIF &timeReader, Storage
     : TmFunnelBase(objectId, tmStore, tmMsgDepth, tcMsgDepth, ipcStore),
       timeReader(timeReader),
       miscStore(objects::MISC_TM_STORE, "tm", "misc", RolloverInterval::HOURLY, 2, currentTv,
-                sdcMan),
-      okStore(objects::OK_TM_STORE, "tm", "ok", RolloverInterval::MINUTELY, 30, currentTv, sdcMan),
+                tmStore, sdcMan),
+      okStore(objects::OK_TM_STORE, "tm", "ok", RolloverInterval::MINUTELY, 30, currentTv, tmStore,
+              sdcMan),
       notOkStore(objects::NOT_OK_TM_STORE, "tm", "nok", RolloverInterval::MINUTELY, 30, currentTv,
-                 sdcMan),
-      hkStore(objects::HK_TM_STORE, "tm", "hk", RolloverInterval::MINUTELY, 15, currentTv, sdcMan),
+                 tmStore, sdcMan),
+      hkStore(objects::HK_TM_STORE, "tm", "hk", RolloverInterval::MINUTELY, 15, currentTv, tmStore,
+              sdcMan),
       sdcMan(sdcMan) {
   Clock::getClock_timeval(&currentTv);
   Clock::getUptime(&lastTvUpdate);

mission/tmtc/TmStore.cpp

@@ -12,12 +12,13 @@ using namespace returnvalue;
 
 TmStore::TmStore(object_id_t objectId, const char* baseDir, std::string baseName,
                  RolloverInterval intervalUnit, uint32_t intervalCount, timeval& currentTv,
-                 SdCardMountedIF& sdcMan)
+                 StorageManagerIF& tmStore, SdCardMountedIF& sdcMan)
     : SystemObject(objectId),
       baseDir(std::move(baseDir)),
       baseName(std::move(baseName)),
       currentTv(currentTv),
-      sdcMan(sdcMan) {
+      sdcMan(sdcMan),
+      tmStore(tmStore) {
   calcDiffSeconds(intervalUnit, intervalCount);
 }
 
@@ -182,18 +183,89 @@ void TmStore::addServiceSubservice(uint8_t service, uint8_t subservice) {
   filter.serviceSubservices.value().push_back({service, subservice});
 }
 
-void TmStore::deleteUpTo(uint32_t unixSeconds) {}
+void TmStore::deleteUpTo(uint32_t unixSeconds) {
+  using namespace std::filesystem;
+  for (auto const& file : directory_iterator(basePath)) {
+    if (file.is_directory() or
+        (mostRecentFile.has_value() and (mostRecentFile.value() == file.path()))) {
+      continue;
+    }
+    Clock::TimeOfDay_t tod;
+    pathToTod(file.path(), tod);
+    timeval time;
+    ReturnValue_t result = Clock::convertTimeOfDayToTimeval(&tod, &time);
+    if (result != returnvalue::OK) {
+      sif::error << "TOD to time conversion failed for file " << file << std::endl;
+      continue;
+    }
+    if (time.tv_sec + rolloverDiffSeconds < unixSeconds) {
+      std::filesystem::remove(file.path());
+    }
+  }
+}
 
 void TmStore::dumpFromUpTo(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds,
                            TmFunnelBase& funnel) {
-  // TODO:
-  // 1. Find all files within the specified range
-  // 2. For each file, dump content into buffer, parse out space packets,
-  // 3. Store each packet into the TM store and send the message to all
-  //    destinations
+  using namespace std::filesystem;
+  for (auto const& file : directory_iterator(basePath)) {
+    if (file.is_directory() or
+        (mostRecentFile.has_value() and (mostRecentFile.value() == file.path()))) {
+      continue;
+    }
+    Clock::TimeOfDay_t tod;
+    pathToTod(file.path(), tod);
+    timeval time;
+    ReturnValue_t result = Clock::convertTimeOfDayToTimeval(&tod, &time);
+    if (result != returnvalue::OK) {
+      sif::error << "TOD to time conversion failed for file " << file << std::endl;
+      continue;
+    }
+    uint32_t timeUnsigned = static_cast<uint32_t>(time.tv_sec);
+    if (timeUnsigned > fromUnixSeconds && timeUnsigned + rolloverDiffSeconds < upToUnixSeconds) {
+      fileToPackets(file, timeUnsigned, funnel);
+    }
+  }
+}
+
+void TmStore::pathToTod(const std::filesystem::path& path, Clock::TimeOfDay_t& tod) {
+  auto pathStr = path.string();
+  size_t splitChar = pathStr.find("_");
+  auto timeOnlyStr = pathStr.substr(splitChar);
+  sscanf(timeOnlyStr.data(),
+         "%04" SCNu32 "-%02" SCNu32 "-%02" SCNu32 "T%02" SCNu32 "-%02" SCNu32 "-%02" SCNu32 "Z",
+         &tod.year, &tod.month, &tod.day, &tod.hour, &tod.minute, &tod.second);
+}
+
+void TmStore::fileToPackets(const std::filesystem::path& path, uint32_t unixStamp,
+                            TmFunnelBase& funnel) {
   store_address_t storeId;
   TmTcMessage message;
-  const uint8_t* packetData = nullptr;
-  size_t size = 0;
-  funnel.sendPacketToDestinations(storeId, message, packetData, size);
+  size_t size = std::filesystem::file_size(path);
+  if (size < 6) {
+    // Can't even read the CCSDS header
+    return;
+  }
+  std::ifstream ifile(path, std::ios::binary);
+  ifile.read(reinterpret_cast<char*>(fileBuf.data()), size);
+  size_t currentIdx = 0;
+  while (currentIdx < size) {
+    PusTmReader reader(&timeReader, fileBuf.data(), fileBuf.size());
+    // CRC check to fully ensure this is a valid TM
+    ReturnValue_t result = reader.parseDataWithCrcCheck();
+    if (result == returnvalue::OK) {
+      ReturnValue_t result =
+          tmStore.addData(&storeId, fileBuf.data() + currentIdx, reader.getFullPacketLen());
+      if (result != returnvalue::OK) {
+        continue;
+      }
+      funnel.sendPacketToDestinations(storeId, message, fileBuf.data() + currentIdx,
+                                      reader.getFullPacketLen());
+      currentIdx += reader.getFullPacketLen();
+    } else {
+      sif::error << "Parsing of PUS TM failed with code " << result << std::endl;
+      triggerEvent(POSSIBLE_FILE_CORRUPTION, result, unixStamp);
+      // Stop for now, do not really know where to continue and we do not trust the file anymore.
+      break;
+    }
+  }
 }

mission/tmtc/TmStore.h

@@ -2,6 +2,7 @@
 #define MISSION_TMTC_TMSTOREBACKEND_H_
 
 #include <fsfw/objectmanager/SystemObject.h>
+#include <fsfw/timemanager/CdsShortTimeStamper.h>
 #include <fsfw/tmstorage/TmStoreFrontendIF.h>
 #include <fsfw/tmtcpacket/pus/tm/PusTmReader.h>
 #include <fsfw/tmtcservices/AcceptsTelemetryIF.h>
@@ -10,6 +11,7 @@
 #include <filesystem>
 
 #include "TmFunnelBase.h"
+#include "eive/eventSubsystemIds.h"
 
 struct PacketFilter {
   std::optional<std::vector<uint16_t>> apid;
@@ -21,9 +23,16 @@ enum class RolloverInterval { MINUTELY, HOURLY, DAILY };
 
 class TmStore : public SystemObject {
  public:
+  static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::PERSISTENT_TM_STORE;
+
+  //! [EXPORT] : [COMMENT]
+  //! P1: Result code of TM packet parser.
+  //! P2: Timestamp of possibly corrupt file as a unix timestamp.
+  static constexpr Event POSSIBLE_FILE_CORRUPTION =
+      event::makeEvent(SUBSYSTEM_ID, 0, severity::LOW);
   TmStore(object_id_t objectId, const char* baseDir, std::string baseName,
           RolloverInterval intervalUnit, uint32_t intervalCount, timeval& currentTv,
-          SdCardMountedIF& sdcMan);
+          StorageManagerIF& tmStore, SdCardMountedIF& sdcMan);
 
   void addApid(uint16_t apid);
   void addService(uint8_t service);
@@ -44,6 +53,7 @@ class TmStore : public SystemObject {
    */
   MessageQueueId_t getCommandQueue();
   PacketFilter filter;
+  CdsShortTimeStamper timeReader;
   bool baseDirUninitialized = true;
   const char* baseDir;
   std::string baseName;
@@ -54,9 +64,12 @@ class TmStore : public SystemObject {
   std::optional<timeval> mostRecentTv;
   std::optional<std::filesystem::path> mostRecentFile;
   SdCardMountedIF& sdcMan;
+  StorageManagerIF& tmStore;
 
   void calcDiffSeconds(RolloverInterval intervalUnit, uint32_t intervalCount);
   void assignAndOrCreateMostRecentFile();
+  void pathToTod(const std::filesystem::path& path, Clock::TimeOfDay_t& tod);
+  void fileToPackets(const std::filesystem::path& path, uint32_t unixStamp, TmFunnelBase& funnel);
   ReturnValue_t storePacket(PusTmReader& reader);
 };