Persistent Tm Store now has dump state

common/config/eive/resultClassIds.h

@@ -42,6 +42,7 @@ enum commonClassIds : uint8_t {
   ACS_DETUMBLE,             // ACSDTB
   SD_CARD_MANAGER,          // SDMA
   LOCAL_PARAM_HANDLER,      // LPH
+  PERSISTENT_TM_STORE,      // PTM
   COMMON_CLASS_ID_END       // [EXPORT] : [END]
 };
 }

mission/tmtc/PersistentTmStore.cpp

@@ -1,6 +1,7 @@
 #include "PersistentTmStore.h"
 
 #include <mission/memory/SdCardMountedIF.h>
+#include <mission/tmtc/DirectTmSinkIF.h>
 
 #include <algorithm>
 #include <cinttypes>
@@ -172,60 +173,122 @@ ReturnValue_t PersistentTmStore::startDumpFromUpTo(uint32_t fromUnixSeconds,
   if (state == State::DUMPING) {
     return returnvalue::FAILED;
   }
-  activeDumpDirIter = directory_iterator(basePath);
+  dumpParams.dirIter = directory_iterator(basePath);
+  dumpParams.fromUnixTime = fromUnixSeconds;
+  dumpParams.untilUnixTime = upToUnixSeconds;
   state = State::DUMPING;
+  if (loadNextDumpFile() == DUMP_DONE) {
+    // State will be set inside the function loading the next file.
+    return DUMP_DONE;
+  }
   return returnvalue::OK;
-  //  for (auto const& file : directory_iterator(basePath)) {
-  //    if (file.is_directory()) {
-  //      continue;
-  //    }
-  //    struct tm fileTime {};
-  //    if (pathToTm(file.path(), fileTime) != returnvalue::OK) {
-  //      sif::error << "Time extraction for file " << file << "failed" << std::endl;
-  //      continue;
-  //    }
-  //    auto fileEpoch = static_cast<uint32_t>(timegm(&fileTime));
-  //    if ((fileEpoch > fromUnixSeconds) and (fileEpoch + rolloverDiffSeconds <= upToUnixSeconds))
-  //    {
-  //      fileToPackets(file, fileEpoch);
-  //    }
-  //  }
 }
 
-void PersistentTmStore::fileToPackets(const std::filesystem::path& path, uint32_t unixStamp) {
+ReturnValue_t PersistentTmStore::loadNextDumpFile() {
-  store_address_t storeId;
+  using namespace std::filesystem;
-  TmTcMessage message;
+  std::error_code e;
-  size_t size = std::filesystem::file_size(path);
+  for (; dumpParams.dirIter != directory_iterator(); dumpParams.dirIter++) {
-  if (size < 6) {
+    if (dumpParams.dirEntry.is_directory(e)) {
-    // Can't even read the CCSDS header
+      continue;
-    return;
+    }
-  }
+    dumpParams.fileSize = std::filesystem::file_size(dumpParams.dirEntry.path());
-  std::ifstream ifile(path, std::ios::binary);
+    // Can't even read CCSDS header.
-  ifile.read(reinterpret_cast<char*>(fileBuf.data()), static_cast<std::streamsize>(size));
+    if (dumpParams.fileSize <= 6) {
-  size_t currentIdx = 0;
+      continue;
-  while (currentIdx < size) {
+    }
-    PusTmReader reader(&timeReader, fileBuf.data(), fileBuf.size());
+    if (dumpParams.fileSize > fileBuf.size()) {
-    // CRC check to fully ensure this is a valid TM
+      sif::error << "PersistentTmStore: File too large, is deleted" << std::endl;
-    ReturnValue_t result = reader.parseDataWithCrcCheck();
+      triggerEvent(FILE_TOO_LARGE, dumpParams.fileSize, fileBuf.size());
-    if (result == returnvalue::OK) {
+      std::remove(dumpParams.dirEntry.path().c_str());
-      // TODO: Blow the data out to the VC directly. Use IF function to do this.
+      continue;
-      //      result = tmStore.addData(&storeId, fileBuf.data() + currentIdx,
+    }
-      //      reader.getFullPacketLen()); if (result != returnvalue::OK) {
+    const path& file = dumpParams.dirEntry.path();
-      //        continue;
+    struct tm fileTime {};
-      //      }
+    if (pathToTime(file, fileTime) != returnvalue::OK) {
-      //      funnel.sendPacketToLiveDestinations(storeId, message, fileBuf.data() + currentIdx,
+      sif::error << "Time extraction for file " << file << "failed" << std::endl;
-      //                                      reader.getFullPacketLen());
+      continue;
-      currentIdx += reader.getFullPacketLen();
+    }
-    } else {
+    auto fileEpoch = static_cast<uint32_t>(timegm(&fileTime));
-      sif::error << "Parsing of PUS TM failed with code " << result << std::endl;
+    if ((fileEpoch > dumpParams.fromUnixTime) and
-      triggerEvent(POSSIBLE_FILE_CORRUPTION, result, unixStamp);
+        (fileEpoch + rolloverDiffSeconds <= dumpParams.untilUnixTime)) {
-      // Stop for now, do not really know where to continue and we do not trust the file anymore.
+      dumpParams.currentSize = 0;
+      dumpParams.currentFileUnixStamp = fileEpoch;
+      std::ifstream ifile(file, std::ios::binary);
+      ifile.read(reinterpret_cast<char*>(fileBuf.data()),
+                 static_cast<std::streamsize>(dumpParams.fileSize));
       break;
     }
   }
+  if (dumpParams.dirIter == directory_iterator()) {
+    state = State::IDLE;
+    return DUMP_DONE;
+  }
+  return returnvalue::OK;
 }
 
-ReturnValue_t PersistentTmStore::dumpNextPacket(size_t& dumpedLen) {}
+// void PersistentTmStore::fileToPackets(const std::filesystem::path& path, uint32_t unixStamp) {
+//   store_address_t storeId;
+//   TmTcMessage message;
+//   size_t size = std::filesystem::file_size(path);
+//   if (size < 6) {
+//     // Can't even read the CCSDS header
+//     return;
+//   }
+//   std::ifstream ifile(path, std::ios::binary);
+//   ifile.read(reinterpret_cast<char*>(fileBuf.data()), static_cast<std::streamsize>(size));
+//   size_t currentIdx = 0;
+//   while (currentIdx < size) {
+//     PusTmReader reader(&timeReader, fileBuf.data(), fileBuf.size());
+//     // CRC check to fully ensure this is a valid TM
+//     ReturnValue_t result = reader.parseDataWithCrcCheck();
+//     if (result == returnvalue::OK) {
+//       // TODO: Blow the data out to the VC directly. Use IF function to do this.
+//       //      result = tmStore.addData(&storeId, fileBuf.data() + currentIdx,
+//       //      reader.getFullPacketLen()); if (result != returnvalue::OK) {
+//       //        continue;
+//       //      }
+//       //      funnel.sendPacketToLiveDestinations(storeId, message, fileBuf.data() + currentIdx,
+//       //                                      reader.getFullPacketLen());
+//       currentIdx += reader.getFullPacketLen();
+//     } else {
+//       sif::error << "Parsing of PUS TM failed with code " << result << std::endl;
+//       triggerEvent(POSSIBLE_FILE_CORRUPTION, result, unixStamp);
+//       // Stop for now, do not really know where to continue and we do not trust the file anymore.
+//       break;
+//     }
+//   }
+// }
+
+ReturnValue_t PersistentTmStore::dumpNextPacket(DirectTmSinkIF& tmSink, size_t& dumpedLen) {
+  if (state == State::IDLE) {
+    return returnvalue::FAILED;
+  }
+  PusTmReader reader(&timeReader, fileBuf.data() + dumpParams.currentSize,
+                     fileBuf.size() - dumpParams.currentSize);
+  // CRC check to fully ensure this is a valid TM
+  ReturnValue_t result = reader.parseDataWithCrcCheck();
+  if (result == returnvalue::OK) {
+    result = tmSink.write(fileBuf.data() + dumpParams.currentSize, reader.getFullPacketLen());
+    if (result != returnvalue::OK) {
+      // TODO: Event?
+      sif::error << "PersistentTmStore: Writing to TM sink failed" << std::endl;
+    }
+    dumpParams.currentSize += reader.getFullPacketLen();
+    dumpedLen = reader.getFullPacketLen();
+    if (dumpParams.currentSize >= dumpParams.fileSize) {
+      return loadNextDumpFile();
+    }
+  } else {
+    sif::error << "Parsing of PUS TM failed with code " << result << std::endl;
+    triggerEvent(POSSIBLE_FILE_CORRUPTION, result, dumpParams.currentFileUnixStamp);
+    // Delete the file and load next. Could use better algorithm to partially
+    // restore the file dump, but for now do not trust the file.
+    dumpedLen = 0;
+    std::remove(dumpParams.dirEntry.path().c_str());
+    return loadNextDumpFile();
+  }
+  return returnvalue::OK;
+}
 
 ReturnValue_t PersistentTmStore::pathToTime(const std::filesystem::path& path, struct tm& time) {
   auto pathStr = path.string();

mission/tmtc/PersistentTmStore.h

@@ -7,17 +7,22 @@
 #include <fsfw/tmtcpacket/pus/tm/PusTmReader.h>
 #include <fsfw/tmtcservices/AcceptsTelemetryIF.h>
 #include <mission/memory/SdCardMountedIF.h>
+#include <mission/tmtc/DirectTmSinkIF.h>
 
 #include <filesystem>
 
 #include "TmFunnelBase.h"
 #include "eive/eventSubsystemIds.h"
+#include "eive/resultClassIds.h"
 
 enum class RolloverInterval { MINUTELY, HOURLY, DAILY };
 
 class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
  public:
   enum class State { IDLE, DUMPING };
+  static constexpr uint8_t INTERFACE_ID = CLASS_ID::PERSISTENT_TM_STORE;
+  static constexpr ReturnValue_t DUMP_DONE = returnvalue::makeCode(INTERFACE_ID, 0);
+
   static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::PERSISTENT_TM_STORE;
 
   //! [EXPORT] : [COMMENT]
@@ -25,6 +30,9 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
   //! P2: Timestamp of possibly corrupt file as a unix timestamp.
   static constexpr Event POSSIBLE_FILE_CORRUPTION =
       event::makeEvent(SUBSYSTEM_ID, 0, severity::LOW);
+  //! [EXPORT] : [COMMENT] File in store too large. P1: Detected file size
+  //! P2: Allowed file size
+  static constexpr Event FILE_TOO_LARGE = event::makeEvent(SUBSYSTEM_ID, 1, severity::LOW);
   PersistentTmStore(object_id_t objectId, const char* baseDir, std::string baseName,
                     RolloverInterval intervalUnit, uint32_t intervalCount,
                     StorageManagerIF& tmStore, SdCardMountedIF& sdcMan);
@@ -35,7 +43,7 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
   void deleteUpTo(uint32_t unixSeconds);
   ReturnValue_t startDumpFrom(uint32_t fromUnixSeconds);
   ReturnValue_t startDumpFromUpTo(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds);
-  ReturnValue_t dumpNextPacket(size_t& dumpedLen);
+  ReturnValue_t dumpNextPacket(DirectTmSinkIF& tmSink, size_t& dumpedLen);
 
   // ReturnValue_t passPacket(PusTmReader& reader);
   ReturnValue_t storePacket(PusTmReader& reader);
@@ -46,6 +54,9 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
   // ISO8601 timestamp.
   static constexpr char FILE_DATE_FORMAT[] = "%FT%H%M%SZ";
 
+  //! [EXPORT] : [SKIP]
+  static constexpr ReturnValue_t INVALID_FILE_DETECTED_AND_DELETED = returnvalue::makeCode(2, 1);
+
   MessageQueueIF* tcQueue;
   State state = State::IDLE;
   // PacketFilter filter;
@@ -59,10 +70,17 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
   std::array<uint8_t, MAX_FILESIZE> fileBuf{};
   timeval currentTv;
   timeval activeFileTv{};
-  std::filesystem::directory_iterator activeDumpDirIter;
+
-  std::filesystem::path activeDumpFile;
+  struct ActiveDumpParams {
-  size_t activeDumpFileSize = 0;
+    uint32_t fromUnixTime = 0;
-  size_t activeDumpCurrentSize = 0;
+    uint32_t untilUnixTime = 0;
+    uint32_t currentFileUnixStamp = 0;
+    std::filesystem::directory_iterator dirIter;
+    std::filesystem::directory_entry dirEntry;
+    size_t fileSize = 0;
+    size_t currentSize = 0;
+  };
+  ActiveDumpParams dumpParams;
   std::optional<std::filesystem::path> activeFile;
   SdCardMountedIF& sdcMan;
   StorageManagerIF& tmStore;
@@ -77,6 +95,7 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
   ReturnValue_t createMostRecentFile(std::optional<uint8_t> suffix);
   static ReturnValue_t pathToTime(const std::filesystem::path& path, struct tm& time);
   void fileToPackets(const std::filesystem::path& path, uint32_t unixStamp);
+  ReturnValue_t loadNextDumpFile();
   bool updateBaseDir();
   ReturnValue_t assignAndOrCreateMostRecentFile();
 };