implement ordered sets

mission/tmtc/PersistentTmStore.cpp

@@ -32,6 +32,57 @@ ReturnValue_t PersistentTmStore::cancelDump() {
   return returnvalue::OK;
 }
 
+ReturnValue_t PersistentTmStore::buildDumpSet(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds) {
+  using namespace std::filesystem;
+  std::error_code e;
+  dumpParams.orderedDumpFilestamps.clear();
+  for (auto const& fileOrDir : directory_iterator(basePath)) {
+    if (not fileOrDir.is_regular_file(e)) {
+      continue;
+    }
+    dumpParams.fileSize = std::filesystem::file_size(dumpParams.dirEntry.path(), e);
+    if (e) {
+      sif::error << "PersistentTmStore: Could not retrieve file size: " << e.message() << std::endl;
+      continue;
+    }
+    // sif::debug << "Path: " << dumpParams.dirEntry.path() << std::endl;
+
+    // File empty or can't even read CCSDS header.
+    if (dumpParams.fileSize <= 6) {
+      continue;
+    }
+    if (dumpParams.fileSize > fileBuf.size()) {
+      sif::error << "PersistentTmStore: File too large, is deleted" << std::endl;
+      triggerEvent(persTmStore::FILE_TOO_LARGE, dumpParams.fileSize, fileBuf.size());
+      std::filesystem::remove(dumpParams.dirEntry.path(), e);
+      continue;
+    }
+    const path& file = fileOrDir.path();
+    struct tm fileTime {};
+    if (pathToTime(file, fileTime) != returnvalue::OK) {
+      sif::error << "Time extraction for file " << file << "failed" << std::endl;
+      continue;
+    }
+    auto fileEpoch = static_cast<uint32_t>(timegm(&fileTime));
+    if ((fileEpoch > dumpParams.fromUnixTime) and
+        (fileEpoch + rolloverDiffSeconds <= dumpParams.untilUnixTime)) {
+      std::ifstream ifile(file, std::ios::binary);
+      if (ifile.bad()) {
+        sif::error << "PersistentTmStore: File is bad" << std::endl;
+        // TODO: Consider deleting file here?
+        continue;
+      }
+
+      // TODO: Check whether this is a suffixed file.
+      DumpIndex dumpIndex;
+      dumpIndex.epoch = fileEpoch;
+      dumpParams.orderedDumpFilestamps.emplace(dumpIndex);
+      return returnvalue::OK;
+    }
+  }
+  return returnvalue::OK;
+}
+
 ReturnValue_t PersistentTmStore::assignAndOrCreateMostRecentFile() {
   if (not activeFile.has_value()) {
     return createMostRecentFile(std::nullopt);
@@ -159,6 +210,10 @@ bool PersistentTmStore::updateBaseDir() {
   if (not exists(basePath, e)) {
     create_directories(basePath);
   }
+  // Each file will have the base name as a prefix again
+  path preparedFullFilePath = basePath / baseName;
+  basePathSize = preparedFullFilePath.string().length();
+  std::memcpy(filePathBuf.data(), preparedFullFilePath.c_str(), basePathSize);
   baseDirUninitialized = false;
   return true;
 }
@@ -189,12 +244,19 @@ ReturnValue_t PersistentTmStore::startDumpFromUpTo(uint32_t fromUnixSeconds,
   if (state == State::DUMPING) {
     return returnvalue::FAILED;
   }
-  dumpParams.dirIter = directory_iterator(basePath);
+  auto dirIter = directory_iterator(basePath);
-  if (dumpParams.dirIter == directory_iterator()) {
+  // Directory empty case.
+  if (dirIter == directory_iterator()) {
     return returnvalue::FAILED;
   }
   dumpParams.fromUnixTime = fromUnixSeconds;
   dumpParams.untilUnixTime = upToUnixSeconds;
+  buildDumpSet(fromUnixSeconds, upToUnixSeconds);
+  // No files in time window found.
+  if (dumpParams.orderedDumpFilestamps.empty()) {
+    return DUMP_DONE;
+  }
+  dumpParams.dumpIter = dumpParams.orderedDumpFilestamps.begin();
   state = State::DUMPING;
   return loadNextDumpFile();
 }
@@ -203,49 +265,23 @@ ReturnValue_t PersistentTmStore::loadNextDumpFile() {
   using namespace std::filesystem;
   dumpParams.currentSize = 0;
   std::error_code e;
-  for (; dumpParams.dirIter != directory_iterator(); dumpParams.dirIter++) {
+  for (; dumpParams.dumpIter != dumpParams.orderedDumpFilestamps.end(); dumpParams.dumpIter++) {
-    dumpParams.dirEntry = *dumpParams.dirIter;
+    DumpIndex dumpIndex = *dumpParams.dumpIter;
-    if (dumpParams.dirEntry.is_directory(e)) {
+    timeval tv{};
+    tv.tv_sec = dumpIndex.epoch;
+    size_t fullPathLength = 0;
+    createFileName(tv, dumpIndex.suffixIdx, fullPathLength);
+    path filePath(std::string(reinterpret_cast<const char*>(filePathBuf.data()), fullPathLength));
+    std::ifstream ifile(filePath, std::ios::binary);
+    if (ifile.bad()) {
+      sif::error << "PersistentTmStore: File is bad" << std::endl;
       continue;
     }
-    dumpParams.fileSize = std::filesystem::file_size(dumpParams.dirEntry.path(), e);
+    ifile.read(reinterpret_cast<char*>(fileBuf.data()),
-    if (e) {
+               static_cast<std::streamsize>(dumpParams.fileSize));
-      sif::error << "PersistentTmStore: Could not retrieve file size: " << e.message() << std::endl;
+    // Increment iterator for next cycle.
-      continue;
+    dumpParams.dumpIter++;
-    }
+    return returnvalue::OK;
-    // sif::debug << "Path: " << dumpParams.dirEntry.path() << std::endl;
-
-    // File empty or can't even read CCSDS header.
-    if (dumpParams.fileSize <= 6) {
-      continue;
-    }
-    if (dumpParams.fileSize > fileBuf.size()) {
-      sif::error << "PersistentTmStore: File too large, is deleted" << std::endl;
-      triggerEvent(persTmStore::FILE_TOO_LARGE, dumpParams.fileSize, fileBuf.size());
-      std::filesystem::remove(dumpParams.dirEntry.path(), e);
-      continue;
-    }
-    const path& file = dumpParams.dirEntry.path();
-    struct tm fileTime {};
-    if (pathToTime(file, fileTime) != returnvalue::OK) {
-      sif::error << "Time extraction for file " << file << "failed" << std::endl;
-      continue;
-    }
-    auto fileEpoch = static_cast<uint32_t>(timegm(&fileTime));
-    if ((fileEpoch > dumpParams.fromUnixTime) and
-        (fileEpoch + rolloverDiffSeconds <= dumpParams.untilUnixTime)) {
-      dumpParams.currentFileUnixStamp = fileEpoch;
-      std::ifstream ifile(file, std::ios::binary);
-      if (ifile.bad()) {
-        sif::error << "PersistentTmStore: File is bad" << std::endl;
-        continue;
-      }
-      ifile.read(reinterpret_cast<char*>(fileBuf.data()),
-                 static_cast<std::streamsize>(dumpParams.fileSize));
-      // Increment iterator for next cycle.
-      dumpParams.dirIter++;
-      return returnvalue::OK;
-    }
   }
   // Directory iterator was consumed and we are done.
   state = State::IDLE;
@@ -302,37 +338,9 @@ ReturnValue_t PersistentTmStore::pathToTime(const std::filesystem::path& path, s
 
 ReturnValue_t PersistentTmStore::createMostRecentFile(std::optional<uint8_t> suffix) {
   using namespace std::filesystem;
-  unsigned currentIdx = 0;
+  size_t currentIdx;
-  path pathStart = basePath / baseName;
+  createFileName(currentTv, suffix, currentIdx);
-  memcpy(fileBuf.data() + currentIdx, pathStart.c_str(), pathStart.string().length());
+  path newPath(std::string(reinterpret_cast<const char*>(filePathBuf.data()), currentIdx));
-  currentIdx += pathStart.string().length();
-  fileBuf[currentIdx] = '_';
-  currentIdx += 1;
-  time_t epoch = currentTv.tv_sec;
-  struct tm* time = gmtime(&epoch);
-  size_t writtenBytes = strftime(reinterpret_cast<char*>(fileBuf.data() + currentIdx),
-                                 fileBuf.size(), config::FILE_DATE_FORMAT, time);
-  if (writtenBytes == 0) {
-    sif::error << "PersistentTmStore::createMostRecentFile: Could not create file timestamp"
-               << std::endl;
-    return returnvalue::FAILED;
-  }
-  currentIdx += writtenBytes;
-  char* res = strcpy(reinterpret_cast<char*>(fileBuf.data() + currentIdx), ".bin");
-  if (res == nullptr) {
-    return returnvalue::FAILED;
-  }
-  currentIdx += 4;
-  if (suffix.has_value()) {
-    std::string fullSuffix = "." + std::to_string(suffix.value());
-    res = strcpy(reinterpret_cast<char*>(fileBuf.data() + currentIdx), fullSuffix.c_str());
-    if (res == nullptr) {
-      return returnvalue::FAILED;
-    }
-    currentIdx += fullSuffix.size();
-  }
-
-  path newPath(std::string(reinterpret_cast<const char*>(fileBuf.data()), currentIdx));
   std::ofstream of(newPath, std::ios::binary);
   activeFile = newPath;
   activeFileTv = currentTv;
@@ -354,3 +362,35 @@ void PersistentTmStore::getStartAndEndTimeCurrentOrLastDump(uint32_t& startTime,
   startTime = dumpParams.fromUnixTime;
   endTime = dumpParams.untilUnixTime;
 }
+
+ReturnValue_t PersistentTmStore::createFileName(timeval& tv, std::optional<uint8_t> suffix,
+                                                size_t& fullPathLength) {
+  unsigned currentIdx = basePathSize;
+  fileBuf[currentIdx] = '_';
+  currentIdx += 1;
+  time_t epoch = tv.tv_sec;
+  struct tm* time = gmtime(&epoch);
+  size_t writtenBytes = strftime(reinterpret_cast<char*>(filePathBuf.data() + currentIdx),
+                                 filePathBuf.size(), config::FILE_DATE_FORMAT, time);
+  if (writtenBytes == 0) {
+    sif::error << "PersistentTmStore::createMostRecentFile: Could not create file timestamp"
+               << std::endl;
+    return returnvalue::FAILED;
+  }
+  currentIdx += writtenBytes;
+  char* res = strcpy(reinterpret_cast<char*>(filePathBuf.data() + currentIdx), ".bin");
+  if (res == nullptr) {
+    return returnvalue::FAILED;
+  }
+  currentIdx += 4;
+  if (suffix.has_value()) {
+    std::string fullSuffix = "." + std::to_string(suffix.value());
+    res = strcpy(reinterpret_cast<char*>(filePathBuf.data() + currentIdx), fullSuffix.c_str());
+    if (res == nullptr) {
+      return returnvalue::FAILED;
+    }
+    currentIdx += fullSuffix.size();
+  }
+  fullPathLength = currentIdx;
+  return returnvalue::OK;
+}

mission/tmtc/PersistentTmStore.h

@@ -10,6 +10,7 @@
 #include <mission/memory/SdCardMountedIF.h>
 
 #include <filesystem>
+#include <set>
 
 #include "eive/eventSubsystemIds.h"
 #include "eive/resultClassIds.h"
@@ -37,6 +38,13 @@ struct PersistentTmStoreArgs {
   SdCardMountedIF& sdcMan;
 };
 
+struct DumpIndex {
+  uint32_t epoch;
+  uint8_t suffixIdx;
+  // Define a custom comparison function based on the epoch variable
+  bool operator<(const DumpIndex& other) const { return epoch < other.epoch; }
+};
+
 class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
  public:
   enum class State { IDLE, DUMPING };
@@ -96,7 +104,10 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
   std::string baseName;
   uint8_t currentSameSecNumber = 0;
   std::filesystem::path basePath;
+  // std::filesystem::path pathStart = basePath / baseName;
   uint32_t rolloverDiffSeconds = 0;
+  std::array<uint8_t, 524> filePathBuf{};
+  size_t basePathSize;
   std::array<uint8_t, MAX_FILESIZE> fileBuf{};
   timeval currentTv;
   timeval activeFileTv{};
@@ -106,8 +117,10 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
     uint32_t fromUnixTime = 0;
     uint32_t untilUnixTime = 0;
     uint32_t currentFileUnixStamp = 0;
-    std::filesystem::directory_iterator dirIter;
+    // std::filesystem::directory_iterator dirIter;
     std::filesystem::directory_entry dirEntry;
+    std::set<DumpIndex> orderedDumpFilestamps{};
+    std::set<DumpIndex>::iterator dumpIter;
     size_t fileSize = 0;
     size_t currentSize = 0;
   };
@@ -122,10 +135,12 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
   [[nodiscard]] MessageQueueId_t getCommandQueue() const override;
 
   void calcDiffSeconds(RolloverInterval intervalUnit, uint32_t intervalCount);
+  ReturnValue_t buildDumpSet(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds);
   ReturnValue_t createMostRecentFile(std::optional<uint8_t> suffix);
   static ReturnValue_t pathToTime(const std::filesystem::path& path, struct tm& time);
   void fileToPackets(const std::filesystem::path& path, uint32_t unixStamp);
   ReturnValue_t loadNextDumpFile();
+  ReturnValue_t createFileName(timeval& tv, std::optional<uint8_t> suffix, size_t& fullPathLength);
   bool updateBaseDir();
   ReturnValue_t assignAndOrCreateMostRecentFile();
 };