Merge pull request 'Ordered dumps' (#706) from ordered-dumps into main

Reviewed-on: #706
Reviewed-by: Marius Eggert <eggertm@irs.uni-stuttgart.de>

CHANGELOG.md

@@ -22,6 +22,10 @@ will consitute of a breaking change warranting a new major release:
 
 # [v5.1.0] to be released
 
+## Changed
+
+- Persistent TM store dumps are now performed in chronological order
+
 # [v5.0.0] 2023-06-26
 
 v3.3.1 and all following version will now be moved to v5.0.0 with the additional changes listed

mission/tmtc/PersistentTmStore.cpp

@@ -17,6 +17,8 @@
 
 using namespace returnvalue;
 
+static constexpr bool DEBUG_DUMPS = false;
+
 PersistentTmStore::PersistentTmStore(PersistentTmStoreArgs args)
     : SystemObject(args.objectId),
       tmStore(args.tmStore),
@@ -32,6 +34,91 @@ ReturnValue_t PersistentTmStore::cancelDump() {
   return returnvalue::OK;
 }
 
+ReturnValue_t PersistentTmStore::buildDumpSet(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds) {
+  using namespace std::filesystem;
+  std::error_code e;
+  dumpParams.orderedDumpFilestamps.clear();
+  for (auto const& fileOrDir : directory_iterator(basePath)) {
+    if (not fileOrDir.is_regular_file(e)) {
+      continue;
+    }
+    dumpParams.fileSize = std::filesystem::file_size(fileOrDir.path(), e);
+    if (e) {
+      sif::error << "PersistentTmStore: Could not retrieve file size: " << e.message() << std::endl;
+      continue;
+    }
+
+    // File empty or can't even read CCSDS header.
+    if (dumpParams.fileSize <= 6) {
+      continue;
+    }
+    if (dumpParams.fileSize > fileBuf.size()) {
+      sif::error << "PersistentTmStore: File too large, is deleted" << std::endl;
+      triggerEvent(persTmStore::FILE_TOO_LARGE, dumpParams.fileSize, fileBuf.size());
+      std::filesystem::remove(fileOrDir.path(), e);
+      continue;
+    }
+    const path& file = fileOrDir.path();
+    struct tm fileTime {};
+    if (pathToTime(file, fileTime) != returnvalue::OK) {
+      sif::error << "Time extraction for file " << file << "failed" << std::endl;
+      continue;
+    }
+    auto fileEpoch = static_cast<uint32_t>(timegm(&fileTime));
+    if ((fileEpoch > dumpParams.fromUnixTime) and
+        (fileEpoch + rolloverDiffSeconds <= dumpParams.untilUnixTime)) {
+      std::ifstream ifile(file, std::ios::binary);
+      if (ifile.bad()) {
+        sif::error << "PersistentTmStore: File is bad" << std::endl;
+        // TODO: Consider deleting file here?
+        continue;
+      }
+
+      if (DEBUG_DUMPS) {
+        sif::debug << "Inserting file " << fileOrDir.path() << std::endl;
+      }
+      DumpIndex dumpIndex;
+      dumpIndex.epoch = fileEpoch;
+      // Multiple files for the same time are supported via a special suffix. We simply count the
+      // number of copies and later try to dump the same number of files with the additional
+      // suffixes
+      auto iter = dumpParams.orderedDumpFilestamps.find(dumpIndex);
+      if (iter != dumpParams.orderedDumpFilestamps.end()) {
+        dumpIndex.epoch = iter->epoch;
+        dumpIndex.additionalFiles = iter->additionalFiles + 1;
+        dumpParams.orderedDumpFilestamps.erase(dumpIndex);
+      } else {
+        dumpIndex.additionalFiles = 0;
+      }
+      dumpParams.orderedDumpFilestamps.emplace(dumpIndex);
+    }
+  }
+  return returnvalue::OK;
+}
+
+std::optional<uint8_t> PersistentTmStore::extractSuffix(const std::string& pathStr) {
+  std::string numberStr;
+  // Find the position of the dot at the end of the file path
+  size_t dotPos = pathStr.find_last_of('.');
+  if ((dotPos < pathStr.length()) and not std::isdigit(pathStr[dotPos + 1])) {
+    return std::nullopt;
+  }
+  // Extract the substring after the dot
+  numberStr = pathStr.substr(dotPos + 1);
+  std::optional<uint8_t> number;
+  try {
+    number = std::stoi(numberStr);
+    if (number.value() > std::numeric_limits<uint8_t>::max()) {
+      return std::nullopt;
+    }
+
+  } catch (std::invalid_argument& exception) {
+    sif::error << "PersistentTmStore::extractSuffix: Exception " << exception.what()
+               << ", invald input string: " << numberStr << std::endl;
+  }
+  return number;
+}
+
 ReturnValue_t PersistentTmStore::assignAndOrCreateMostRecentFile() {
   if (not activeFile.has_value()) {
     return createMostRecentFile(std::nullopt);
@@ -159,6 +246,12 @@ bool PersistentTmStore::updateBaseDir() {
   if (not exists(basePath, e)) {
     create_directories(basePath);
   }
+  // Each file will have the base name as a prefix again
+  path preparedFullFilePath = basePath / baseName;
+  basePathSize = preparedFullFilePath.string().length();
+  std::memcpy(filePathBuf.data(), preparedFullFilePath.c_str(), basePathSize);
+  filePathBuf[basePathSize] = '_';
+  basePathSize += 1;
   baseDirUninitialized = false;
   return true;
 }
@@ -189,12 +282,20 @@ ReturnValue_t PersistentTmStore::startDumpFromUpTo(uint32_t fromUnixSeconds,
   if (state == State::DUMPING) {
     return returnvalue::FAILED;
   }
-  dumpParams.dirIter = directory_iterator(basePath);
+  auto dirIter = directory_iterator(basePath);
-  if (dumpParams.dirIter == directory_iterator()) {
+  // Directory empty case.
+  if (dirIter == directory_iterator()) {
     return returnvalue::FAILED;
   }
   dumpParams.fromUnixTime = fromUnixSeconds;
   dumpParams.untilUnixTime = upToUnixSeconds;
+  buildDumpSet(fromUnixSeconds, upToUnixSeconds);
+  // No files in time window found.
+  if (dumpParams.orderedDumpFilestamps.empty()) {
+    return DUMP_DONE;
+  }
+  dumpParams.dumpIter = dumpParams.orderedDumpFilestamps.begin();
+  dumpParams.currentSameFileIdx = std::nullopt;
   state = State::DUMPING;
   return loadNextDumpFile();
 }
@@ -203,50 +304,55 @@ ReturnValue_t PersistentTmStore::loadNextDumpFile() {
   using namespace std::filesystem;
   dumpParams.currentSize = 0;
   std::error_code e;
-  for (; dumpParams.dirIter != directory_iterator(); dumpParams.dirIter++) {
+  // Handle iteration, which does not necessarily have to increment the set iterator
-    dumpParams.dirEntry = *dumpParams.dirIter;
+  // if there are multiple files for a given timestamp.
-    if (dumpParams.dirEntry.is_directory(e)) {
+  auto handleIteration = [&](DumpIndex& dumpIndex) {
-      continue;
+    if (dumpIndex.additionalFiles > 0) {
+      if (not dumpParams.currentSameFileIdx.has_value()) {
+        // Initialize the file index and stay on same file
+        dumpParams.currentSameFileIdx = 0;
+        return;
+      } else if (dumpParams.currentSameFileIdx.value() < dumpIndex.additionalFiles - 1) {
+        dumpParams.currentSameFileIdx = dumpParams.currentSameFileIdx.value() + 1;
+        return;
       }
-    dumpParams.fileSize = std::filesystem::file_size(dumpParams.dirEntry.path(), e);
+    }
+    // File will change, reset this field for correct state-keeping.
+    dumpParams.currentSameFileIdx = std::nullopt;
+    // Increment iterator for next cycle.
+    dumpParams.dumpIter++;
+  };
+  while (dumpParams.dumpIter != dumpParams.orderedDumpFilestamps.end()) {
+    DumpIndex dumpIndex = *dumpParams.dumpIter;
+    timeval tv{};
+    tv.tv_sec = dumpIndex.epoch;
+    size_t fullPathLength = 0;
+    createFileName(tv, dumpParams.currentSameFileIdx, fullPathLength);
+    dumpParams.currentFile =
+        path(std::string(reinterpret_cast<const char*>(filePathBuf.data()), fullPathLength));
+    if (DEBUG_DUMPS) {
+      sif::debug << baseName << " dump: Loading " << dumpParams.currentFile << std::endl;
+    }
+    dumpParams.fileSize = std::filesystem::file_size(dumpParams.currentFile, e);
     if (e) {
-      sif::error << "PersistentTmStore: Could not retrieve file size: " << e.message() << std::endl;
+      // TODO: Event?
+      sif::error << "PersistentTmStore: Could not load next dump file: " << e.message()
+                 << std::endl;
+      handleIteration(dumpIndex);
       continue;
     }
-    // sif::debug << "Path: " << dumpParams.dirEntry.path() << std::endl;
+    std::ifstream ifile(dumpParams.currentFile, std::ios::binary);
-
-    // File empty or can't even read CCSDS header.
-    if (dumpParams.fileSize <= 6) {
-      continue;
-    }
-    if (dumpParams.fileSize > fileBuf.size()) {
-      sif::error << "PersistentTmStore: File too large, is deleted" << std::endl;
-      triggerEvent(persTmStore::FILE_TOO_LARGE, dumpParams.fileSize, fileBuf.size());
-      std::filesystem::remove(dumpParams.dirEntry.path(), e);
-      continue;
-    }
-    const path& file = dumpParams.dirEntry.path();
-    struct tm fileTime {};
-    if (pathToTime(file, fileTime) != returnvalue::OK) {
-      sif::error << "Time extraction for file " << file << "failed" << std::endl;
-      continue;
-    }
-    auto fileEpoch = static_cast<uint32_t>(timegm(&fileTime));
-    if ((fileEpoch > dumpParams.fromUnixTime) and
-        (fileEpoch + rolloverDiffSeconds <= dumpParams.untilUnixTime)) {
-      dumpParams.currentFileUnixStamp = fileEpoch;
-      std::ifstream ifile(file, std::ios::binary);
     if (ifile.bad()) {
-        sif::error << "PersistentTmStore: File is bad" << std::endl;
+      sif::error << "PersistentTmStore: File is bad. Loading next file" << std::endl;
+      handleIteration(dumpIndex);
       continue;
     }
     ifile.read(reinterpret_cast<char*>(fileBuf.data()),
                static_cast<std::streamsize>(dumpParams.fileSize));
-      // Increment iterator for next cycle.
+    // Next file is loaded, exit the loop.
-      dumpParams.dirIter++;
+    handleIteration(dumpIndex);
     return returnvalue::OK;
   }
-  }
   // Directory iterator was consumed and we are done.
   state = State::IDLE;
   return DUMP_DONE;
@@ -267,8 +373,8 @@ ReturnValue_t PersistentTmStore::getNextDumpPacket(PusTmReader& reader, bool& fi
     // Delete the file and load next. Could use better algorithm to partially
     // restore the file dump, but for now do not trust the file.
     std::error_code e;
-    std::filesystem::remove(dumpParams.dirEntry.path().c_str(), e);
+    std::filesystem::remove(dumpParams.currentFile.c_str(), e);
-    if (dumpParams.dirEntry.path() == activeFile) {
+    if (dumpParams.currentFile == activeFile) {
       activeFile == std::nullopt;
       assignAndOrCreateMostRecentFile();
     }
@@ -302,37 +408,9 @@ ReturnValue_t PersistentTmStore::pathToTime(const std::filesystem::path& path, s
 
 ReturnValue_t PersistentTmStore::createMostRecentFile(std::optional<uint8_t> suffix) {
   using namespace std::filesystem;
-  unsigned currentIdx = 0;
+  size_t currentIdx;
-  path pathStart = basePath / baseName;
+  createFileName(currentTv, suffix, currentIdx);
-  memcpy(fileBuf.data() + currentIdx, pathStart.c_str(), pathStart.string().length());
+  path newPath(std::string(reinterpret_cast<const char*>(filePathBuf.data()), currentIdx));
-  currentIdx += pathStart.string().length();
-  fileBuf[currentIdx] = '_';
-  currentIdx += 1;
-  time_t epoch = currentTv.tv_sec;
-  struct tm* time = gmtime(&epoch);
-  size_t writtenBytes = strftime(reinterpret_cast<char*>(fileBuf.data() + currentIdx),
-                                 fileBuf.size(), config::FILE_DATE_FORMAT, time);
-  if (writtenBytes == 0) {
-    sif::error << "PersistentTmStore::createMostRecentFile: Could not create file timestamp"
-               << std::endl;
-    return returnvalue::FAILED;
-  }
-  currentIdx += writtenBytes;
-  char* res = strcpy(reinterpret_cast<char*>(fileBuf.data() + currentIdx), ".bin");
-  if (res == nullptr) {
-    return returnvalue::FAILED;
-  }
-  currentIdx += 4;
-  if (suffix.has_value()) {
-    std::string fullSuffix = "." + std::to_string(suffix.value());
-    res = strcpy(reinterpret_cast<char*>(fileBuf.data() + currentIdx), fullSuffix.c_str());
-    if (res == nullptr) {
-      return returnvalue::FAILED;
-    }
-    currentIdx += fullSuffix.size();
-  }
-
-  path newPath(std::string(reinterpret_cast<const char*>(fileBuf.data()), currentIdx));
   std::ofstream of(newPath, std::ios::binary);
   activeFile = newPath;
   activeFileTv = currentTv;
@@ -354,3 +432,33 @@ void PersistentTmStore::getStartAndEndTimeCurrentOrLastDump(uint32_t& startTime,
   startTime = dumpParams.fromUnixTime;
   endTime = dumpParams.untilUnixTime;
 }
+
+ReturnValue_t PersistentTmStore::createFileName(timeval& tv, std::optional<uint8_t> suffix,
+                                                size_t& fullPathLength) {
+  unsigned currentIdx = basePathSize;
+  time_t epoch = tv.tv_sec;
+  struct tm* time = gmtime(&epoch);
+  size_t writtenBytes = strftime(reinterpret_cast<char*>(filePathBuf.data() + currentIdx),
+                                 filePathBuf.size(), config::FILE_DATE_FORMAT, time);
+  if (writtenBytes == 0) {
+    sif::error << "PersistentTmStore::createMostRecentFile: Could not create file timestamp"
+               << std::endl;
+    return returnvalue::FAILED;
+  }
+  currentIdx += writtenBytes;
+  char* res = strcpy(reinterpret_cast<char*>(filePathBuf.data() + currentIdx), ".bin");
+  if (res == nullptr) {
+    return returnvalue::FAILED;
+  }
+  currentIdx += 4;
+  if (suffix.has_value()) {
+    std::string fullSuffix = "." + std::to_string(suffix.value());
+    res = strcpy(reinterpret_cast<char*>(filePathBuf.data() + currentIdx), fullSuffix.c_str());
+    if (res == nullptr) {
+      return returnvalue::FAILED;
+    }
+    currentIdx += fullSuffix.size();
+  }
+  fullPathLength = currentIdx;
+  return returnvalue::OK;
+}

mission/tmtc/PersistentTmStore.h

@@ -10,6 +10,7 @@
 #include <mission/memory/SdCardMountedIF.h>
 
 #include <filesystem>
+#include <set>
 
 #include "eive/eventSubsystemIds.h"
 #include "eive/resultClassIds.h"
@@ -37,6 +38,14 @@ struct PersistentTmStoreArgs {
   SdCardMountedIF& sdcMan;
 };
 
+struct DumpIndex {
+  uint32_t epoch = 0;
+  // Number of additional files with a suffix like .0, .1 etc.
+  uint8_t additionalFiles = 0;
+  // Define a custom comparison function based on the epoch variable
+  bool operator<(const DumpIndex& other) const { return epoch < other.epoch; }
+};
+
 class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
  public:
   enum class State { IDLE, DUMPING };
@@ -96,7 +105,10 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
   std::string baseName;
   uint8_t currentSameSecNumber = 0;
   std::filesystem::path basePath;
+  // std::filesystem::path pathStart = basePath / baseName;
   uint32_t rolloverDiffSeconds = 0;
+  std::array<uint8_t, 524> filePathBuf{};
+  size_t basePathSize;
   std::array<uint8_t, MAX_FILESIZE> fileBuf{};
   timeval currentTv;
   timeval activeFileTv{};
@@ -106,8 +118,10 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
     uint32_t fromUnixTime = 0;
     uint32_t untilUnixTime = 0;
     uint32_t currentFileUnixStamp = 0;
-    std::filesystem::directory_iterator dirIter;
+    std::filesystem::path currentFile;
-    std::filesystem::directory_entry dirEntry;
+    std::set<DumpIndex> orderedDumpFilestamps{};
+    std::set<DumpIndex>::iterator dumpIter;
+    std::optional<uint8_t> currentSameFileIdx = 0;
     size_t fileSize = 0;
     size_t currentSize = 0;
   };
@@ -122,10 +136,13 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
   [[nodiscard]] MessageQueueId_t getCommandQueue() const override;
 
   void calcDiffSeconds(RolloverInterval intervalUnit, uint32_t intervalCount);
+  ReturnValue_t buildDumpSet(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds);
   ReturnValue_t createMostRecentFile(std::optional<uint8_t> suffix);
   static ReturnValue_t pathToTime(const std::filesystem::path& path, struct tm& time);
   void fileToPackets(const std::filesystem::path& path, uint32_t unixStamp);
   ReturnValue_t loadNextDumpFile();
+  ReturnValue_t createFileName(timeval& tv, std::optional<uint8_t> suffix, size_t& fullPathLength);
+  std::optional<uint8_t> extractSuffix(const std::string& pathStr);
   bool updateBaseDir();
   ReturnValue_t assignAndOrCreateMostRecentFile();
 };

unittest/CMakeLists.txt

@@ -4,7 +4,7 @@ add_subdirectory(mocks)
 target_sources(${UNITTEST_NAME} PRIVATE
     main.cpp
     testEnvironment.cpp
-    testStampInFilename.cpp
+    testGenericFilesystem.cpp
     hdlcEncodingRw.cpp
     printChar.cpp
 )

unittest/testGenericFilesystem.cpp

@@ -0,0 +1,43 @@
+
+#include <catch2/catch_test_macros.hpp>
+#include <cinttypes>
+
+#include "fsfw/timemanager/Clock.h"
+
+uint8_t extractSuffix(const std::string& pathStr) {
+  std::string numberStr;
+  // Find the position of the dot at the end of the file path
+  size_t dotPos = pathStr.find_last_of('.');
+  if (dotPos != std::string::npos && dotPos < pathStr.length() - 1) {
+    // Extract the substring after the dot
+    numberStr = pathStr.substr(dotPos + 1);
+  }
+  int number = std::stoi(numberStr);
+  if (number < 0 or number > std::numeric_limits<uint8_t>::max()) {
+    return 0;
+  }
+  return static_cast<uint8_t>(number);
+}
+
+TEST_CASE("Stamp in Filename", "[Stamp In Filename]") {
+  Clock::TimeOfDay_t tod;
+  std::string baseName = "verif";
+  std::string pathStr = "verif_2022-05-25T16:55:23Z.bin";
+  unsigned int underscorePos = pathStr.find_last_of('_');
+  std::string stampStr = pathStr.substr(underscorePos + 1);
+  float seconds = 0.0;
+  char* prefix = nullptr;
+  int count =
+      sscanf(stampStr.c_str(),
+             "%4" SCNu32 "-%2" SCNu32 "-%2" SCNu32 "T%2" SCNu32 ":%2" SCNu32 ":%2" SCNu32 "Z",
+             &tod.year, &tod.month, &tod.day, &tod.hour, &tod.minute, &tod.second);
+  static_cast<void>(count);
+  CHECK(count == 6);
+}
+
+TEST_CASE("Suffix Extraction") {
+  std::string pathStr = "/mnt/sd0/tm/hk/hk-some-stamp.bin.0";
+  CHECK(extractSuffix(pathStr) == 0);
+  pathStr = "/mnt/sd0/tm/hk/hk-some-stamp.bin.2";
+  CHECK(extractSuffix(pathStr) == 2);
+}

unittest/testStampInFilename.cpp

@@ -1,21 +0,0 @@
-
-#include <catch2/catch_test_macros.hpp>
-#include <cinttypes>
-
-#include "fsfw/timemanager/Clock.h"
-
-TEST_CASE("Stamp in Filename", "[Stamp In Filename]") {
-  Clock::TimeOfDay_t tod;
-  std::string baseName = "verif";
-  std::string pathStr = "verif_2022-05-25T16:55:23Z.bin";
-  unsigned int underscorePos = pathStr.find_last_of('_');
-  std::string stampStr = pathStr.substr(underscorePos + 1);
-  float seconds = 0.0;
-  char* prefix = nullptr;
-  int count =
-      sscanf(stampStr.c_str(),
-             "%4" SCNu32 "-%2" SCNu32 "-%2" SCNu32 "T%2" SCNu32 ":%2" SCNu32 ":%2" SCNu32 "Z",
-             &tod.year, &tod.month, &tod.day, &tod.hour, &tod.minute, &tod.second);
-  static_cast<void>(count);
-  CHECK(count == 6);
-}