diff --git a/CHANGELOG.md b/CHANGELOG.md index 45c4bacb..02b5e20f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ will consitute of a breaking change warranting a new major release: # [v5.1.0] to be released +## Changed + +- Persistent TM store dumps are now performed in chronological order. + ## Added - Sequence counters for PUS and CFDP packets are now stored persistently across graceful reboots. diff --git a/fsfw b/fsfw index c3572e31..7e8845f2 160000 --- a/fsfw +++ b/fsfw @@ -1 +1 @@ -Subproject commit c3572e31a8d6845442cf8e54802d94477b2db64e +Subproject commit 7e8845f2c2565b1486ff37be0519d3e15b4e1a13 diff --git a/mission/tmtc/PersistentTmStore.cpp b/mission/tmtc/PersistentTmStore.cpp index 08a7ea45..acef0b3a 100644 --- a/mission/tmtc/PersistentTmStore.cpp +++ b/mission/tmtc/PersistentTmStore.cpp @@ -17,6 +17,8 @@ using namespace returnvalue; +static constexpr bool DEBUG_DUMPS = false; + PersistentTmStore::PersistentTmStore(PersistentTmStoreArgs args) : SystemObject(args.objectId), tmStore(args.tmStore), @@ -32,6 +34,91 @@ ReturnValue_t PersistentTmStore::cancelDump() { return returnvalue::OK; } +ReturnValue_t PersistentTmStore::buildDumpSet(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds) { + using namespace std::filesystem; + std::error_code e; + dumpParams.orderedDumpFilestamps.clear(); + for (auto const& fileOrDir : directory_iterator(basePath)) { + if (not fileOrDir.is_regular_file(e)) { + continue; + } + dumpParams.fileSize = std::filesystem::file_size(fileOrDir.path(), e); + if (e) { + sif::error << "PersistentTmStore: Could not retrieve file size: " << e.message() << std::endl; + continue; + } + + // File empty or can't even read CCSDS header. + if (dumpParams.fileSize <= 6) { + continue; + } + if (dumpParams.fileSize > fileBuf.size()) { + sif::error << "PersistentTmStore: File too large, is deleted" << std::endl; + triggerEvent(persTmStore::FILE_TOO_LARGE, dumpParams.fileSize, fileBuf.size()); + std::filesystem::remove(fileOrDir.path(), e); + continue; + } + const path& file = fileOrDir.path(); + struct tm fileTime {}; + if (pathToTime(file, fileTime) != returnvalue::OK) { + sif::error << "Time extraction for file " << file << "failed" << std::endl; + continue; + } + auto fileEpoch = static_cast(timegm(&fileTime)); + if ((fileEpoch > dumpParams.fromUnixTime) and + (fileEpoch + rolloverDiffSeconds <= dumpParams.untilUnixTime)) { + std::ifstream ifile(file, std::ios::binary); + if (ifile.bad()) { + sif::error << "PersistentTmStore: File is bad" << std::endl; + // TODO: Consider deleting file here? + continue; + } + + if (DEBUG_DUMPS) { + sif::debug << "Inserting file " << fileOrDir.path() << std::endl; + } + DumpIndex dumpIndex; + dumpIndex.epoch = fileEpoch; + // Multiple files for the same time are supported via a special suffix. We simply count the + // number of copies and later try to dump the same number of files with the additional + // suffixes + auto iter = dumpParams.orderedDumpFilestamps.find(dumpIndex); + if (iter != dumpParams.orderedDumpFilestamps.end()) { + dumpIndex.epoch = iter->epoch; + dumpIndex.additionalFiles = iter->additionalFiles + 1; + dumpParams.orderedDumpFilestamps.erase(dumpIndex); + } else { + dumpIndex.additionalFiles = 0; + } + dumpParams.orderedDumpFilestamps.emplace(dumpIndex); + } + } + return returnvalue::OK; +} + +std::optional PersistentTmStore::extractSuffix(const std::string& pathStr) { + std::string numberStr; + // Find the position of the dot at the end of the file path + size_t dotPos = pathStr.find_last_of('.'); + if ((dotPos < pathStr.length()) and not std::isdigit(pathStr[dotPos + 1])) { + return std::nullopt; + } + // Extract the substring after the dot + numberStr = pathStr.substr(dotPos + 1); + std::optional number; + try { + number = std::stoi(numberStr); + if (number.value() > std::numeric_limits::max()) { + return std::nullopt; + } + + } catch (std::invalid_argument& exception) { + sif::error << "PersistentTmStore::extractSuffix: Exception " << exception.what() + << ", invald input string: " << numberStr << std::endl; + } + return number; +} + ReturnValue_t PersistentTmStore::assignAndOrCreateMostRecentFile() { if (not activeFile.has_value()) { return createMostRecentFile(std::nullopt); @@ -159,6 +246,12 @@ bool PersistentTmStore::updateBaseDir() { if (not exists(basePath, e)) { create_directories(basePath); } + // Each file will have the base name as a prefix again + path preparedFullFilePath = basePath / baseName; + basePathSize = preparedFullFilePath.string().length(); + std::memcpy(filePathBuf.data(), preparedFullFilePath.c_str(), basePathSize); + filePathBuf[basePathSize] = '_'; + basePathSize += 1; baseDirUninitialized = false; return true; } @@ -189,12 +282,20 @@ ReturnValue_t PersistentTmStore::startDumpFromUpTo(uint32_t fromUnixSeconds, if (state == State::DUMPING) { return returnvalue::FAILED; } - dumpParams.dirIter = directory_iterator(basePath); - if (dumpParams.dirIter == directory_iterator()) { + auto dirIter = directory_iterator(basePath); + // Directory empty case. + if (dirIter == directory_iterator()) { return returnvalue::FAILED; } dumpParams.fromUnixTime = fromUnixSeconds; dumpParams.untilUnixTime = upToUnixSeconds; + buildDumpSet(fromUnixSeconds, upToUnixSeconds); + // No files in time window found. + if (dumpParams.orderedDumpFilestamps.empty()) { + return DUMP_DONE; + } + dumpParams.dumpIter = dumpParams.orderedDumpFilestamps.begin(); + dumpParams.currentSameFileIdx = std::nullopt; state = State::DUMPING; return loadNextDumpFile(); } @@ -203,49 +304,54 @@ ReturnValue_t PersistentTmStore::loadNextDumpFile() { using namespace std::filesystem; dumpParams.currentSize = 0; std::error_code e; - for (; dumpParams.dirIter != directory_iterator(); dumpParams.dirIter++) { - dumpParams.dirEntry = *dumpParams.dirIter; - if (dumpParams.dirEntry.is_directory(e)) { - continue; - } - dumpParams.fileSize = std::filesystem::file_size(dumpParams.dirEntry.path(), e); - if (e) { - sif::error << "PersistentTmStore: Could not retrieve file size: " << e.message() << std::endl; - continue; - } - // sif::debug << "Path: " << dumpParams.dirEntry.path() << std::endl; - - // File empty or can't even read CCSDS header. - if (dumpParams.fileSize <= 6) { - continue; - } - if (dumpParams.fileSize > fileBuf.size()) { - sif::error << "PersistentTmStore: File too large, is deleted" << std::endl; - triggerEvent(persTmStore::FILE_TOO_LARGE, dumpParams.fileSize, fileBuf.size()); - std::filesystem::remove(dumpParams.dirEntry.path(), e); - continue; - } - const path& file = dumpParams.dirEntry.path(); - struct tm fileTime {}; - if (pathToTime(file, fileTime) != returnvalue::OK) { - sif::error << "Time extraction for file " << file << "failed" << std::endl; - continue; - } - auto fileEpoch = static_cast(timegm(&fileTime)); - if ((fileEpoch > dumpParams.fromUnixTime) and - (fileEpoch + rolloverDiffSeconds <= dumpParams.untilUnixTime)) { - dumpParams.currentFileUnixStamp = fileEpoch; - std::ifstream ifile(file, std::ios::binary); - if (ifile.bad()) { - sif::error << "PersistentTmStore: File is bad" << std::endl; - continue; + // Handle iteration, which does not necessarily have to increment the set iterator + // if there are multiple files for a given timestamp. + auto handleIteration = [&](DumpIndex& dumpIndex) { + if (dumpIndex.additionalFiles > 0) { + if (not dumpParams.currentSameFileIdx.has_value()) { + // Initialize the file index and stay on same file + dumpParams.currentSameFileIdx = 0; + return; + } else if (dumpParams.currentSameFileIdx.value() < dumpIndex.additionalFiles - 1) { + dumpParams.currentSameFileIdx = dumpParams.currentSameFileIdx.value() + 1; + return; } - ifile.read(reinterpret_cast(fileBuf.data()), - static_cast(dumpParams.fileSize)); - // Increment iterator for next cycle. - dumpParams.dirIter++; - return returnvalue::OK; } + // File will change, reset this field for correct state-keeping. + dumpParams.currentSameFileIdx = std::nullopt; + // Increment iterator for next cycle. + dumpParams.dumpIter++; + }; + while (dumpParams.dumpIter != dumpParams.orderedDumpFilestamps.end()) { + DumpIndex dumpIndex = *dumpParams.dumpIter; + timeval tv{}; + tv.tv_sec = dumpIndex.epoch; + size_t fullPathLength = 0; + createFileName(tv, dumpParams.currentSameFileIdx, fullPathLength); + dumpParams.currentFile = + path(std::string(reinterpret_cast(filePathBuf.data()), fullPathLength)); + if (DEBUG_DUMPS) { + sif::debug << baseName << " dump: Loading " << dumpParams.currentFile << std::endl; + } + dumpParams.fileSize = std::filesystem::file_size(dumpParams.currentFile, e); + if (e) { + // TODO: Event? + sif::error << "PersistentTmStore: Could not load next dump file: " << e.message() + << std::endl; + handleIteration(dumpIndex); + continue; + } + std::ifstream ifile(dumpParams.currentFile, std::ios::binary); + if (ifile.bad()) { + sif::error << "PersistentTmStore: File is bad. Loading next file" << std::endl; + handleIteration(dumpIndex); + continue; + } + ifile.read(reinterpret_cast(fileBuf.data()), + static_cast(dumpParams.fileSize)); + // Next file is loaded, exit the loop. + handleIteration(dumpIndex); + return returnvalue::OK; } // Directory iterator was consumed and we are done. state = State::IDLE; @@ -267,8 +373,8 @@ ReturnValue_t PersistentTmStore::getNextDumpPacket(PusTmReader& reader, bool& fi // Delete the file and load next. Could use better algorithm to partially // restore the file dump, but for now do not trust the file. std::error_code e; - std::filesystem::remove(dumpParams.dirEntry.path().c_str(), e); - if (dumpParams.dirEntry.path() == activeFile) { + std::filesystem::remove(dumpParams.currentFile.c_str(), e); + if (dumpParams.currentFile == activeFile) { activeFile == std::nullopt; assignAndOrCreateMostRecentFile(); } @@ -302,37 +408,9 @@ ReturnValue_t PersistentTmStore::pathToTime(const std::filesystem::path& path, s ReturnValue_t PersistentTmStore::createMostRecentFile(std::optional suffix) { using namespace std::filesystem; - unsigned currentIdx = 0; - path pathStart = basePath / baseName; - memcpy(fileBuf.data() + currentIdx, pathStart.c_str(), pathStart.string().length()); - currentIdx += pathStart.string().length(); - fileBuf[currentIdx] = '_'; - currentIdx += 1; - time_t epoch = currentTv.tv_sec; - struct tm* time = gmtime(&epoch); - size_t writtenBytes = strftime(reinterpret_cast(fileBuf.data() + currentIdx), - fileBuf.size(), config::FILE_DATE_FORMAT, time); - if (writtenBytes == 0) { - sif::error << "PersistentTmStore::createMostRecentFile: Could not create file timestamp" - << std::endl; - return returnvalue::FAILED; - } - currentIdx += writtenBytes; - char* res = strcpy(reinterpret_cast(fileBuf.data() + currentIdx), ".bin"); - if (res == nullptr) { - return returnvalue::FAILED; - } - currentIdx += 4; - if (suffix.has_value()) { - std::string fullSuffix = "." + std::to_string(suffix.value()); - res = strcpy(reinterpret_cast(fileBuf.data() + currentIdx), fullSuffix.c_str()); - if (res == nullptr) { - return returnvalue::FAILED; - } - currentIdx += fullSuffix.size(); - } - - path newPath(std::string(reinterpret_cast(fileBuf.data()), currentIdx)); + size_t currentIdx; + createFileName(currentTv, suffix, currentIdx); + path newPath(std::string(reinterpret_cast(filePathBuf.data()), currentIdx)); std::ofstream of(newPath, std::ios::binary); activeFile = newPath; activeFileTv = currentTv; @@ -354,3 +432,33 @@ void PersistentTmStore::getStartAndEndTimeCurrentOrLastDump(uint32_t& startTime, startTime = dumpParams.fromUnixTime; endTime = dumpParams.untilUnixTime; } + +ReturnValue_t PersistentTmStore::createFileName(timeval& tv, std::optional suffix, + size_t& fullPathLength) { + unsigned currentIdx = basePathSize; + time_t epoch = tv.tv_sec; + struct tm* time = gmtime(&epoch); + size_t writtenBytes = strftime(reinterpret_cast(filePathBuf.data() + currentIdx), + filePathBuf.size(), config::FILE_DATE_FORMAT, time); + if (writtenBytes == 0) { + sif::error << "PersistentTmStore::createMostRecentFile: Could not create file timestamp" + << std::endl; + return returnvalue::FAILED; + } + currentIdx += writtenBytes; + char* res = strcpy(reinterpret_cast(filePathBuf.data() + currentIdx), ".bin"); + if (res == nullptr) { + return returnvalue::FAILED; + } + currentIdx += 4; + if (suffix.has_value()) { + std::string fullSuffix = "." + std::to_string(suffix.value()); + res = strcpy(reinterpret_cast(filePathBuf.data() + currentIdx), fullSuffix.c_str()); + if (res == nullptr) { + return returnvalue::FAILED; + } + currentIdx += fullSuffix.size(); + } + fullPathLength = currentIdx; + return returnvalue::OK; +} diff --git a/mission/tmtc/PersistentTmStore.h b/mission/tmtc/PersistentTmStore.h index 17d2c9e7..e86acaaf 100644 --- a/mission/tmtc/PersistentTmStore.h +++ b/mission/tmtc/PersistentTmStore.h @@ -10,6 +10,7 @@ #include #include +#include #include "eive/eventSubsystemIds.h" #include "eive/resultClassIds.h" @@ -37,6 +38,14 @@ struct PersistentTmStoreArgs { SdCardMountedIF& sdcMan; }; +struct DumpIndex { + uint32_t epoch = 0; + // Number of additional files with a suffix like .0, .1 etc. + uint8_t additionalFiles = 0; + // Define a custom comparison function based on the epoch variable + bool operator<(const DumpIndex& other) const { return epoch < other.epoch; } +}; + class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject { public: enum class State { IDLE, DUMPING }; @@ -96,7 +105,10 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject { std::string baseName; uint8_t currentSameSecNumber = 0; std::filesystem::path basePath; + // std::filesystem::path pathStart = basePath / baseName; uint32_t rolloverDiffSeconds = 0; + std::array filePathBuf{}; + size_t basePathSize; std::array fileBuf{}; timeval currentTv; timeval activeFileTv{}; @@ -106,8 +118,10 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject { uint32_t fromUnixTime = 0; uint32_t untilUnixTime = 0; uint32_t currentFileUnixStamp = 0; - std::filesystem::directory_iterator dirIter; - std::filesystem::directory_entry dirEntry; + std::filesystem::path currentFile; + std::set orderedDumpFilestamps{}; + std::set::iterator dumpIter; + std::optional currentSameFileIdx = 0; size_t fileSize = 0; size_t currentSize = 0; }; @@ -122,10 +136,13 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject { [[nodiscard]] MessageQueueId_t getCommandQueue() const override; void calcDiffSeconds(RolloverInterval intervalUnit, uint32_t intervalCount); + ReturnValue_t buildDumpSet(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds); ReturnValue_t createMostRecentFile(std::optional suffix); static ReturnValue_t pathToTime(const std::filesystem::path& path, struct tm& time); void fileToPackets(const std::filesystem::path& path, uint32_t unixStamp); ReturnValue_t loadNextDumpFile(); + ReturnValue_t createFileName(timeval& tv, std::optional suffix, size_t& fullPathLength); + std::optional extractSuffix(const std::string& pathStr); bool updateBaseDir(); ReturnValue_t assignAndOrCreateMostRecentFile(); }; diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index 20b98979..e9a01891 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -4,7 +4,7 @@ add_subdirectory(mocks) target_sources(${UNITTEST_NAME} PRIVATE main.cpp testEnvironment.cpp - testStampInFilename.cpp + testGenericFilesystem.cpp hdlcEncodingRw.cpp printChar.cpp ) \ No newline at end of file diff --git a/unittest/testGenericFilesystem.cpp b/unittest/testGenericFilesystem.cpp new file mode 100644 index 00000000..e5279bab --- /dev/null +++ b/unittest/testGenericFilesystem.cpp @@ -0,0 +1,43 @@ + +#include +#include + +#include "fsfw/timemanager/Clock.h" + +uint8_t extractSuffix(const std::string& pathStr) { + std::string numberStr; + // Find the position of the dot at the end of the file path + size_t dotPos = pathStr.find_last_of('.'); + if (dotPos != std::string::npos && dotPos < pathStr.length() - 1) { + // Extract the substring after the dot + numberStr = pathStr.substr(dotPos + 1); + } + int number = std::stoi(numberStr); + if (number < 0 or number > std::numeric_limits::max()) { + return 0; + } + return static_cast(number); +} + +TEST_CASE("Stamp in Filename", "[Stamp In Filename]") { + Clock::TimeOfDay_t tod; + std::string baseName = "verif"; + std::string pathStr = "verif_2022-05-25T16:55:23Z.bin"; + unsigned int underscorePos = pathStr.find_last_of('_'); + std::string stampStr = pathStr.substr(underscorePos + 1); + float seconds = 0.0; + char* prefix = nullptr; + int count = + sscanf(stampStr.c_str(), + "%4" SCNu32 "-%2" SCNu32 "-%2" SCNu32 "T%2" SCNu32 ":%2" SCNu32 ":%2" SCNu32 "Z", + &tod.year, &tod.month, &tod.day, &tod.hour, &tod.minute, &tod.second); + static_cast(count); + CHECK(count == 6); +} + +TEST_CASE("Suffix Extraction") { + std::string pathStr = "/mnt/sd0/tm/hk/hk-some-stamp.bin.0"; + CHECK(extractSuffix(pathStr) == 0); + pathStr = "/mnt/sd0/tm/hk/hk-some-stamp.bin.2"; + CHECK(extractSuffix(pathStr) == 2); +} diff --git a/unittest/testStampInFilename.cpp b/unittest/testStampInFilename.cpp deleted file mode 100644 index c66bdce6..00000000 --- a/unittest/testStampInFilename.cpp +++ /dev/null @@ -1,21 +0,0 @@ - -#include -#include - -#include "fsfw/timemanager/Clock.h" - -TEST_CASE("Stamp in Filename", "[Stamp In Filename]") { - Clock::TimeOfDay_t tod; - std::string baseName = "verif"; - std::string pathStr = "verif_2022-05-25T16:55:23Z.bin"; - unsigned int underscorePos = pathStr.find_last_of('_'); - std::string stampStr = pathStr.substr(underscorePos + 1); - float seconds = 0.0; - char* prefix = nullptr; - int count = - sscanf(stampStr.c_str(), - "%4" SCNu32 "-%2" SCNu32 "-%2" SCNu32 "T%2" SCNu32 ":%2" SCNu32 ":%2" SCNu32 "Z", - &tod.year, &tod.month, &tod.day, &tod.hour, &tod.minute, &tod.second); - static_cast(count); - CHECK(count == 6); -}