diff --git a/common/config/eive/resultClassIds.h b/common/config/eive/resultClassIds.h index df47d935..125dd79a 100644 --- a/common/config/eive/resultClassIds.h +++ b/common/config/eive/resultClassIds.h @@ -42,6 +42,7 @@ enum commonClassIds : uint8_t { ACS_DETUMBLE, // ACSDTB SD_CARD_MANAGER, // SDMA LOCAL_PARAM_HANDLER, // LPH + PERSISTENT_TM_STORE, // PTM COMMON_CLASS_ID_END // [EXPORT] : [END] }; } diff --git a/mission/tmtc/PersistentTmStore.cpp b/mission/tmtc/PersistentTmStore.cpp index ea1b6c6f..5784fdba 100644 --- a/mission/tmtc/PersistentTmStore.cpp +++ b/mission/tmtc/PersistentTmStore.cpp @@ -1,6 +1,7 @@ #include "PersistentTmStore.h" #include +#include #include #include @@ -172,60 +173,122 @@ ReturnValue_t PersistentTmStore::startDumpFromUpTo(uint32_t fromUnixSeconds, if (state == State::DUMPING) { return returnvalue::FAILED; } - activeDumpDirIter = directory_iterator(basePath); + dumpParams.dirIter = directory_iterator(basePath); + dumpParams.fromUnixTime = fromUnixSeconds; + dumpParams.untilUnixTime = upToUnixSeconds; state = State::DUMPING; + if (loadNextDumpFile() == DUMP_DONE) { + // State will be set inside the function loading the next file. + return DUMP_DONE; + } return returnvalue::OK; - // for (auto const& file : directory_iterator(basePath)) { - // if (file.is_directory()) { - // continue; - // } - // struct tm fileTime {}; - // if (pathToTm(file.path(), fileTime) != returnvalue::OK) { - // sif::error << "Time extraction for file " << file << "failed" << std::endl; - // continue; - // } - // auto fileEpoch = static_cast(timegm(&fileTime)); - // if ((fileEpoch > fromUnixSeconds) and (fileEpoch + rolloverDiffSeconds <= upToUnixSeconds)) - // { - // fileToPackets(file, fileEpoch); - // } - // } } -void PersistentTmStore::fileToPackets(const std::filesystem::path& path, uint32_t unixStamp) { - store_address_t storeId; - TmTcMessage message; - size_t size = std::filesystem::file_size(path); - if (size < 6) { - // Can't even read the CCSDS header - return; - } - std::ifstream ifile(path, std::ios::binary); - ifile.read(reinterpret_cast(fileBuf.data()), static_cast(size)); - size_t currentIdx = 0; - while (currentIdx < size) { - PusTmReader reader(&timeReader, fileBuf.data(), fileBuf.size()); - // CRC check to fully ensure this is a valid TM - ReturnValue_t result = reader.parseDataWithCrcCheck(); - if (result == returnvalue::OK) { - // TODO: Blow the data out to the VC directly. Use IF function to do this. - // result = tmStore.addData(&storeId, fileBuf.data() + currentIdx, - // reader.getFullPacketLen()); if (result != returnvalue::OK) { - // continue; - // } - // funnel.sendPacketToLiveDestinations(storeId, message, fileBuf.data() + currentIdx, - // reader.getFullPacketLen()); - currentIdx += reader.getFullPacketLen(); - } else { - sif::error << "Parsing of PUS TM failed with code " << result << std::endl; - triggerEvent(POSSIBLE_FILE_CORRUPTION, result, unixStamp); - // Stop for now, do not really know where to continue and we do not trust the file anymore. +ReturnValue_t PersistentTmStore::loadNextDumpFile() { + using namespace std::filesystem; + std::error_code e; + for (; dumpParams.dirIter != directory_iterator(); dumpParams.dirIter++) { + if (dumpParams.dirEntry.is_directory(e)) { + continue; + } + dumpParams.fileSize = std::filesystem::file_size(dumpParams.dirEntry.path()); + // Can't even read CCSDS header. + if (dumpParams.fileSize <= 6) { + continue; + } + if (dumpParams.fileSize > fileBuf.size()) { + sif::error << "PersistentTmStore: File too large, is deleted" << std::endl; + triggerEvent(FILE_TOO_LARGE, dumpParams.fileSize, fileBuf.size()); + std::remove(dumpParams.dirEntry.path().c_str()); + continue; + } + const path& file = dumpParams.dirEntry.path(); + struct tm fileTime {}; + if (pathToTime(file, fileTime) != returnvalue::OK) { + sif::error << "Time extraction for file " << file << "failed" << std::endl; + continue; + } + auto fileEpoch = static_cast(timegm(&fileTime)); + if ((fileEpoch > dumpParams.fromUnixTime) and + (fileEpoch + rolloverDiffSeconds <= dumpParams.untilUnixTime)) { + dumpParams.currentSize = 0; + dumpParams.currentFileUnixStamp = fileEpoch; + std::ifstream ifile(file, std::ios::binary); + ifile.read(reinterpret_cast(fileBuf.data()), + static_cast(dumpParams.fileSize)); break; } } + if (dumpParams.dirIter == directory_iterator()) { + state = State::IDLE; + return DUMP_DONE; + } + return returnvalue::OK; } -ReturnValue_t PersistentTmStore::dumpNextPacket(size_t& dumpedLen) {} +// void PersistentTmStore::fileToPackets(const std::filesystem::path& path, uint32_t unixStamp) { +// store_address_t storeId; +// TmTcMessage message; +// size_t size = std::filesystem::file_size(path); +// if (size < 6) { +// // Can't even read the CCSDS header +// return; +// } +// std::ifstream ifile(path, std::ios::binary); +// ifile.read(reinterpret_cast(fileBuf.data()), static_cast(size)); +// size_t currentIdx = 0; +// while (currentIdx < size) { +// PusTmReader reader(&timeReader, fileBuf.data(), fileBuf.size()); +// // CRC check to fully ensure this is a valid TM +// ReturnValue_t result = reader.parseDataWithCrcCheck(); +// if (result == returnvalue::OK) { +// // TODO: Blow the data out to the VC directly. Use IF function to do this. +// // result = tmStore.addData(&storeId, fileBuf.data() + currentIdx, +// // reader.getFullPacketLen()); if (result != returnvalue::OK) { +// // continue; +// // } +// // funnel.sendPacketToLiveDestinations(storeId, message, fileBuf.data() + currentIdx, +// // reader.getFullPacketLen()); +// currentIdx += reader.getFullPacketLen(); +// } else { +// sif::error << "Parsing of PUS TM failed with code " << result << std::endl; +// triggerEvent(POSSIBLE_FILE_CORRUPTION, result, unixStamp); +// // Stop for now, do not really know where to continue and we do not trust the file anymore. +// break; +// } +// } +// } + +ReturnValue_t PersistentTmStore::dumpNextPacket(DirectTmSinkIF& tmSink, size_t& dumpedLen) { + if (state == State::IDLE) { + return returnvalue::FAILED; + } + PusTmReader reader(&timeReader, fileBuf.data() + dumpParams.currentSize, + fileBuf.size() - dumpParams.currentSize); + // CRC check to fully ensure this is a valid TM + ReturnValue_t result = reader.parseDataWithCrcCheck(); + if (result == returnvalue::OK) { + result = tmSink.write(fileBuf.data() + dumpParams.currentSize, reader.getFullPacketLen()); + if (result != returnvalue::OK) { + // TODO: Event? + sif::error << "PersistentTmStore: Writing to TM sink failed" << std::endl; + } + dumpParams.currentSize += reader.getFullPacketLen(); + dumpedLen = reader.getFullPacketLen(); + if (dumpParams.currentSize >= dumpParams.fileSize) { + return loadNextDumpFile(); + } + } else { + sif::error << "Parsing of PUS TM failed with code " << result << std::endl; + triggerEvent(POSSIBLE_FILE_CORRUPTION, result, dumpParams.currentFileUnixStamp); + // Delete the file and load next. Could use better algorithm to partially + // restore the file dump, but for now do not trust the file. + dumpedLen = 0; + std::remove(dumpParams.dirEntry.path().c_str()); + return loadNextDumpFile(); + } + return returnvalue::OK; +} ReturnValue_t PersistentTmStore::pathToTime(const std::filesystem::path& path, struct tm& time) { auto pathStr = path.string(); diff --git a/mission/tmtc/PersistentTmStore.h b/mission/tmtc/PersistentTmStore.h index ad7ff4e0..88c79443 100644 --- a/mission/tmtc/PersistentTmStore.h +++ b/mission/tmtc/PersistentTmStore.h @@ -7,17 +7,22 @@ #include #include #include +#include #include #include "TmFunnelBase.h" #include "eive/eventSubsystemIds.h" +#include "eive/resultClassIds.h" enum class RolloverInterval { MINUTELY, HOURLY, DAILY }; class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject { public: enum class State { IDLE, DUMPING }; + static constexpr uint8_t INTERFACE_ID = CLASS_ID::PERSISTENT_TM_STORE; + static constexpr ReturnValue_t DUMP_DONE = returnvalue::makeCode(INTERFACE_ID, 0); + static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::PERSISTENT_TM_STORE; //! [EXPORT] : [COMMENT] @@ -25,6 +30,9 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject { //! P2: Timestamp of possibly corrupt file as a unix timestamp. static constexpr Event POSSIBLE_FILE_CORRUPTION = event::makeEvent(SUBSYSTEM_ID, 0, severity::LOW); + //! [EXPORT] : [COMMENT] File in store too large. P1: Detected file size + //! P2: Allowed file size + static constexpr Event FILE_TOO_LARGE = event::makeEvent(SUBSYSTEM_ID, 1, severity::LOW); PersistentTmStore(object_id_t objectId, const char* baseDir, std::string baseName, RolloverInterval intervalUnit, uint32_t intervalCount, StorageManagerIF& tmStore, SdCardMountedIF& sdcMan); @@ -35,7 +43,7 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject { void deleteUpTo(uint32_t unixSeconds); ReturnValue_t startDumpFrom(uint32_t fromUnixSeconds); ReturnValue_t startDumpFromUpTo(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds); - ReturnValue_t dumpNextPacket(size_t& dumpedLen); + ReturnValue_t dumpNextPacket(DirectTmSinkIF& tmSink, size_t& dumpedLen); // ReturnValue_t passPacket(PusTmReader& reader); ReturnValue_t storePacket(PusTmReader& reader); @@ -46,6 +54,9 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject { // ISO8601 timestamp. static constexpr char FILE_DATE_FORMAT[] = "%FT%H%M%SZ"; + //! [EXPORT] : [SKIP] + static constexpr ReturnValue_t INVALID_FILE_DETECTED_AND_DELETED = returnvalue::makeCode(2, 1); + MessageQueueIF* tcQueue; State state = State::IDLE; // PacketFilter filter; @@ -59,10 +70,17 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject { std::array fileBuf{}; timeval currentTv; timeval activeFileTv{}; - std::filesystem::directory_iterator activeDumpDirIter; - std::filesystem::path activeDumpFile; - size_t activeDumpFileSize = 0; - size_t activeDumpCurrentSize = 0; + + struct ActiveDumpParams { + uint32_t fromUnixTime = 0; + uint32_t untilUnixTime = 0; + uint32_t currentFileUnixStamp = 0; + std::filesystem::directory_iterator dirIter; + std::filesystem::directory_entry dirEntry; + size_t fileSize = 0; + size_t currentSize = 0; + }; + ActiveDumpParams dumpParams; std::optional activeFile; SdCardMountedIF& sdcMan; StorageManagerIF& tmStore; @@ -77,6 +95,7 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject { ReturnValue_t createMostRecentFile(std::optional suffix); static ReturnValue_t pathToTime(const std::filesystem::path& path, struct tm& time); void fileToPackets(const std::filesystem::path& path, uint32_t unixStamp); + ReturnValue_t loadNextDumpFile(); bool updateBaseDir(); ReturnValue_t assignAndOrCreateMostRecentFile(); };