Persistent Tm Store now has dump state
Some checks failed
EIVE/eive-obsw/pipeline/head There was a failure building this commit
EIVE/eive-obsw/pipeline/pr-develop There was a failure building this commit

This commit is contained in:
Robin Müller 2023-03-09 13:15:42 +01:00
parent 49e3002abc
commit eb61996f91
No known key found for this signature in database
GPG Key ID: 11D4952C8CCEF814
3 changed files with 133 additions and 50 deletions

View File

@ -42,6 +42,7 @@ enum commonClassIds : uint8_t {
ACS_DETUMBLE, // ACSDTB ACS_DETUMBLE, // ACSDTB
SD_CARD_MANAGER, // SDMA SD_CARD_MANAGER, // SDMA
LOCAL_PARAM_HANDLER, // LPH LOCAL_PARAM_HANDLER, // LPH
PERSISTENT_TM_STORE, // PTM
COMMON_CLASS_ID_END // [EXPORT] : [END] COMMON_CLASS_ID_END // [EXPORT] : [END]
}; };
} }

View File

@ -1,6 +1,7 @@
#include "PersistentTmStore.h" #include "PersistentTmStore.h"
#include <mission/memory/SdCardMountedIF.h> #include <mission/memory/SdCardMountedIF.h>
#include <mission/tmtc/DirectTmSinkIF.h>
#include <algorithm> #include <algorithm>
#include <cinttypes> #include <cinttypes>
@ -172,60 +173,122 @@ ReturnValue_t PersistentTmStore::startDumpFromUpTo(uint32_t fromUnixSeconds,
if (state == State::DUMPING) { if (state == State::DUMPING) {
return returnvalue::FAILED; return returnvalue::FAILED;
} }
activeDumpDirIter = directory_iterator(basePath); dumpParams.dirIter = directory_iterator(basePath);
dumpParams.fromUnixTime = fromUnixSeconds;
dumpParams.untilUnixTime = upToUnixSeconds;
state = State::DUMPING; state = State::DUMPING;
if (loadNextDumpFile() == DUMP_DONE) {
// State will be set inside the function loading the next file.
return DUMP_DONE;
}
return returnvalue::OK; return returnvalue::OK;
// for (auto const& file : directory_iterator(basePath)) {
// if (file.is_directory()) {
// continue;
// }
// struct tm fileTime {};
// if (pathToTm(file.path(), fileTime) != returnvalue::OK) {
// sif::error << "Time extraction for file " << file << "failed" << std::endl;
// continue;
// }
// auto fileEpoch = static_cast<uint32_t>(timegm(&fileTime));
// if ((fileEpoch > fromUnixSeconds) and (fileEpoch + rolloverDiffSeconds <= upToUnixSeconds))
// {
// fileToPackets(file, fileEpoch);
// }
// }
} }
void PersistentTmStore::fileToPackets(const std::filesystem::path& path, uint32_t unixStamp) { ReturnValue_t PersistentTmStore::loadNextDumpFile() {
store_address_t storeId; using namespace std::filesystem;
TmTcMessage message; std::error_code e;
size_t size = std::filesystem::file_size(path); for (; dumpParams.dirIter != directory_iterator(); dumpParams.dirIter++) {
if (size < 6) { if (dumpParams.dirEntry.is_directory(e)) {
// Can't even read the CCSDS header continue;
return; }
} dumpParams.fileSize = std::filesystem::file_size(dumpParams.dirEntry.path());
std::ifstream ifile(path, std::ios::binary); // Can't even read CCSDS header.
ifile.read(reinterpret_cast<char*>(fileBuf.data()), static_cast<std::streamsize>(size)); if (dumpParams.fileSize <= 6) {
size_t currentIdx = 0; continue;
while (currentIdx < size) { }
PusTmReader reader(&timeReader, fileBuf.data(), fileBuf.size()); if (dumpParams.fileSize > fileBuf.size()) {
// CRC check to fully ensure this is a valid TM sif::error << "PersistentTmStore: File too large, is deleted" << std::endl;
ReturnValue_t result = reader.parseDataWithCrcCheck(); triggerEvent(FILE_TOO_LARGE, dumpParams.fileSize, fileBuf.size());
if (result == returnvalue::OK) { std::remove(dumpParams.dirEntry.path().c_str());
// TODO: Blow the data out to the VC directly. Use IF function to do this. continue;
// result = tmStore.addData(&storeId, fileBuf.data() + currentIdx, }
// reader.getFullPacketLen()); if (result != returnvalue::OK) { const path& file = dumpParams.dirEntry.path();
// continue; struct tm fileTime {};
// } if (pathToTime(file, fileTime) != returnvalue::OK) {
// funnel.sendPacketToLiveDestinations(storeId, message, fileBuf.data() + currentIdx, sif::error << "Time extraction for file " << file << "failed" << std::endl;
// reader.getFullPacketLen()); continue;
currentIdx += reader.getFullPacketLen(); }
} else { auto fileEpoch = static_cast<uint32_t>(timegm(&fileTime));
sif::error << "Parsing of PUS TM failed with code " << result << std::endl; if ((fileEpoch > dumpParams.fromUnixTime) and
triggerEvent(POSSIBLE_FILE_CORRUPTION, result, unixStamp); (fileEpoch + rolloverDiffSeconds <= dumpParams.untilUnixTime)) {
// Stop for now, do not really know where to continue and we do not trust the file anymore. dumpParams.currentSize = 0;
dumpParams.currentFileUnixStamp = fileEpoch;
std::ifstream ifile(file, std::ios::binary);
ifile.read(reinterpret_cast<char*>(fileBuf.data()),
static_cast<std::streamsize>(dumpParams.fileSize));
break; break;
} }
} }
if (dumpParams.dirIter == directory_iterator()) {
state = State::IDLE;
return DUMP_DONE;
}
return returnvalue::OK;
} }
ReturnValue_t PersistentTmStore::dumpNextPacket(size_t& dumpedLen) {} // void PersistentTmStore::fileToPackets(const std::filesystem::path& path, uint32_t unixStamp) {
// store_address_t storeId;
// TmTcMessage message;
// size_t size = std::filesystem::file_size(path);
// if (size < 6) {
// // Can't even read the CCSDS header
// return;
// }
// std::ifstream ifile(path, std::ios::binary);
// ifile.read(reinterpret_cast<char*>(fileBuf.data()), static_cast<std::streamsize>(size));
// size_t currentIdx = 0;
// while (currentIdx < size) {
// PusTmReader reader(&timeReader, fileBuf.data(), fileBuf.size());
// // CRC check to fully ensure this is a valid TM
// ReturnValue_t result = reader.parseDataWithCrcCheck();
// if (result == returnvalue::OK) {
// // TODO: Blow the data out to the VC directly. Use IF function to do this.
// // result = tmStore.addData(&storeId, fileBuf.data() + currentIdx,
// // reader.getFullPacketLen()); if (result != returnvalue::OK) {
// // continue;
// // }
// // funnel.sendPacketToLiveDestinations(storeId, message, fileBuf.data() + currentIdx,
// // reader.getFullPacketLen());
// currentIdx += reader.getFullPacketLen();
// } else {
// sif::error << "Parsing of PUS TM failed with code " << result << std::endl;
// triggerEvent(POSSIBLE_FILE_CORRUPTION, result, unixStamp);
// // Stop for now, do not really know where to continue and we do not trust the file anymore.
// break;
// }
// }
// }
ReturnValue_t PersistentTmStore::dumpNextPacket(DirectTmSinkIF& tmSink, size_t& dumpedLen) {
if (state == State::IDLE) {
return returnvalue::FAILED;
}
PusTmReader reader(&timeReader, fileBuf.data() + dumpParams.currentSize,
fileBuf.size() - dumpParams.currentSize);
// CRC check to fully ensure this is a valid TM
ReturnValue_t result = reader.parseDataWithCrcCheck();
if (result == returnvalue::OK) {
result = tmSink.write(fileBuf.data() + dumpParams.currentSize, reader.getFullPacketLen());
if (result != returnvalue::OK) {
// TODO: Event?
sif::error << "PersistentTmStore: Writing to TM sink failed" << std::endl;
}
dumpParams.currentSize += reader.getFullPacketLen();
dumpedLen = reader.getFullPacketLen();
if (dumpParams.currentSize >= dumpParams.fileSize) {
return loadNextDumpFile();
}
} else {
sif::error << "Parsing of PUS TM failed with code " << result << std::endl;
triggerEvent(POSSIBLE_FILE_CORRUPTION, result, dumpParams.currentFileUnixStamp);
// Delete the file and load next. Could use better algorithm to partially
// restore the file dump, but for now do not trust the file.
dumpedLen = 0;
std::remove(dumpParams.dirEntry.path().c_str());
return loadNextDumpFile();
}
return returnvalue::OK;
}
ReturnValue_t PersistentTmStore::pathToTime(const std::filesystem::path& path, struct tm& time) { ReturnValue_t PersistentTmStore::pathToTime(const std::filesystem::path& path, struct tm& time) {
auto pathStr = path.string(); auto pathStr = path.string();

View File

@ -7,17 +7,22 @@
#include <fsfw/tmtcpacket/pus/tm/PusTmReader.h> #include <fsfw/tmtcpacket/pus/tm/PusTmReader.h>
#include <fsfw/tmtcservices/AcceptsTelemetryIF.h> #include <fsfw/tmtcservices/AcceptsTelemetryIF.h>
#include <mission/memory/SdCardMountedIF.h> #include <mission/memory/SdCardMountedIF.h>
#include <mission/tmtc/DirectTmSinkIF.h>
#include <filesystem> #include <filesystem>
#include "TmFunnelBase.h" #include "TmFunnelBase.h"
#include "eive/eventSubsystemIds.h" #include "eive/eventSubsystemIds.h"
#include "eive/resultClassIds.h"
enum class RolloverInterval { MINUTELY, HOURLY, DAILY }; enum class RolloverInterval { MINUTELY, HOURLY, DAILY };
class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject { class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
public: public:
enum class State { IDLE, DUMPING }; enum class State { IDLE, DUMPING };
static constexpr uint8_t INTERFACE_ID = CLASS_ID::PERSISTENT_TM_STORE;
static constexpr ReturnValue_t DUMP_DONE = returnvalue::makeCode(INTERFACE_ID, 0);
static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::PERSISTENT_TM_STORE; static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::PERSISTENT_TM_STORE;
//! [EXPORT] : [COMMENT] //! [EXPORT] : [COMMENT]
@ -25,6 +30,9 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
//! P2: Timestamp of possibly corrupt file as a unix timestamp. //! P2: Timestamp of possibly corrupt file as a unix timestamp.
static constexpr Event POSSIBLE_FILE_CORRUPTION = static constexpr Event POSSIBLE_FILE_CORRUPTION =
event::makeEvent(SUBSYSTEM_ID, 0, severity::LOW); event::makeEvent(SUBSYSTEM_ID, 0, severity::LOW);
//! [EXPORT] : [COMMENT] File in store too large. P1: Detected file size
//! P2: Allowed file size
static constexpr Event FILE_TOO_LARGE = event::makeEvent(SUBSYSTEM_ID, 1, severity::LOW);
PersistentTmStore(object_id_t objectId, const char* baseDir, std::string baseName, PersistentTmStore(object_id_t objectId, const char* baseDir, std::string baseName,
RolloverInterval intervalUnit, uint32_t intervalCount, RolloverInterval intervalUnit, uint32_t intervalCount,
StorageManagerIF& tmStore, SdCardMountedIF& sdcMan); StorageManagerIF& tmStore, SdCardMountedIF& sdcMan);
@ -35,7 +43,7 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
void deleteUpTo(uint32_t unixSeconds); void deleteUpTo(uint32_t unixSeconds);
ReturnValue_t startDumpFrom(uint32_t fromUnixSeconds); ReturnValue_t startDumpFrom(uint32_t fromUnixSeconds);
ReturnValue_t startDumpFromUpTo(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds); ReturnValue_t startDumpFromUpTo(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds);
ReturnValue_t dumpNextPacket(size_t& dumpedLen); ReturnValue_t dumpNextPacket(DirectTmSinkIF& tmSink, size_t& dumpedLen);
// ReturnValue_t passPacket(PusTmReader& reader); // ReturnValue_t passPacket(PusTmReader& reader);
ReturnValue_t storePacket(PusTmReader& reader); ReturnValue_t storePacket(PusTmReader& reader);
@ -46,6 +54,9 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
// ISO8601 timestamp. // ISO8601 timestamp.
static constexpr char FILE_DATE_FORMAT[] = "%FT%H%M%SZ"; static constexpr char FILE_DATE_FORMAT[] = "%FT%H%M%SZ";
//! [EXPORT] : [SKIP]
static constexpr ReturnValue_t INVALID_FILE_DETECTED_AND_DELETED = returnvalue::makeCode(2, 1);
MessageQueueIF* tcQueue; MessageQueueIF* tcQueue;
State state = State::IDLE; State state = State::IDLE;
// PacketFilter filter; // PacketFilter filter;
@ -59,10 +70,17 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
std::array<uint8_t, MAX_FILESIZE> fileBuf{}; std::array<uint8_t, MAX_FILESIZE> fileBuf{};
timeval currentTv; timeval currentTv;
timeval activeFileTv{}; timeval activeFileTv{};
std::filesystem::directory_iterator activeDumpDirIter;
std::filesystem::path activeDumpFile; struct ActiveDumpParams {
size_t activeDumpFileSize = 0; uint32_t fromUnixTime = 0;
size_t activeDumpCurrentSize = 0; uint32_t untilUnixTime = 0;
uint32_t currentFileUnixStamp = 0;
std::filesystem::directory_iterator dirIter;
std::filesystem::directory_entry dirEntry;
size_t fileSize = 0;
size_t currentSize = 0;
};
ActiveDumpParams dumpParams;
std::optional<std::filesystem::path> activeFile; std::optional<std::filesystem::path> activeFile;
SdCardMountedIF& sdcMan; SdCardMountedIF& sdcMan;
StorageManagerIF& tmStore; StorageManagerIF& tmStore;
@ -77,6 +95,7 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
ReturnValue_t createMostRecentFile(std::optional<uint8_t> suffix); ReturnValue_t createMostRecentFile(std::optional<uint8_t> suffix);
static ReturnValue_t pathToTime(const std::filesystem::path& path, struct tm& time); static ReturnValue_t pathToTime(const std::filesystem::path& path, struct tm& time);
void fileToPackets(const std::filesystem::path& path, uint32_t unixStamp); void fileToPackets(const std::filesystem::path& path, uint32_t unixStamp);
ReturnValue_t loadNextDumpFile();
bool updateBaseDir(); bool updateBaseDir();
ReturnValue_t assignAndOrCreateMostRecentFile(); ReturnValue_t assignAndOrCreateMostRecentFile();
}; };