eive/eive-obsw
eive/eive-obsw
13
4
Fork 0
Code Issues 4 Pull Requests 1 Releases 120 Wiki Activity

Ordered dumps #706

Merged
muellerr merged 16 commits from ordered-dumps into main 2023-06-26 18:11:08 +02:00
Conversation 0 Commits 16 Files Changed 6 +251 -100
6 changed files with 251 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -22,6 +22,10 @@ will consitute of a breaking change warranting a new major release:
# [v5.1.0] to be released
## Changed
- Persistent TM store dumps are now performed in chronological order
# [v5.0.0] 2023-06-26
v3.3.1 and all following version will now be moved to v5.0.0 with the additional changes listed

260
mission/tmtc/PersistentTmStore.cpp
View File

@ -17,6 +17,8 @@
using namespace returnvalue;
static constexpr bool DEBUG_DUMPS = false;
PersistentTmStore::PersistentTmStore(PersistentTmStoreArgs args)
: SystemObject(args.objectId),
tmStore(args.tmStore),
@ -32,6 +34,91 @@ ReturnValue_t PersistentTmStore::cancelDump() {
return returnvalue::OK;
}
ReturnValue_t PersistentTmStore::buildDumpSet(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds) {
using namespace std::filesystem;
std::error_code e;
dumpParams.orderedDumpFilestamps.clear();
for (auto const& fileOrDir : directory_iterator(basePath)) {
if (not fileOrDir.is_regular_file(e)) {
continue;
}
dumpParams.fileSize = std::filesystem::file_size(fileOrDir.path(), e);
if (e) {
sif::error << "PersistentTmStore: Could not retrieve file size: " << e.message() << std::endl;
continue;
}
// File empty or can't even read CCSDS header.
if (dumpParams.fileSize <= 6) {
continue;
}
if (dumpParams.fileSize > fileBuf.size()) {
sif::error << "PersistentTmStore: File too large, is deleted" << std::endl;
triggerEvent(persTmStore::FILE_TOO_LARGE, dumpParams.fileSize, fileBuf.size());
std::filesystem::remove(fileOrDir.path(), e);
continue;
}
const path& file = fileOrDir.path();
struct tm fileTime {};
if (pathToTime(file, fileTime) != returnvalue::OK) {
sif::error << "Time extraction for file " << file << "failed" << std::endl;
continue;
}
auto fileEpoch = static_cast<uint32_t>(timegm(&fileTime));
if ((fileEpoch > dumpParams.fromUnixTime) and
(fileEpoch + rolloverDiffSeconds <= dumpParams.untilUnixTime)) {
std::ifstream ifile(file, std::ios::binary);
if (ifile.bad()) {
sif::error << "PersistentTmStore: File is bad" << std::endl;
// TODO: Consider deleting file here?
continue;
}
if (DEBUG_DUMPS) {
sif::debug << "Inserting file " << fileOrDir.path() << std::endl;
}
DumpIndex dumpIndex;
dumpIndex.epoch = fileEpoch;
// Multiple files for the same time are supported via a special suffix. We simply count the
// number of copies and later try to dump the same number of files with the additional
// suffixes
auto iter = dumpParams.orderedDumpFilestamps.find(dumpIndex);
if (iter != dumpParams.orderedDumpFilestamps.end()) {
dumpIndex.epoch = iter->epoch;
dumpIndex.additionalFiles = iter->additionalFiles + 1;
dumpParams.orderedDumpFilestamps.erase(dumpIndex);
} else {
dumpIndex.additionalFiles = 0;
}
dumpParams.orderedDumpFilestamps.emplace(dumpIndex);
}
}
return returnvalue::OK;
}
std::optional<uint8_t> PersistentTmStore::extractSuffix(const std::string& pathStr) {
std::string numberStr;
// Find the position of the dot at the end of the file path
size_t dotPos = pathStr.find_last_of('.');
if ((dotPos < pathStr.length()) and not std::isdigit(pathStr[dotPos + 1])) {
return std::nullopt;
}
// Extract the substring after the dot
numberStr = pathStr.substr(dotPos + 1);
std::optional<uint8_t> number;
try {
number = std::stoi(numberStr);
if (number.value() > std::numeric_limits<uint8_t>::max()) {
return std::nullopt;
}
} catch (std::invalid_argument& exception) {
sif::error << "PersistentTmStore::extractSuffix: Exception " << exception.what()
<< ", invald input string: " << numberStr << std::endl;
}
return number;
}
ReturnValue_t PersistentTmStore::assignAndOrCreateMostRecentFile() {
if (not activeFile.has_value()) {
return createMostRecentFile(std::nullopt);
@ -159,6 +246,12 @@ bool PersistentTmStore::updateBaseDir() {
if (not exists(basePath, e)) {
create_directories(basePath);
}
// Each file will have the base name as a prefix again
path preparedFullFilePath = basePath / baseName;
basePathSize = preparedFullFilePath.string().length();
std::memcpy(filePathBuf.data(), preparedFullFilePath.c_str(), basePathSize);
filePathBuf[basePathSize] = '_';
basePathSize += 1;
baseDirUninitialized = false;
return true;
}
@ -189,12 +282,20 @@ ReturnValue_t PersistentTmStore::startDumpFromUpTo(uint32_t fromUnixSeconds,
if (state == State::DUMPING) {
return returnvalue::FAILED;
}
dumpParams.dirIter = directory_iterator(basePath);
if (dumpParams.dirIter == directory_iterator()) {
auto dirIter = directory_iterator(basePath);
// Directory empty case.
if (dirIter == directory_iterator()) {
return returnvalue::FAILED;
}
dumpParams.fromUnixTime = fromUnixSeconds;
dumpParams.untilUnixTime = upToUnixSeconds;
buildDumpSet(fromUnixSeconds, upToUnixSeconds);
// No files in time window found.
if (dumpParams.orderedDumpFilestamps.empty()) {
return DUMP_DONE;
}
dumpParams.dumpIter = dumpParams.orderedDumpFilestamps.begin();
dumpParams.currentSameFileIdx = std::nullopt;
state = State::DUMPING;
return loadNextDumpFile();
}
@ -203,49 +304,54 @@ ReturnValue_t PersistentTmStore::loadNextDumpFile() {
using namespace std::filesystem;
dumpParams.currentSize = 0;
std::error_code e;
for (; dumpParams.dirIter != directory_iterator(); dumpParams.dirIter++) {
dumpParams.dirEntry = *dumpParams.dirIter;
if (dumpParams.dirEntry.is_directory(e)) {
continue;
}
dumpParams.fileSize = std::filesystem::file_size(dumpParams.dirEntry.path(), e);
if (e) {
sif::error << "PersistentTmStore: Could not retrieve file size: " << e.message() << std::endl;
continue;
}
// sif::debug << "Path: " << dumpParams.dirEntry.path() << std::endl;
// File empty or can't even read CCSDS header.
if (dumpParams.fileSize <= 6) {
continue;
}
if (dumpParams.fileSize > fileBuf.size()) {
sif::error << "PersistentTmStore: File too large, is deleted" << std::endl;
triggerEvent(persTmStore::FILE_TOO_LARGE, dumpParams.fileSize, fileBuf.size());
std::filesystem::remove(dumpParams.dirEntry.path(), e);
continue;
}
const path& file = dumpParams.dirEntry.path();
struct tm fileTime {};
if (pathToTime(file, fileTime) != returnvalue::OK) {
sif::error << "Time extraction for file " << file << "failed" << std::endl;
continue;
}
auto fileEpoch = static_cast<uint32_t>(timegm(&fileTime));
if ((fileEpoch > dumpParams.fromUnixTime) and
(fileEpoch + rolloverDiffSeconds <= dumpParams.untilUnixTime)) {
dumpParams.currentFileUnixStamp = fileEpoch;
std::ifstream ifile(file, std::ios::binary);
if (ifile.bad()) {
sif::error << "PersistentTmStore: File is bad" << std::endl;
continue;
// Handle iteration, which does not necessarily have to increment the set iterator
// if there are multiple files for a given timestamp.
auto handleIteration = [&](DumpIndex& dumpIndex) {
if (dumpIndex.additionalFiles > 0) {
if (not dumpParams.currentSameFileIdx.has_value()) {
// Initialize the file index and stay on same file
dumpParams.currentSameFileIdx = 0;
return;
} else if (dumpParams.currentSameFileIdx.value() < dumpIndex.additionalFiles - 1) {
dumpParams.currentSameFileIdx = dumpParams.currentSameFileIdx.value() + 1;
return;
}
ifile.read(reinterpret_cast<char*>(fileBuf.data()),
static_cast<std::streamsize>(dumpParams.fileSize));
// Increment iterator for next cycle.
dumpParams.dirIter++;
return returnvalue::OK;
}
// File will change, reset this field for correct state-keeping.
dumpParams.currentSameFileIdx = std::nullopt;
// Increment iterator for next cycle.
dumpParams.dumpIter++;
};
while (dumpParams.dumpIter != dumpParams.orderedDumpFilestamps.end()) {
DumpIndex dumpIndex = *dumpParams.dumpIter;
timeval tv{};
tv.tv_sec = dumpIndex.epoch;
size_t fullPathLength = 0;
createFileName(tv, dumpParams.currentSameFileIdx, fullPathLength);
dumpParams.currentFile =
path(std::string(reinterpret_cast<const char*>(filePathBuf.data()), fullPathLength));
if (DEBUG_DUMPS) {
sif::debug << baseName << " dump: Loading " << dumpParams.currentFile << std::endl;
}
dumpParams.fileSize = std::filesystem::file_size(dumpParams.currentFile, e);
if (e) {
// TODO: Event?
sif::error << "PersistentTmStore: Could not load next dump file: " << e.message()
<< std::endl;
handleIteration(dumpIndex);
continue;
}
std::ifstream ifile(dumpParams.currentFile, std::ios::binary);
if (ifile.bad()) {
sif::error << "PersistentTmStore: File is bad. Loading next file" << std::endl;
handleIteration(dumpIndex);
continue;
}
ifile.read(reinterpret_cast<char*>(fileBuf.data()),
static_cast<std::streamsize>(dumpParams.fileSize));
// Next file is loaded, exit the loop.
handleIteration(dumpIndex);
return returnvalue::OK;
}
// Directory iterator was consumed and we are done.
state = State::IDLE;
@ -267,8 +373,8 @@ ReturnValue_t PersistentTmStore::getNextDumpPacket(PusTmReader& reader, bool& fi
// Delete the file and load next. Could use better algorithm to partially
// restore the file dump, but for now do not trust the file.
std::error_code e;
std::filesystem::remove(dumpParams.dirEntry.path().c_str(), e);
if (dumpParams.dirEntry.path() == activeFile) {
std::filesystem::remove(dumpParams.currentFile.c_str(), e);
if (dumpParams.currentFile == activeFile) {
activeFile == std::nullopt;
assignAndOrCreateMostRecentFile();
}
@ -302,37 +408,9 @@ ReturnValue_t PersistentTmStore::pathToTime(const std::filesystem::path& path, s
ReturnValue_t PersistentTmStore::createMostRecentFile(std::optional<uint8_t> suffix) {
using namespace std::filesystem;
unsigned currentIdx = 0;
path pathStart = basePath / baseName;
memcpy(fileBuf.data() + currentIdx, pathStart.c_str(), pathStart.string().length());
currentIdx += pathStart.string().length();
fileBuf[currentIdx] = '_';
currentIdx += 1;
time_t epoch = currentTv.tv_sec;
struct tm* time = gmtime(&epoch);
size_t writtenBytes = strftime(reinterpret_cast<char*>(fileBuf.data() + currentIdx),
fileBuf.size(), config::FILE_DATE_FORMAT, time);
if (writtenBytes == 0) {
sif::error << "PersistentTmStore::createMostRecentFile: Could not create file timestamp"
<< std::endl;
return returnvalue::FAILED;
}
currentIdx += writtenBytes;
char* res = strcpy(reinterpret_cast<char*>(fileBuf.data() + currentIdx), ".bin");
if (res == nullptr) {
return returnvalue::FAILED;
}
currentIdx += 4;
if (suffix.has_value()) {
std::string fullSuffix = "." + std::to_string(suffix.value());
res = strcpy(reinterpret_cast<char*>(fileBuf.data() + currentIdx), fullSuffix.c_str());
if (res == nullptr) {
return returnvalue::FAILED;
}
currentIdx += fullSuffix.size();
}
path newPath(std::string(reinterpret_cast<const char*>(fileBuf.data()), currentIdx));
size_t currentIdx;
createFileName(currentTv, suffix, currentIdx);
path newPath(std::string(reinterpret_cast<const char*>(filePathBuf.data()), currentIdx));
std::ofstream of(newPath, std::ios::binary);
activeFile = newPath;
activeFileTv = currentTv;
@ -354,3 +432,33 @@ void PersistentTmStore::getStartAndEndTimeCurrentOrLastDump(uint32_t& startTime,
startTime = dumpParams.fromUnixTime;
endTime = dumpParams.untilUnixTime;
}
ReturnValue_t PersistentTmStore::createFileName(timeval& tv, std::optional<uint8_t> suffix,
size_t& fullPathLength) {
unsigned currentIdx = basePathSize;
time_t epoch = tv.tv_sec;
struct tm* time = gmtime(&epoch);
size_t writtenBytes = strftime(reinterpret_cast<char*>(filePathBuf.data() + currentIdx),
filePathBuf.size(), config::FILE_DATE_FORMAT, time);
if (writtenBytes == 0) {
sif::error << "PersistentTmStore::createMostRecentFile: Could not create file timestamp"
<< std::endl;
return returnvalue::FAILED;
}
currentIdx += writtenBytes;
char* res = strcpy(reinterpret_cast<char*>(filePathBuf.data() + currentIdx), ".bin");
if (res == nullptr) {
return returnvalue::FAILED;
}
currentIdx += 4;
if (suffix.has_value()) {
std::string fullSuffix = "." + std::to_string(suffix.value());
res = strcpy(reinterpret_cast<char*>(filePathBuf.data() + currentIdx), fullSuffix.c_str());
if (res == nullptr) {
return returnvalue::FAILED;
}
currentIdx += fullSuffix.size();
}
fullPathLength = currentIdx;
return returnvalue::OK;
}

21
mission/tmtc/PersistentTmStore.h
View File

@ -10,6 +10,7 @@
#include <mission/memory/SdCardMountedIF.h>
#include <filesystem>
#include <set>
#include "eive/eventSubsystemIds.h"
#include "eive/resultClassIds.h"
@ -37,6 +38,14 @@ struct PersistentTmStoreArgs {
SdCardMountedIF& sdcMan;
};
struct DumpIndex {
uint32_t epoch = 0;
// Number of additional files with a suffix like .0, .1 etc.
uint8_t additionalFiles = 0;
// Define a custom comparison function based on the epoch variable
bool operator<(const DumpIndex& other) const { return epoch < other.epoch; }
};
class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
public:
enum class State { IDLE, DUMPING };
@ -96,7 +105,10 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
std::string baseName;
uint8_t currentSameSecNumber = 0;
std::filesystem::path basePath;
// std::filesystem::path pathStart = basePath / baseName;
uint32_t rolloverDiffSeconds = 0;
std::array<uint8_t, 524> filePathBuf{};
size_t basePathSize;
std::array<uint8_t, MAX_FILESIZE> fileBuf{};
timeval currentTv;
timeval activeFileTv{};
@ -106,8 +118,10 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
uint32_t fromUnixTime = 0;
uint32_t untilUnixTime = 0;
uint32_t currentFileUnixStamp = 0;
std::filesystem::directory_iterator dirIter;
std::filesystem::directory_entry dirEntry;
std::filesystem::path currentFile;
std::set<DumpIndex> orderedDumpFilestamps{};
std::set<DumpIndex>::iterator dumpIter;
std::optional<uint8_t> currentSameFileIdx = 0;
size_t fileSize = 0;
size_t currentSize = 0;
};
@ -122,10 +136,13 @@ class PersistentTmStore : public TmStoreFrontendSimpleIF, public SystemObject {
[[nodiscard]] MessageQueueId_t getCommandQueue() const override;
void calcDiffSeconds(RolloverInterval intervalUnit, uint32_t intervalCount);
ReturnValue_t buildDumpSet(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds);
ReturnValue_t createMostRecentFile(std::optional<uint8_t> suffix);
static ReturnValue_t pathToTime(const std::filesystem::path& path, struct tm& time);
void fileToPackets(const std::filesystem::path& path, uint32_t unixStamp);
ReturnValue_t loadNextDumpFile();
ReturnValue_t createFileName(timeval& tv, std::optional<uint8_t> suffix, size_t& fullPathLength);
std::optional<uint8_t> extractSuffix(const std::string& pathStr);
bool updateBaseDir();
ReturnValue_t assignAndOrCreateMostRecentFile();
};

2
unittest/CMakeLists.txt
View File

@ -4,7 +4,7 @@ add_subdirectory(mocks)
target_sources(${UNITTEST_NAME} PRIVATE
main.cpp
testEnvironment.cpp
testStampInFilename.cpp
testGenericFilesystem.cpp
hdlcEncodingRw.cpp
printChar.cpp
)

43
unittest/testGenericFilesystem.cpp Normal file
View File

@ -0,0 +1,43 @@
#include <catch2/catch_test_macros.hpp>
#include <cinttypes>
#include "fsfw/timemanager/Clock.h"
uint8_t extractSuffix(const std::string& pathStr) {
std::string numberStr;
// Find the position of the dot at the end of the file path
size_t dotPos = pathStr.find_last_of('.');
if (dotPos != std::string::npos && dotPos < pathStr.length() - 1) {
// Extract the substring after the dot
numberStr = pathStr.substr(dotPos + 1);
}
int number = std::stoi(numberStr);
if (number < 0 or number > std::numeric_limits<uint8_t>::max()) {
return 0;
}
return static_cast<uint8_t>(number);
}
TEST_CASE("Stamp in Filename", "[Stamp In Filename]") {
Clock::TimeOfDay_t tod;
std::string baseName = "verif";
std::string pathStr = "verif_2022-05-25T16:55:23Z.bin";
unsigned int underscorePos = pathStr.find_last_of('_');
std::string stampStr = pathStr.substr(underscorePos + 1);
float seconds = 0.0;
char* prefix = nullptr;
int count =
sscanf(stampStr.c_str(),
"%4" SCNu32 "-%2" SCNu32 "-%2" SCNu32 "T%2" SCNu32 ":%2" SCNu32 ":%2" SCNu32 "Z",
&tod.year, &tod.month, &tod.day, &tod.hour, &tod.minute, &tod.second);
static_cast<void>(count);
CHECK(count == 6);
}
TEST_CASE("Suffix Extraction") {
std::string pathStr = "/mnt/sd0/tm/hk/hk-some-stamp.bin.0";
CHECK(extractSuffix(pathStr) == 0);
pathStr = "/mnt/sd0/tm/hk/hk-some-stamp.bin.2";
CHECK(extractSuffix(pathStr) == 2);
}

21
unittest/testStampInFilename.cpp
View File

@ -1,21 +0,0 @@
#include <catch2/catch_test_macros.hpp>
#include <cinttypes>
#include "fsfw/timemanager/Clock.h"
TEST_CASE("Stamp in Filename", "[Stamp In Filename]") {
Clock::TimeOfDay_t tod;
std::string baseName = "verif";
std::string pathStr = "verif_2022-05-25T16:55:23Z.bin";
unsigned int underscorePos = pathStr.find_last_of('_');
std::string stampStr = pathStr.substr(underscorePos + 1);
float seconds = 0.0;
char* prefix = nullptr;
int count =
sscanf(stampStr.c_str(),
"%4" SCNu32 "-%2" SCNu32 "-%2" SCNu32 "T%2" SCNu32 ":%2" SCNu32 ":%2" SCNu32 "Z",
&tod.year, &tod.month, &tod.day, &tod.hour, &tod.minute, &tod.second);
static_cast<void>(count);
CHECK(count == 6);
}