469 lines
17 KiB
C++
469 lines
17 KiB
C++
#include "PersistentTmStore.h"
|
|
|
|
#include <mission/memory/SdCardMountedIF.h>
|
|
#include <mission/tmtc/DirectTmSinkIF.h>
|
|
|
|
#include <algorithm>
|
|
#include <cinttypes>
|
|
#include <filesystem>
|
|
#include <fstream>
|
|
#include <utility>
|
|
|
|
#include "eive/definitions.h"
|
|
#include "fsfw/ipc/CommandMessage.h"
|
|
#include "fsfw/ipc/QueueFactory.h"
|
|
#include "fsfw/tmstorage/TmStoreMessage.h"
|
|
#include "mission/persistentTmStoreDefs.h"
|
|
|
|
using namespace returnvalue;
|
|
|
|
static constexpr bool DEBUG_DUMPS = false;
|
|
|
|
PersistentTmStore::PersistentTmStore(PersistentTmStoreArgs args)
|
|
: SystemObject(args.objectId),
|
|
tmStore(args.tmStore),
|
|
baseDir(args.baseDir),
|
|
baseName(std::move(args.baseName)),
|
|
sdcMan(args.sdcMan) {
|
|
tcQueue = QueueFactory::instance()->createMessageQueue();
|
|
calcDiffSeconds(args.intervalUnit, args.intervalCount);
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::cancelDump() {
|
|
state = State::IDLE;
|
|
return returnvalue::OK;
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::buildDumpSet(uint32_t fromUnixSeconds, uint32_t upToUnixSeconds) {
|
|
using namespace std::filesystem;
|
|
std::error_code e;
|
|
dumpParams.orderedDumpFilestamps.clear();
|
|
for (auto const& fileOrDir : directory_iterator(basePath)) {
|
|
if (not fileOrDir.is_regular_file(e)) {
|
|
continue;
|
|
}
|
|
dumpParams.fileSize = std::filesystem::file_size(fileOrDir.path(), e);
|
|
if (e) {
|
|
sif::error << "PersistentTmStore: Could not retrieve file size: " << e.message() << std::endl;
|
|
continue;
|
|
}
|
|
|
|
// File empty or can't even read CCSDS header.
|
|
if (dumpParams.fileSize <= 6) {
|
|
continue;
|
|
}
|
|
if (dumpParams.fileSize > fileBuf.size()) {
|
|
sif::error << "PersistentTmStore: File too large, is deleted" << std::endl;
|
|
triggerEvent(persTmStore::FILE_TOO_LARGE, dumpParams.fileSize, fileBuf.size());
|
|
std::filesystem::remove(fileOrDir.path(), e);
|
|
continue;
|
|
}
|
|
const path& file = fileOrDir.path();
|
|
struct tm fileTime {};
|
|
if (pathToTime(file, fileTime) != returnvalue::OK) {
|
|
sif::error << "Time extraction for file " << file << "failed" << std::endl;
|
|
continue;
|
|
}
|
|
auto fileEpoch = static_cast<uint32_t>(timegm(&fileTime));
|
|
if ((fileEpoch > dumpParams.fromUnixTime) and
|
|
(fileEpoch + rolloverDiffSeconds <= dumpParams.untilUnixTime)) {
|
|
std::ifstream ifile(file, std::ios::binary);
|
|
if (ifile.bad()) {
|
|
sif::error << "PersistentTmStore: File is bad" << std::endl;
|
|
// TODO: Consider deleting file here?
|
|
continue;
|
|
}
|
|
|
|
if (DEBUG_DUMPS) {
|
|
sif::debug << "Inserting file " << fileOrDir.path() << std::endl;
|
|
}
|
|
DumpIndex dumpIndex;
|
|
dumpIndex.epoch = fileEpoch;
|
|
// Multiple files for the same time are supported via a special suffix. We simply count the
|
|
// number of copies and later try to dump the same number of files with the additional
|
|
// suffixes
|
|
auto iter = dumpParams.orderedDumpFilestamps.find(dumpIndex);
|
|
if (iter != dumpParams.orderedDumpFilestamps.end()) {
|
|
dumpIndex.epoch = iter->epoch;
|
|
dumpIndex.additionalFiles = iter->additionalFiles + 1;
|
|
dumpParams.orderedDumpFilestamps.erase(dumpIndex);
|
|
} else {
|
|
dumpIndex.additionalFiles = 0;
|
|
}
|
|
dumpParams.orderedDumpFilestamps.emplace(dumpIndex);
|
|
}
|
|
}
|
|
return returnvalue::OK;
|
|
}
|
|
|
|
std::optional<uint8_t> PersistentTmStore::extractSuffix(const std::string& pathStr) {
|
|
std::string numberStr;
|
|
// Find the position of the dot at the end of the file path
|
|
size_t dotPos = pathStr.find_last_of('.');
|
|
if ((dotPos < pathStr.length()) and not std::isdigit(pathStr[dotPos + 1])) {
|
|
return std::nullopt;
|
|
}
|
|
// Extract the substring after the dot
|
|
numberStr = pathStr.substr(dotPos + 1);
|
|
std::optional<uint8_t> number;
|
|
try {
|
|
number = std::stoi(numberStr);
|
|
if (number.value() > std::numeric_limits<uint8_t>::max()) {
|
|
return std::nullopt;
|
|
}
|
|
|
|
} catch (std::invalid_argument& exception) {
|
|
sif::error << "PersistentTmStore::extractSuffix: Exception " << exception.what()
|
|
<< ", invald input string: " << numberStr << std::endl;
|
|
}
|
|
return number;
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::assignAndOrCreateMostRecentFile() {
|
|
if (not activeFile.has_value()) {
|
|
return createMostRecentFile(std::nullopt);
|
|
}
|
|
return returnvalue::OK;
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::handleCommandQueue(StorageManagerIF& ipcStore,
|
|
Command_t& execCmd) {
|
|
execCmd = CommandMessageIF::CMD_NONE;
|
|
CommandMessage cmdMessage;
|
|
ReturnValue_t result = tcQueue->receiveMessage(&cmdMessage);
|
|
if (result != returnvalue::OK) {
|
|
return result;
|
|
}
|
|
if (cmdMessage.getMessageType() == messagetypes::TM_STORE) {
|
|
Command_t cmd = cmdMessage.getCommand();
|
|
if (cmd == TmStoreMessage::DELETE_STORE_CONTENT_TIME) {
|
|
Clock::getClock_timeval(¤tTv);
|
|
store_address_t storeId = TmStoreMessage::getStoreId(&cmdMessage);
|
|
auto accessor = ipcStore.getData(storeId);
|
|
uint32_t deleteUpToUnixSeconds = 0;
|
|
size_t size = accessor.second.size();
|
|
SerializeAdapter::deSerialize(&deleteUpToUnixSeconds, accessor.second.data(), &size,
|
|
SerializeIF::Endianness::NETWORK);
|
|
execCmd = cmd;
|
|
deleteUpTo(deleteUpToUnixSeconds);
|
|
} else if (cmd == TmStoreMessage::DOWNLINK_STORE_CONTENT_TIME) {
|
|
Clock::getClock_timeval(¤tTv);
|
|
store_address_t storeId = TmStoreMessage::getStoreId(&cmdMessage);
|
|
auto accessor = ipcStore.getData(storeId);
|
|
if (accessor.second.size() < 8) {
|
|
return returnvalue::FAILED;
|
|
}
|
|
uint32_t dumpFromUnixSeconds = 0;
|
|
uint32_t dumpUntilUnixSeconds = 0;
|
|
size_t size = 8;
|
|
SerializeAdapter::deSerialize(&dumpFromUnixSeconds, accessor.second.data(), &size,
|
|
SerializeIF::Endianness::NETWORK);
|
|
SerializeAdapter::deSerialize(&dumpUntilUnixSeconds, accessor.second.data() + 4, &size,
|
|
SerializeIF::Endianness::NETWORK);
|
|
result = startDumpFromUpTo(dumpFromUnixSeconds, dumpUntilUnixSeconds);
|
|
if (result == BUSY_DUMPING) {
|
|
triggerEvent(persTmStore::BUSY_DUMPING_EVENT);
|
|
return result;
|
|
}
|
|
execCmd = cmd;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::startDumpFrom(uint32_t fromUnixSeconds) {
|
|
return startDumpFromUpTo(fromUnixSeconds, currentTv.tv_sec);
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::storePacket(PusTmReader& reader) {
|
|
using namespace std::filesystem;
|
|
if (baseDirUninitialized) {
|
|
updateBaseDir();
|
|
}
|
|
Clock::getClock_timeval(¤tTv);
|
|
// It is assumed here that the filesystem is usable.
|
|
if (not activeFile.has_value()) {
|
|
ReturnValue_t result = assignAndOrCreateMostRecentFile();
|
|
if (result != returnvalue::OK) {
|
|
return result;
|
|
}
|
|
}
|
|
|
|
bool createNewFile = false;
|
|
std::optional<uint8_t> suffix = std::nullopt;
|
|
std::error_code e;
|
|
size_t fileSize = file_size(activeFile.value(), e);
|
|
if (e) {
|
|
sif::error << "PersistentTmStore: Could not retrieve file size, "
|
|
"error "
|
|
<< e.message() << std::endl;
|
|
return returnvalue::FAILED;
|
|
}
|
|
if (currentTv.tv_sec > activeFileTv.tv_sec + static_cast<int>(rolloverDiffSeconds)) {
|
|
createNewFile = true;
|
|
currentSameSecNumber = 0;
|
|
} else if (fileSize + reader.getFullPacketLen() > fileBuf.size()) {
|
|
createNewFile = true;
|
|
if (currentSameSecNumber >= MAX_FILES_IN_ONE_SECOND) {
|
|
currentSameSecNumber = 0;
|
|
}
|
|
if (currentTv.tv_sec == activeFileTv.tv_sec) {
|
|
suffix = currentSameSecNumber++;
|
|
} else {
|
|
currentSameSecNumber = 0;
|
|
}
|
|
}
|
|
if (createNewFile) {
|
|
createMostRecentFile(suffix);
|
|
}
|
|
|
|
// Rollover conditions were handled, write to file now
|
|
std::ofstream of(activeFile.value(), std::ios::app | std::ios::binary);
|
|
of.write(reinterpret_cast<const char*>(reader.getFullData()),
|
|
static_cast<std::streamsize>(reader.getFullPacketLen()));
|
|
return returnvalue::OK;
|
|
}
|
|
|
|
MessageQueueId_t PersistentTmStore::getCommandQueue() const { return tcQueue->getId(); }
|
|
|
|
void PersistentTmStore::calcDiffSeconds(RolloverInterval intervalUnit, uint32_t intervalCount) {
|
|
if (intervalUnit == RolloverInterval::MINUTELY) {
|
|
rolloverDiffSeconds = 60 * intervalCount;
|
|
} else if (intervalUnit == RolloverInterval::HOURLY) {
|
|
rolloverDiffSeconds = 60 * 60 * intervalCount;
|
|
} else if (intervalUnit == RolloverInterval::DAILY) {
|
|
rolloverDiffSeconds = 60 * 60 * 24 * intervalCount;
|
|
}
|
|
}
|
|
|
|
bool PersistentTmStore::updateBaseDir() {
|
|
using namespace std::filesystem;
|
|
const char* currentPrefix = sdcMan.getCurrentMountPrefix();
|
|
if (currentPrefix == nullptr) {
|
|
return false;
|
|
}
|
|
basePath = path(currentPrefix) / baseDir / baseName;
|
|
std::error_code e;
|
|
if (not exists(basePath, e)) {
|
|
create_directories(basePath);
|
|
}
|
|
// Each file will have the base name as a prefix again
|
|
path preparedFullFilePath = basePath / baseName;
|
|
basePathSize = preparedFullFilePath.string().length();
|
|
std::memcpy(filePathBuf.data(), preparedFullFilePath.c_str(), basePathSize);
|
|
filePathBuf[basePathSize] = '_';
|
|
basePathSize += 1;
|
|
baseDirUninitialized = false;
|
|
return true;
|
|
}
|
|
|
|
void PersistentTmStore::deleteUpTo(uint32_t unixSeconds) {
|
|
using namespace std::filesystem;
|
|
for (auto const& file : directory_iterator(basePath)) {
|
|
if (file.is_directory() or (activeFile.has_value() and (activeFile.value() == file.path()))) {
|
|
continue;
|
|
}
|
|
// Convert file time to the UNIX epoch
|
|
struct tm fileTime {};
|
|
if (pathToTime(file.path(), fileTime) != returnvalue::OK) {
|
|
sif::error << "Time extraction for " << file << " failed" << std::endl;
|
|
continue;
|
|
}
|
|
time_t fileEpoch = timegm(&fileTime);
|
|
if (fileEpoch + rolloverDiffSeconds < unixSeconds) {
|
|
std::error_code e;
|
|
std::filesystem::remove(file.path(), e);
|
|
}
|
|
}
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::startDumpFromUpTo(uint32_t fromUnixSeconds,
|
|
uint32_t upToUnixSeconds) {
|
|
using namespace std::filesystem;
|
|
if (state == State::DUMPING) {
|
|
return returnvalue::FAILED;
|
|
}
|
|
auto dirIter = directory_iterator(basePath);
|
|
// Directory empty case.
|
|
if (dirIter == directory_iterator()) {
|
|
return returnvalue::FAILED;
|
|
}
|
|
dumpParams.fromUnixTime = fromUnixSeconds;
|
|
dumpParams.untilUnixTime = upToUnixSeconds;
|
|
buildDumpSet(fromUnixSeconds, upToUnixSeconds);
|
|
// No files in time window found.
|
|
if (dumpParams.orderedDumpFilestamps.empty()) {
|
|
return DUMP_DONE;
|
|
}
|
|
dumpParams.dumpIter = dumpParams.orderedDumpFilestamps.begin();
|
|
dumpParams.currentSameFileIdx = std::nullopt;
|
|
state = State::DUMPING;
|
|
return loadNextDumpFile();
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::loadNextDumpFile() {
|
|
using namespace std::filesystem;
|
|
dumpParams.currentSize = 0;
|
|
std::error_code e;
|
|
// Handle iteration, which does not necessarily have to increment the set iterator
|
|
// if there are multiple files for a given timestamp.
|
|
auto handleIteration = [&](DumpIndex& dumpIndex) {
|
|
if (dumpIndex.additionalFiles > 0) {
|
|
if (not dumpParams.currentSameFileIdx.has_value()) {
|
|
// Initialize the file index and stay on same file
|
|
dumpParams.currentSameFileIdx = 0;
|
|
return;
|
|
} else if (dumpParams.currentSameFileIdx.value() < dumpIndex.additionalFiles - 1) {
|
|
dumpParams.currentSameFileIdx = dumpParams.currentSameFileIdx.value() + 1;
|
|
return;
|
|
}
|
|
}
|
|
// File will change, reset this field for correct state-keeping.
|
|
dumpParams.currentSameFileIdx = std::nullopt;
|
|
// Increment iterator for next cycle.
|
|
dumpParams.dumpIter++;
|
|
};
|
|
while (dumpParams.dumpIter != dumpParams.orderedDumpFilestamps.end()) {
|
|
DumpIndex dumpIndex = *dumpParams.dumpIter;
|
|
timeval tv{};
|
|
tv.tv_sec = dumpIndex.epoch;
|
|
size_t fullPathLength = 0;
|
|
createFileName(tv, dumpParams.currentSameFileIdx, fullPathLength);
|
|
dumpParams.currentFile =
|
|
path(std::string(reinterpret_cast<const char*>(filePathBuf.data()), fullPathLength));
|
|
if (DEBUG_DUMPS) {
|
|
sif::debug << baseName << " dump: Loading " << dumpParams.currentFile << std::endl;
|
|
}
|
|
dumpParams.fileSize = std::filesystem::file_size(dumpParams.currentFile, e);
|
|
if (e) {
|
|
// TODO: Event?
|
|
sif::error << "PersistentTmStore: Could not load next dump file: " << e.message()
|
|
<< std::endl;
|
|
handleIteration(dumpIndex);
|
|
continue;
|
|
}
|
|
std::ifstream ifile(dumpParams.currentFile, std::ios::binary);
|
|
if (ifile.bad()) {
|
|
sif::error << "PersistentTmStore: File is bad. Loading next file" << std::endl;
|
|
handleIteration(dumpIndex);
|
|
continue;
|
|
}
|
|
ifile.read(reinterpret_cast<char*>(fileBuf.data()),
|
|
static_cast<std::streamsize>(dumpParams.fileSize));
|
|
// Next file is loaded, exit the loop.
|
|
handleIteration(dumpIndex);
|
|
return returnvalue::OK;
|
|
}
|
|
// Directory iterator was consumed and we are done.
|
|
state = State::IDLE;
|
|
return DUMP_DONE;
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::getNextDumpPacket(PusTmReader& reader, bool& fileHasSwapped) {
|
|
if (state == State::IDLE) {
|
|
return DUMP_DONE;
|
|
}
|
|
if (dumpParams.pendingPacketDump) {
|
|
return returnvalue::FAILED;
|
|
}
|
|
fileHasSwapped = false;
|
|
reader.setReadOnlyData(fileBuf.data() + dumpParams.currentSize,
|
|
fileBuf.size() - dumpParams.currentSize);
|
|
// CRC check to fully ensure this is a valid TM
|
|
ReturnValue_t result = reader.parseDataWithCrcCheck();
|
|
if (result != returnvalue::OK) {
|
|
sif::error << "PersistentTmStore: Parsing of PUS TM failed with code " << result << std::endl;
|
|
triggerEvent(persTmStore::POSSIBLE_FILE_CORRUPTION, result, dumpParams.currentFileUnixStamp);
|
|
// Delete the file and load next. Could use better algorithm to partially
|
|
// restore the file dump, but for now do not trust the file.
|
|
std::error_code e;
|
|
std::filesystem::remove(dumpParams.currentFile.c_str(), e);
|
|
if (dumpParams.currentFile == activeFile) {
|
|
activeFile == std::nullopt;
|
|
assignAndOrCreateMostRecentFile();
|
|
}
|
|
fileHasSwapped = true;
|
|
return loadNextDumpFile();
|
|
}
|
|
dumpParams.pendingPacketDump = true;
|
|
return returnvalue::OK;
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::confirmDump(const PusTmReader& reader, bool& fileHasSwapped) {
|
|
dumpParams.pendingPacketDump = false;
|
|
dumpParams.currentSize += reader.getFullPacketLen();
|
|
if (dumpParams.currentSize >= dumpParams.fileSize) {
|
|
fileHasSwapped = true;
|
|
return loadNextDumpFile();
|
|
}
|
|
fileHasSwapped = false;
|
|
return returnvalue::OK;
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::pathToTime(const std::filesystem::path& path, struct tm& time) {
|
|
auto pathStr = path.string();
|
|
size_t splitChar = pathStr.find('_');
|
|
auto timeOnlyStr = pathStr.substr(splitChar + 1);
|
|
if (nullptr == strptime(timeOnlyStr.c_str(), config::FILE_DATE_FORMAT, &time)) {
|
|
return returnvalue::FAILED;
|
|
}
|
|
return returnvalue::OK;
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::createMostRecentFile(std::optional<uint8_t> suffix) {
|
|
using namespace std::filesystem;
|
|
size_t currentIdx;
|
|
createFileName(currentTv, suffix, currentIdx);
|
|
path newPath(std::string(reinterpret_cast<const char*>(filePathBuf.data()), currentIdx));
|
|
std::ofstream of(newPath, std::ios::binary);
|
|
activeFile = newPath;
|
|
activeFileTv = currentTv;
|
|
return returnvalue::OK;
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::initializeTmStore() {
|
|
Clock::getClock_timeval(¤tTv);
|
|
updateBaseDir();
|
|
// Reset active file, base directory might have changed.
|
|
activeFile = std::nullopt;
|
|
return assignAndOrCreateMostRecentFile();
|
|
}
|
|
|
|
PersistentTmStore::State PersistentTmStore::getState() const { return state; }
|
|
|
|
void PersistentTmStore::getStartAndEndTimeCurrentOrLastDump(uint32_t& startTime,
|
|
uint32_t& endTime) const {
|
|
startTime = dumpParams.fromUnixTime;
|
|
endTime = dumpParams.untilUnixTime;
|
|
}
|
|
|
|
ReturnValue_t PersistentTmStore::createFileName(timeval& tv, std::optional<uint8_t> suffix,
|
|
size_t& fullPathLength) {
|
|
unsigned currentIdx = basePathSize;
|
|
time_t epoch = tv.tv_sec;
|
|
struct tm* time = gmtime(&epoch);
|
|
size_t writtenBytes = strftime(reinterpret_cast<char*>(filePathBuf.data() + currentIdx),
|
|
filePathBuf.size(), config::FILE_DATE_FORMAT, time);
|
|
if (writtenBytes == 0) {
|
|
sif::error << "PersistentTmStore::createMostRecentFile: Could not create file timestamp"
|
|
<< std::endl;
|
|
return returnvalue::FAILED;
|
|
}
|
|
currentIdx += writtenBytes;
|
|
char* res = strcpy(reinterpret_cast<char*>(filePathBuf.data() + currentIdx), ".bin");
|
|
if (res == nullptr) {
|
|
return returnvalue::FAILED;
|
|
}
|
|
currentIdx += 4;
|
|
if (suffix.has_value()) {
|
|
std::string fullSuffix = "." + std::to_string(suffix.value());
|
|
res = strcpy(reinterpret_cast<char*>(filePathBuf.data() + currentIdx), fullSuffix.c_str());
|
|
if (res == nullptr) {
|
|
return returnvalue::FAILED;
|
|
}
|
|
currentIdx += fullSuffix.size();
|
|
}
|
|
fullPathLength = currentIdx;
|
|
return returnvalue::OK;
|
|
}
|