From b8d010cd39a4a1e08bf04b6c7bbb4d78c7dd0ef6 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 15 Apr 2023 13:01:41 +0200 Subject: [PATCH 1/9] CHANGELOG + FS helpers --- CHANGELOG.md | 14 +++ bsp_q7s/core/CoreController.cpp | 201 +++++++++++++++++++++++++------- bsp_q7s/core/CoreController.h | 7 ++ mission/sysDefs.h | 199 ++++++++++++++++++++++++++++++- 4 files changed, 379 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c3e4e50..fde054ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,20 @@ will consitute of a breaking change warranting a new major release: # [unreleased] +## Added + +- Added `mv`, `cp` and `rm` action helpers for the core controller for common filesystem operations. +- Extended directory listing helpers. There is now a directory listing helper which dumps the + directory listing as an action data reply immediately. For smaller directory listings, this + allows a listing without requiring a separate file downlink (which also has not been implemented + yet) + +## Changed + +- The directory listing action commands now allow compressing of either the target output file + for the directory listing into file action command, or compression in the helper which dumps + the directory listing directly. + # [v1.45.0] 2023-04-14 - q7s-package: v2.5.0 diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 49145f9f..d9d30119 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -233,6 +233,67 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_ case (LIST_DIRECTORY_INTO_FILE): { return actionListDirectoryIntoFile(actionId, commandedBy, data, size); } + case (LIST_DIRECTORY_DUMP_DIRECTLY): { + return actionListDirectoryDumpDirectly(actionId, commandedBy, data, size); + } + case (CP_HELPER): { + CpHelperParser parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; + } + std::ostringstream oss("cp ", std::ostringstream::ate); + if (parser.isRecursiveOptSet()) { + oss << "-r "; + } + auto &sourceTgt = parser.destTgtPair(); + oss << sourceTgt.sourceName << " " << sourceTgt.targetName; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + return returnvalue::FAILED; + } + return EXECUTION_FINISHED; + } + case (MV_HELPER): { + MvHelperParser parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; + } + std::ostringstream oss("mv ", std::ostringstream::ate); + auto &sourceTgt = parser.destTgtPair(); + oss << sourceTgt.sourceName << " " << sourceTgt.targetName; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + return returnvalue::FAILED; + } + return EXECUTION_FINISHED; + } + case (RM_HELPER): { + RmHelperParser parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; + } + std::ostringstream oss("rm ", std::ostringstream::ate); + if (parser.isRecursiveOptSet() or parser.isForceOptSet()) { + oss << "-"; + } + if (parser.isRecursiveOptSet()) { + oss << "r"; + } + if (parser.isForceOptSet()) { + oss << "f"; + } + size_t removeTargetSize = 0; + const char *removeTgt = parser.getRemoveTarget(removeTargetSize); + oss << " " << removeTgt; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + return returnvalue::FAILED; + } + return EXECUTION_FINISHED; + } case (SWITCH_REBOOT_FILE_HANDLING): { if (size < 1) { return HasActionsIF::INVALID_PARAMETERS; @@ -911,57 +972,115 @@ ReturnValue_t CoreController::initVersionFile() { return returnvalue::OK; } -ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId, - MessageQueueId_t commandedBy, - const uint8_t *data, size_t size) { - // TODO: Packet definition for clean deserialization - // 2 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with - // null termination, at least 7 bytes for minimum target file name /tmp/a with - // null termination. - if (size < 14) { - return HasActionsIF::INVALID_PARAMETERS; +ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionId, + MessageQueueId_t commandedBy, + const uint8_t *data, size_t size) { + core::ListDirectoryCmdBase parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; } - // We could also make -l optional, but I can't think of a reason why to not use -l.. - // This flag specifies to run ls with -a - bool aFlag = data[0]; - data += 1; - // This flag specifies to run ls with -R - bool RFlag = data[1]; - data += 1; - - size_t remainingSize = size - 2; - // One larger for null termination, which prevents undefined behaviour if the sent - // strings are not 0 terminated properly - std::vector repoAndTargetFileBuffer(remainingSize + 1, 0); - std::memcpy(repoAndTargetFileBuffer.data(), data, remainingSize); - const char *currentCharPtr = reinterpret_cast(repoAndTargetFileBuffer.data()); - // Full target file name - std::string repoName(currentCharPtr); - size_t repoLength = repoName.length(); - // The other string needs to be at least one letter plus NULL termination to be valid at all - // The first string also needs to be NULL terminated, but the termination is not included - // in the string length, so this is subtracted from the remaining size as well - if (repoLength > remainingSize - 3) { - return HasActionsIF::INVALID_PARAMETERS; - } - // The file length will not include the NULL termination, so we skip it - currentCharPtr += repoLength + 1; - std::string targetFileName(currentCharPtr); - std::ostringstream oss; + std::ostringstream oss("ls -l", std::ostringstream::ate); oss << "ls -l"; - if (aFlag) { + if (parser.aFlagSet()) { oss << "a"; } - if (RFlag) { + if (parser.rFlagSet()) { oss << "R"; } + size_t repoNameLen = 0; + const char *repoName = parser.getRepoName(repoNameLen); + + oss << " " << repoName << " > " << LIST_DIR_DUMP_WORK_FILE; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); + return returnvalue::FAILED; + } + if (parser.compressionOptionSet()) { + std::string compressedName = LIST_DIR_DUMP_WORK_FILE + std::string(".tar.xz"); + oss.str(""); + oss << "tar -cJvf /tmp/dir_listing_compressed.tmp " << LIST_DIR_DUMP_WORK_FILE; + ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); + return returnvalue::FAILED; + } + oss.str(""); + // Overwrite the old work file with the compressed archive. + oss << "mv /tmp/dir_listing_compressed.tmp " << LIST_DIR_DUMP_WORK_FILE; + ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); + return returnvalue::FAILED; + } + } + std::array dirListingBuf{}; + std::ifstream ifile("/tmp/dir_listing.tmp", std::ios::binary); + std::error_code e; + size_t totalFileSize = std::filesystem::file_size(LIST_DIR_DUMP_WORK_FILE, e); + size_t dumpedBytes = 0; + size_t nextDumpLen = 0; + while (dumpedBytes < totalFileSize) { + nextDumpLen = 1024; + if (totalFileSize - dumpedBytes < 1024) { + nextDumpLen = totalFileSize - dumpedBytes; + } + ifile.read(reinterpret_cast(dirListingBuf.data()), nextDumpLen); + result = actionHelper.reportData(commandedBy, actionId, dirListingBuf.data(), nextDumpLen); + if (result != returnvalue::OK) { + // Remove work file when we are done + std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); + return result; + } + } + // Remove work file when we are done + std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); + return returnvalue::OK; +} + +ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId, + MessageQueueId_t commandedBy, + const uint8_t *data, size_t size) { + core::ListDirectoryIntoFile parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; + } + + std::ostringstream oss("ls -l", std::ostringstream::ate); + oss << "ls -l"; + if (parser.aFlagSet()) { + oss << "a"; + } + if (parser.rFlagSet()) { + oss << "R"; + } + + size_t repoNameLen = 0; + const char *repoName = parser.getRepoName(repoNameLen); + size_t targetFileNameLen = 0; + const char *targetFileName = parser.getTargetName(targetFileNameLen); oss << " " << repoName << " > " << targetFileName; - int result = std::system(oss.str().c_str()); - if (result != 0) { + sif::info << "Executing list directory request, command: " << oss.str() << std::endl; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile"); - actionHelper.finish(false, commandedBy, actionId); + return returnvalue::FAILED; + } + + if (parser.compressionOptionSet()) { + std::string compressedName = targetFileName + std::string(".tar.xz"); + oss.str(""); + oss << "tar -cJvf " << compressedName << " " << targetFileName; + sif::info << "Compressing directory listing: " << oss.str() << std::endl; + ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile"); + return returnvalue::FAILED; + } } return returnvalue::OK; } diff --git a/bsp_q7s/core/CoreController.h b/bsp_q7s/core/CoreController.h index c1f5e40d..d44907e8 100644 --- a/bsp_q7s/core/CoreController.h +++ b/bsp_q7s/core/CoreController.h @@ -66,6 +66,7 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe static constexpr char CHIP_0_COPY_1_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi0-gold-rootfs"; static constexpr char CHIP_1_COPY_0_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi1-nom-rootfs"; static constexpr char CHIP_1_COPY_1_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi1-gold-rootfs"; + static constexpr char LIST_DIR_DUMP_WORK_FILE[] = "/tmp/dir_listing.tmp"; static constexpr dur_millis_t INIT_SD_CARD_CHECK_TIMEOUT = 5000; static constexpr dur_millis_t DEFAULT_SD_CARD_CHECK_TIMEOUT = 60000; @@ -250,6 +251,12 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe ReturnValue_t actionListDirectoryIntoFile(ActionId_t actionId, MessageQueueId_t commandedBy, const uint8_t* data, size_t size); + ReturnValue_t actionListDirectoryDumpDirectly(ActionId_t actionId, MessageQueueId_t commandedBy, + const uint8_t* data, size_t size); + + ReturnValue_t actionListDirectoryCommonCommandCreator(const uint8_t* data, size_t size, + std::ostringstream& oss); + ReturnValue_t actionXscReboot(const uint8_t* data, size_t size); ReturnValue_t actionReboot(const uint8_t* data, size_t size); diff --git a/mission/sysDefs.h b/mission/sysDefs.h index 424b5752..3ec3f7e0 100644 --- a/mission/sysDefs.h +++ b/mission/sysDefs.h @@ -32,7 +32,6 @@ static constexpr char VERSION_FILE_NAME[] = "version.txt"; static constexpr char REBOOT_FILE_NAME[] = "reboot.txt"; static constexpr char TIME_FILE_NAME[] = "time_backup.txt"; -static constexpr ActionId_t LIST_DIRECTORY_INTO_FILE = 0; static constexpr ActionId_t ANNOUNCE_VERSION = 1; static constexpr ActionId_t ANNOUNCE_CURRENT_IMAGE = 2; static constexpr ActionId_t ANNOUNCE_BOOT_COUNTS = 3; @@ -59,6 +58,12 @@ static constexpr ActionId_t EXECUTE_SHELL_CMD_BLOCKING = 40; static constexpr ActionId_t EXECUTE_SHELL_CMD_NON_BLOCKING = 41; static constexpr ActionId_t SYSTEMCTL_CMD_EXECUTOR = 42; +static constexpr ActionId_t LIST_DIRECTORY_INTO_FILE = 50; +static constexpr ActionId_t LIST_DIRECTORY_DUMP_DIRECTLY = 51; +static constexpr ActionId_t CP_HELPER = 52; +static constexpr ActionId_t MV_HELPER = 53; +static constexpr ActionId_t RM_HELPER = 54; + static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::CORE; static constexpr Event ALLOC_FAILURE = event::makeEvent(SUBSYSTEM_ID, 0, severity::MEDIUM); @@ -96,6 +101,198 @@ static constexpr Event I2C_REBOOT = event::makeEvent(SUBSYSTEM_ID, 11, severity: //! [EXPORT] : [COMMENT] PDEC recovery through reset was not possible, performing full reboot. static constexpr Event PDEC_REBOOT = event::makeEvent(SUBSYSTEM_ID, 12, severity::HIGH); +class ListDirectoryCmdBase { + public: // TODO: Packet definition for clean deserialization + // 3 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with + // null termination + static constexpr size_t MIN_DATA_LEN = 8; + + ListDirectoryCmdBase(const uint8_t* data, size_t maxSize) : data(data), maxSize(maxSize) {} + virtual ~ListDirectoryCmdBase() = default; + + virtual ReturnValue_t parse() { + if (maxSize < MIN_DATA_LEN) { + return SerializeIF::STREAM_TOO_SHORT; + } + aFlag = data[0]; + rFlag = data[1]; + compressOption = data[2]; + + repoNameLen = strnlen(reinterpret_cast(data + 3), maxSize - 3); + // Last byte MUST be null terminated! + if (repoNameLen >= maxSize - 3) { + return HasActionsIF::INVALID_PARAMETERS; + } + repoName = reinterpret_cast(data + 3); + return returnvalue::OK; + } + + bool aFlagSet() const { return this->aFlag; } + bool rFlagSet() const { return this->rFlag; } + + bool compressionOptionSet() const { return this->compressOption; } + + const char* getRepoName(size_t& repoNameLen) const { + return this->repoName; + repoNameLen = this->repoNameLen; + } + + size_t getBaseSize() { + // Include NULL termination + if (repoName != nullptr) { + return 3 + repoNameLen + 1; + } + return 0; + } + + protected: + const uint8_t* data; + size_t maxSize; + + bool aFlag = false; + bool rFlag = false; + bool compressOption = false; + const char* repoName = nullptr; + size_t repoNameLen = 0; +}; + +class ListDirectoryIntoFile : public ListDirectoryCmdBase { + public: + // TODO: Packet definition for clean deserialization + // 3 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with + // null termination, at least 7 bytes for minimum target file name /tmp/a with + // null termination. + static constexpr size_t MIN_DATA_LEN = 15; + + ListDirectoryIntoFile(const uint8_t* data, size_t maxSize) + : ListDirectoryCmdBase(data, maxSize) {} + ReturnValue_t parse() override { + if (maxSize < MIN_DATA_LEN) { + return SerializeIF::STREAM_TOO_SHORT; + } + ReturnValue_t result = ListDirectoryCmdBase::parse(); + if (result != returnvalue::OK) { + return result; + } + + targetNameLen = + strnlen(reinterpret_cast(data + getBaseSize()), maxSize - getBaseSize()); + if (targetNameLen >= maxSize - getBaseSize()) { + // Again: String MUST be null terminated. + return HasActionsIF::INVALID_PARAMETERS; + } + targetName = reinterpret_cast(data + getBaseSize()); + return result; + } + const char* getTargetName(size_t& targetNameLen) const { + return this->targetName; + targetNameLen = this->targetNameLen; + } + + private: + const char* targetName = nullptr; + size_t targetNameLen = 0; +}; + +struct SourceTargetPair { + const char* sourceName = nullptr; + size_t sourceNameSize = 0; + const char* targetName = nullptr; + size_t targetNameSize = 0; +}; + +static ReturnValue_t parseDestTargetString(const uint8_t* data, size_t maxLen, + SourceTargetPair& destTgt) { + if (maxLen < 4) { + return SerializeIF::STREAM_TOO_SHORT; + } + destTgt.sourceNameSize = strnlen(reinterpret_cast(data), maxLen); + if (destTgt.sourceNameSize >= maxLen) { + return HasActionsIF::INVALID_PARAMETERS; + } + destTgt.sourceName = reinterpret_cast(data); + size_t remainingLen = maxLen - destTgt.sourceNameSize + 1; + if (remainingLen == 0) { + return HasActionsIF::INVALID_PARAMETERS; + } + destTgt.targetNameSize = + strnlen(reinterpret_cast(data + destTgt.sourceNameSize + 1), remainingLen); + if (destTgt.targetNameSize >= remainingLen) { + return HasActionsIF::INVALID_PARAMETERS; + } + destTgt.targetName = reinterpret_cast(data + destTgt.targetNameSize + 1); + return returnvalue::OK; +} + +class CpHelperParser { + public: + CpHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {} + + ReturnValue_t parse() { + if (maxLen < 1) { + return SerializeIF::STREAM_TOO_SHORT; + } + recursiveOpt = data[0]; + return parseDestTargetString(data + 1, maxLen - 1, destTgt); + } + const SourceTargetPair& destTgtPair() const { return destTgt; } + bool isRecursiveOptSet() const { return recursiveOpt; } + + private: + const uint8_t* data; + size_t maxLen; + bool recursiveOpt = false; + SourceTargetPair destTgt; +}; + +class MvHelperParser { + public: + MvHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {} + + ReturnValue_t parse() { return parseDestTargetString(data, maxLen, destTgt); } + const SourceTargetPair& destTgtPair() const { return destTgt; } + + private: + const uint8_t* data; + size_t maxLen; + SourceTargetPair destTgt; +}; + +class RmHelperParser { + public: + RmHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {} + + ReturnValue_t parse() { + if (maxLen < 2) { + return SerializeIF::STREAM_TOO_SHORT; + } + recursiveOpt = data[0]; + forceOpt = data[1]; + removeTargetSize = strnlen(reinterpret_cast(data + 2), maxLen - 2); + // Must be null-terminated + if (removeTargetSize >= maxLen - 2) { + return HasActionsIF::INVALID_PARAMETERS; + } + removeTarget = reinterpret_cast(data + 2); + return returnvalue::OK; + } + bool isRecursiveOptSet() const { return recursiveOpt; } + bool isForceOptSet() const { return forceOpt; } + + const char* getRemoveTarget(size_t& removeTargetSize) { + removeTargetSize = this->removeTargetSize; + return removeTarget; + } + + private: + const uint8_t* data; + size_t maxLen; + bool recursiveOpt = false; + bool forceOpt = false; + const char* removeTarget = nullptr; + size_t removeTargetSize = 0; +}; + } // namespace core #endif /* MISSION_SYSDEFS_H_ */ From d439aedee740bddf8a64873f5a649d441b7041d5 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 15 Apr 2023 13:03:44 +0200 Subject: [PATCH 2/9] bugfix --- bsp_q7s/core/CoreController.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index d9d30119..4e30e2d7 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -982,7 +982,6 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI } std::ostringstream oss("ls -l", std::ostringstream::ate); - oss << "ls -l"; if (parser.aFlagSet()) { oss << "a"; } @@ -1051,7 +1050,6 @@ ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId, } std::ostringstream oss("ls -l", std::ostringstream::ate); - oss << "ls -l"; if (parser.aFlagSet()) { oss << "a"; } From 143002de486a47d38bb341ed7dc9d1569ddaa7ed Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 15 Apr 2023 13:06:16 +0200 Subject: [PATCH 3/9] some improvements --- bsp_q7s/core/CoreController.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 4e30e2d7..08889d70 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -993,6 +993,7 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI const char *repoName = parser.getRepoName(repoNameLen); oss << " " << repoName << " > " << LIST_DIR_DUMP_WORK_FILE; + sif::info << "Executing " << oss.str() << " for direct dump" << std::endl; int ret = std::system(oss.str().c_str()); if (ret != 0) { utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); @@ -1017,7 +1018,10 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI } } std::array dirListingBuf{}; - std::ifstream ifile("/tmp/dir_listing.tmp", std::ios::binary); + std::ifstream ifile(LIST_DIR_DUMP_WORK_FILE, std::ios::binary); + if (ifile.bad()) { + return returnvalue::FAILED; + } std::error_code e; size_t totalFileSize = std::filesystem::file_size(LIST_DIR_DUMP_WORK_FILE, e); size_t dumpedBytes = 0; From 0185691dba780776e2de65fcd503f1f20042690a Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 15 Apr 2023 20:51:46 +0200 Subject: [PATCH 4/9] add more context info for dumped dir listing --- bsp_q7s/core/CoreController.cpp | 17 ++++++++++++++--- tmtc | 2 +- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 08889d70..a2e1d5c4 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -1018,26 +1018,37 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI } } std::array dirListingBuf{}; + // First four bytes reserved for segment index. + std::memcpy(dirListingBuf.data() + sizeof(uint32_t), repoName, repoNameLen); + // Null termination for ground parser. + dirListingBuf[sizeof(uint32_t) + repoNameLen] = '\0'; std::ifstream ifile(LIST_DIR_DUMP_WORK_FILE, std::ios::binary); if (ifile.bad()) { return returnvalue::FAILED; } std::error_code e; size_t totalFileSize = std::filesystem::file_size(LIST_DIR_DUMP_WORK_FILE, e); + uint32_t segmentIdx = 0; size_t dumpedBytes = 0; size_t nextDumpLen = 0; + size_t dummy = 0; + size_t maxDumpLen = dirListingBuf.size() - repoNameLen - sizeof(uint32_t); while (dumpedBytes < totalFileSize) { - nextDumpLen = 1024; - if (totalFileSize - dumpedBytes < 1024) { + nextDumpLen = maxDumpLen; + if (totalFileSize - dumpedBytes < maxDumpLen) { nextDumpLen = totalFileSize - dumpedBytes; } - ifile.read(reinterpret_cast(dirListingBuf.data()), nextDumpLen); + SerializeAdapter::serialize(&segmentIdx, dirListingBuf.data(), &dummy, dirListingBuf.size(), + SerializeIF::Endianness::NETWORK); + ifile.read(reinterpret_cast(dirListingBuf.data() + sizeof(uint32_t) + repoNameLen + 1), + nextDumpLen); result = actionHelper.reportData(commandedBy, actionId, dirListingBuf.data(), nextDumpLen); if (result != returnvalue::OK) { // Remove work file when we are done std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); return result; } + segmentIdx++; } // Remove work file when we are done std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); diff --git a/tmtc b/tmtc index d00e4247..37bb164c 160000 --- a/tmtc +++ b/tmtc @@ -1 +1 @@ -Subproject commit d00e4247f66eb2f010f1fe53ee7f59b7fb992481 +Subproject commit 37bb164cc461abd51668038e02c5f76db70e7ca6 From 7bf880a29f6bb965d0326bdc0fb7c7179f8e7ed4 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 15 Apr 2023 22:17:50 +0200 Subject: [PATCH 5/9] use gzip directly --- bsp_q7s/core/CoreController.cpp | 29 +++++++++++++++++++++++------ mission/sysDefs.h | 5 +++-- tmtc | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index a2e1d5c4..659e1f97 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -248,6 +248,7 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_ } auto &sourceTgt = parser.destTgtPair(); oss << sourceTgt.sourceName << " " << sourceTgt.targetName; + sif::info << "CoreController: Performing copy command: " << oss.str() << std::endl; int ret = std::system(oss.str().c_str()); if (ret != 0) { return returnvalue::FAILED; @@ -263,6 +264,7 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_ std::ostringstream oss("mv ", std::ostringstream::ate); auto &sourceTgt = parser.destTgtPair(); oss << sourceTgt.sourceName << " " << sourceTgt.targetName; + sif::info << "CoreController: Performing move command: " << oss.str() << std::endl; int ret = std::system(oss.str().c_str()); if (ret != 0) { return returnvalue::FAILED; @@ -288,6 +290,21 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_ size_t removeTargetSize = 0; const char *removeTgt = parser.getRemoveTarget(removeTargetSize); oss << " " << removeTgt; + sif::info << "CoreController: Performing remove command: " << oss.str() << std::endl; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + return returnvalue::FAILED; + } + return EXECUTION_FINISHED; + } + case (MKDIR_HELPER): { + if (size < 1) { + return HasActionsIF::INVALID_PARAMETERS; + } + std::string createdDir = std::string(reinterpret_cast(data), size); + std::ostringstream oss("mkdir ", std::ostringstream::ate); + oss << createdDir; + sif::info << "CoreController: Performing directory creation: " << oss.str() << std::endl; int ret = std::system(oss.str().c_str()); if (ret != 0) { return returnvalue::FAILED; @@ -1000,17 +1017,17 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI return returnvalue::FAILED; } if (parser.compressionOptionSet()) { - std::string compressedName = LIST_DIR_DUMP_WORK_FILE + std::string(".tar.xz"); + std::string compressedName = LIST_DIR_DUMP_WORK_FILE + std::string(".gz"); oss.str(""); - oss << "tar -cJvf /tmp/dir_listing_compressed.tmp " << LIST_DIR_DUMP_WORK_FILE; + oss << "gzip " << LIST_DIR_DUMP_WORK_FILE; ret = std::system(oss.str().c_str()); if (ret != 0) { utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); return returnvalue::FAILED; } oss.str(""); - // Overwrite the old work file with the compressed archive. - oss << "mv /tmp/dir_listing_compressed.tmp " << LIST_DIR_DUMP_WORK_FILE; + // Overwrite the work file with the compressed archive. + oss << "mv " << compressedName << " " << LIST_DIR_DUMP_WORK_FILE; ret = std::system(oss.str().c_str()); if (ret != 0) { utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); @@ -1085,9 +1102,9 @@ ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId, } if (parser.compressionOptionSet()) { - std::string compressedName = targetFileName + std::string(".tar.xz"); + std::string compressedName = targetFileName + std::string(".gz"); oss.str(""); - oss << "tar -cJvf " << compressedName << " " << targetFileName; + oss << "gzip -c " << compressedName << " > " << targetFileName; sif::info << "Compressing directory listing: " << oss.str() << std::endl; ret = std::system(oss.str().c_str()); if (ret != 0) { diff --git a/mission/sysDefs.h b/mission/sysDefs.h index 3ec3f7e0..cc654e90 100644 --- a/mission/sysDefs.h +++ b/mission/sysDefs.h @@ -63,6 +63,7 @@ static constexpr ActionId_t LIST_DIRECTORY_DUMP_DIRECTLY = 51; static constexpr ActionId_t CP_HELPER = 52; static constexpr ActionId_t MV_HELPER = 53; static constexpr ActionId_t RM_HELPER = 54; +static constexpr ActionId_t MKDIR_HELPER = 55; static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::CORE; @@ -211,7 +212,7 @@ static ReturnValue_t parseDestTargetString(const uint8_t* data, size_t maxLen, return HasActionsIF::INVALID_PARAMETERS; } destTgt.sourceName = reinterpret_cast(data); - size_t remainingLen = maxLen - destTgt.sourceNameSize + 1; + size_t remainingLen = maxLen - destTgt.sourceNameSize - 1; if (remainingLen == 0) { return HasActionsIF::INVALID_PARAMETERS; } @@ -220,7 +221,7 @@ static ReturnValue_t parseDestTargetString(const uint8_t* data, size_t maxLen, if (destTgt.targetNameSize >= remainingLen) { return HasActionsIF::INVALID_PARAMETERS; } - destTgt.targetName = reinterpret_cast(data + destTgt.targetNameSize + 1); + destTgt.targetName = reinterpret_cast(data + destTgt.sourceNameSize + 1); return returnvalue::OK; } diff --git a/tmtc b/tmtc index 37bb164c..77e90328 160000 --- a/tmtc +++ b/tmtc @@ -1 +1 @@ -Subproject commit 37bb164cc461abd51668038e02c5f76db70e7ca6 +Subproject commit 77e90328a1e1ffd0d79bc4c26932ad48adbf2920 From f2c71d962af022537e012131a91ea656e802977a Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 15 Apr 2023 23:00:34 +0200 Subject: [PATCH 6/9] done soon --- bsp_q7s/core/CoreController.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 659e1f97..763a6856 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -1066,10 +1066,11 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI return result; } segmentIdx++; + dumpedBytes += nextDumpLen; } // Remove work file when we are done std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); - return returnvalue::OK; + return EXECUTION_FINISHED; } ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId, @@ -1112,7 +1113,7 @@ ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId, return returnvalue::FAILED; } } - return returnvalue::OK; + return EXECUTION_FINISHED; } ReturnValue_t CoreController::initBootCopyFile() { @@ -2198,6 +2199,8 @@ ReturnValue_t CoreController::executeSwUpdate(SwUpdateSources sourceDir, const u if (not exists(archivePath, e)) { return HasFileSystemIF::FILE_DOES_NOT_EXIST; } + // TODO: Decompressing without limiting memory usage with xz is actually a bit risky.. + // But has not been an issue so far. ostringstream cmd("tar -xJf", ios::app); cmd << " " << archivePath << " -C " << prefixPath; int result = system(cmd.str().c_str()); From 767a0eda3041b4580e0fdd66eca5a8d502c9bbd3 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 15 Apr 2023 23:11:00 +0200 Subject: [PATCH 7/9] hopefully last bugfix --- bsp_q7s/core/CoreController.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 763a6856..46abcc5c 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -1010,7 +1010,11 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI const char *repoName = parser.getRepoName(repoNameLen); oss << " " << repoName << " > " << LIST_DIR_DUMP_WORK_FILE; - sif::info << "Executing " << oss.str() << " for direct dump" << std::endl; + sif::info << "Executing " << oss.str() << " for direct dump"; + if (parser.compressionOptionSet()) { + sif::info << " with compression"; + } + sif::info << std::endl; int ret = std::system(oss.str().c_str()); if (ret != 0) { utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); @@ -1102,10 +1106,11 @@ ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId, return returnvalue::FAILED; } + // Compression will add a .gz ending. I don't have any issue with this, it makes it explicit + // that this is a compressed file. if (parser.compressionOptionSet()) { - std::string compressedName = targetFileName + std::string(".gz"); oss.str(""); - oss << "gzip -c " << compressedName << " > " << targetFileName; + oss << "gzip " << targetFileName; sif::info << "Compressing directory listing: " << oss.str() << std::endl; ret = std::system(oss.str().c_str()); if (ret != 0) { From 016fab105eb883044dfc720b8caa24a1b9c9e61a Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 15 Apr 2023 23:13:04 +0200 Subject: [PATCH 8/9] bump tmtc --- tmtc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tmtc b/tmtc index 77e90328..bb463aa0 160000 --- a/tmtc +++ b/tmtc @@ -1 +1 @@ -Subproject commit 77e90328a1e1ffd0d79bc4c26932ad48adbf2920 +Subproject commit bb463aa05cf90809edf852d348ab1f7593ece74e From f4fedd20c984f51b7d086af9d106ec61f41ee957 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sun, 16 Apr 2023 02:56:46 +0200 Subject: [PATCH 9/9] added more bugfixes for list dir dump --- bsp_q7s/core/CoreController.cpp | 24 ++++++++++++++++-------- mission/sysDefs.h | 4 ++-- tmtc | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 46abcc5c..5c4c4d62 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -1039,10 +1039,9 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI } } std::array dirListingBuf{}; - // First four bytes reserved for segment index. - std::memcpy(dirListingBuf.data() + sizeof(uint32_t), repoName, repoNameLen); - // Null termination for ground parser. - dirListingBuf[sizeof(uint32_t) + repoNameLen] = '\0'; + dirListingBuf[8] = parser.compressionOptionSet(); + // First four bytes reserved for segment index. One byte for compression option information + std::strcpy(reinterpret_cast(dirListingBuf.data() + 2 * sizeof(uint32_t) + 1), repoName); std::ifstream ifile(LIST_DIR_DUMP_WORK_FILE, std::ios::binary); if (ifile.bad()) { return returnvalue::FAILED; @@ -1053,17 +1052,26 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI size_t dumpedBytes = 0; size_t nextDumpLen = 0; size_t dummy = 0; - size_t maxDumpLen = dirListingBuf.size() - repoNameLen - sizeof(uint32_t); + size_t maxDumpLen = dirListingBuf.size() - 2 * sizeof(uint32_t) - 1 - repoNameLen - 1; + size_t listingDataOffset = 2 * sizeof(uint32_t) + 1 + repoNameLen + 1; + uint32_t chunks = totalFileSize / maxDumpLen; + if (totalFileSize % maxDumpLen != 0) { + chunks++; + } + SerializeAdapter::serialize(&chunks, dirListingBuf.data() + sizeof(uint32_t), &dummy, + dirListingBuf.size() - sizeof(uint32_t), + SerializeIF::Endianness::NETWORK); while (dumpedBytes < totalFileSize) { + ifile.seekg(dumpedBytes, std::ios::beg); nextDumpLen = maxDumpLen; if (totalFileSize - dumpedBytes < maxDumpLen) { nextDumpLen = totalFileSize - dumpedBytes; } SerializeAdapter::serialize(&segmentIdx, dirListingBuf.data(), &dummy, dirListingBuf.size(), SerializeIF::Endianness::NETWORK); - ifile.read(reinterpret_cast(dirListingBuf.data() + sizeof(uint32_t) + repoNameLen + 1), - nextDumpLen); - result = actionHelper.reportData(commandedBy, actionId, dirListingBuf.data(), nextDumpLen); + ifile.read(reinterpret_cast(dirListingBuf.data() + listingDataOffset), nextDumpLen); + result = actionHelper.reportData(commandedBy, actionId, dirListingBuf.data(), + listingDataOffset + nextDumpLen); if (result != returnvalue::OK) { // Remove work file when we are done std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); diff --git a/mission/sysDefs.h b/mission/sysDefs.h index cc654e90..e8303524 100644 --- a/mission/sysDefs.h +++ b/mission/sysDefs.h @@ -134,8 +134,8 @@ class ListDirectoryCmdBase { bool compressionOptionSet() const { return this->compressOption; } const char* getRepoName(size_t& repoNameLen) const { - return this->repoName; repoNameLen = this->repoNameLen; + return this->repoName; } size_t getBaseSize() { @@ -186,8 +186,8 @@ class ListDirectoryIntoFile : public ListDirectoryCmdBase { return result; } const char* getTargetName(size_t& targetNameLen) const { - return this->targetName; targetNameLen = this->targetNameLen; + return this->targetName; } private: diff --git a/tmtc b/tmtc index bb463aa0..d623e83b 160000 --- a/tmtc +++ b/tmtc @@ -1 +1 @@ -Subproject commit bb463aa05cf90809edf852d348ab1f7593ece74e +Subproject commit d623e83be8536f8887f1da00a1f0c4be191db1e8