From b8d010cd39a4a1e08bf04b6c7bbb4d78c7dd0ef6 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 15 Apr 2023 13:01:41 +0200 Subject: [PATCH] CHANGELOG + FS helpers --- CHANGELOG.md | 14 +++ bsp_q7s/core/CoreController.cpp | 201 +++++++++++++++++++++++++------- bsp_q7s/core/CoreController.h | 7 ++ mission/sysDefs.h | 199 ++++++++++++++++++++++++++++++- 4 files changed, 379 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c3e4e50..fde054ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,20 @@ will consitute of a breaking change warranting a new major release: # [unreleased] +## Added + +- Added `mv`, `cp` and `rm` action helpers for the core controller for common filesystem operations. +- Extended directory listing helpers. There is now a directory listing helper which dumps the + directory listing as an action data reply immediately. For smaller directory listings, this + allows a listing without requiring a separate file downlink (which also has not been implemented + yet) + +## Changed + +- The directory listing action commands now allow compressing of either the target output file + for the directory listing into file action command, or compression in the helper which dumps + the directory listing directly. + # [v1.45.0] 2023-04-14 - q7s-package: v2.5.0 diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 49145f9f..d9d30119 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -233,6 +233,67 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_ case (LIST_DIRECTORY_INTO_FILE): { return actionListDirectoryIntoFile(actionId, commandedBy, data, size); } + case (LIST_DIRECTORY_DUMP_DIRECTLY): { + return actionListDirectoryDumpDirectly(actionId, commandedBy, data, size); + } + case (CP_HELPER): { + CpHelperParser parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; + } + std::ostringstream oss("cp ", std::ostringstream::ate); + if (parser.isRecursiveOptSet()) { + oss << "-r "; + } + auto &sourceTgt = parser.destTgtPair(); + oss << sourceTgt.sourceName << " " << sourceTgt.targetName; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + return returnvalue::FAILED; + } + return EXECUTION_FINISHED; + } + case (MV_HELPER): { + MvHelperParser parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; + } + std::ostringstream oss("mv ", std::ostringstream::ate); + auto &sourceTgt = parser.destTgtPair(); + oss << sourceTgt.sourceName << " " << sourceTgt.targetName; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + return returnvalue::FAILED; + } + return EXECUTION_FINISHED; + } + case (RM_HELPER): { + RmHelperParser parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; + } + std::ostringstream oss("rm ", std::ostringstream::ate); + if (parser.isRecursiveOptSet() or parser.isForceOptSet()) { + oss << "-"; + } + if (parser.isRecursiveOptSet()) { + oss << "r"; + } + if (parser.isForceOptSet()) { + oss << "f"; + } + size_t removeTargetSize = 0; + const char *removeTgt = parser.getRemoveTarget(removeTargetSize); + oss << " " << removeTgt; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + return returnvalue::FAILED; + } + return EXECUTION_FINISHED; + } case (SWITCH_REBOOT_FILE_HANDLING): { if (size < 1) { return HasActionsIF::INVALID_PARAMETERS; @@ -911,57 +972,115 @@ ReturnValue_t CoreController::initVersionFile() { return returnvalue::OK; } -ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId, - MessageQueueId_t commandedBy, - const uint8_t *data, size_t size) { - // TODO: Packet definition for clean deserialization - // 2 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with - // null termination, at least 7 bytes for minimum target file name /tmp/a with - // null termination. - if (size < 14) { - return HasActionsIF::INVALID_PARAMETERS; +ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionId, + MessageQueueId_t commandedBy, + const uint8_t *data, size_t size) { + core::ListDirectoryCmdBase parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; } - // We could also make -l optional, but I can't think of a reason why to not use -l.. - // This flag specifies to run ls with -a - bool aFlag = data[0]; - data += 1; - // This flag specifies to run ls with -R - bool RFlag = data[1]; - data += 1; - - size_t remainingSize = size - 2; - // One larger for null termination, which prevents undefined behaviour if the sent - // strings are not 0 terminated properly - std::vector repoAndTargetFileBuffer(remainingSize + 1, 0); - std::memcpy(repoAndTargetFileBuffer.data(), data, remainingSize); - const char *currentCharPtr = reinterpret_cast(repoAndTargetFileBuffer.data()); - // Full target file name - std::string repoName(currentCharPtr); - size_t repoLength = repoName.length(); - // The other string needs to be at least one letter plus NULL termination to be valid at all - // The first string also needs to be NULL terminated, but the termination is not included - // in the string length, so this is subtracted from the remaining size as well - if (repoLength > remainingSize - 3) { - return HasActionsIF::INVALID_PARAMETERS; - } - // The file length will not include the NULL termination, so we skip it - currentCharPtr += repoLength + 1; - std::string targetFileName(currentCharPtr); - std::ostringstream oss; + std::ostringstream oss("ls -l", std::ostringstream::ate); oss << "ls -l"; - if (aFlag) { + if (parser.aFlagSet()) { oss << "a"; } - if (RFlag) { + if (parser.rFlagSet()) { oss << "R"; } + size_t repoNameLen = 0; + const char *repoName = parser.getRepoName(repoNameLen); + + oss << " " << repoName << " > " << LIST_DIR_DUMP_WORK_FILE; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); + return returnvalue::FAILED; + } + if (parser.compressionOptionSet()) { + std::string compressedName = LIST_DIR_DUMP_WORK_FILE + std::string(".tar.xz"); + oss.str(""); + oss << "tar -cJvf /tmp/dir_listing_compressed.tmp " << LIST_DIR_DUMP_WORK_FILE; + ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); + return returnvalue::FAILED; + } + oss.str(""); + // Overwrite the old work file with the compressed archive. + oss << "mv /tmp/dir_listing_compressed.tmp " << LIST_DIR_DUMP_WORK_FILE; + ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); + return returnvalue::FAILED; + } + } + std::array dirListingBuf{}; + std::ifstream ifile("/tmp/dir_listing.tmp", std::ios::binary); + std::error_code e; + size_t totalFileSize = std::filesystem::file_size(LIST_DIR_DUMP_WORK_FILE, e); + size_t dumpedBytes = 0; + size_t nextDumpLen = 0; + while (dumpedBytes < totalFileSize) { + nextDumpLen = 1024; + if (totalFileSize - dumpedBytes < 1024) { + nextDumpLen = totalFileSize - dumpedBytes; + } + ifile.read(reinterpret_cast(dirListingBuf.data()), nextDumpLen); + result = actionHelper.reportData(commandedBy, actionId, dirListingBuf.data(), nextDumpLen); + if (result != returnvalue::OK) { + // Remove work file when we are done + std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); + return result; + } + } + // Remove work file when we are done + std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); + return returnvalue::OK; +} + +ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId, + MessageQueueId_t commandedBy, + const uint8_t *data, size_t size) { + core::ListDirectoryIntoFile parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; + } + + std::ostringstream oss("ls -l", std::ostringstream::ate); + oss << "ls -l"; + if (parser.aFlagSet()) { + oss << "a"; + } + if (parser.rFlagSet()) { + oss << "R"; + } + + size_t repoNameLen = 0; + const char *repoName = parser.getRepoName(repoNameLen); + size_t targetFileNameLen = 0; + const char *targetFileName = parser.getTargetName(targetFileNameLen); oss << " " << repoName << " > " << targetFileName; - int result = std::system(oss.str().c_str()); - if (result != 0) { + sif::info << "Executing list directory request, command: " << oss.str() << std::endl; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile"); - actionHelper.finish(false, commandedBy, actionId); + return returnvalue::FAILED; + } + + if (parser.compressionOptionSet()) { + std::string compressedName = targetFileName + std::string(".tar.xz"); + oss.str(""); + oss << "tar -cJvf " << compressedName << " " << targetFileName; + sif::info << "Compressing directory listing: " << oss.str() << std::endl; + ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile"); + return returnvalue::FAILED; + } } return returnvalue::OK; } diff --git a/bsp_q7s/core/CoreController.h b/bsp_q7s/core/CoreController.h index c1f5e40d..d44907e8 100644 --- a/bsp_q7s/core/CoreController.h +++ b/bsp_q7s/core/CoreController.h @@ -66,6 +66,7 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe static constexpr char CHIP_0_COPY_1_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi0-gold-rootfs"; static constexpr char CHIP_1_COPY_0_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi1-nom-rootfs"; static constexpr char CHIP_1_COPY_1_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi1-gold-rootfs"; + static constexpr char LIST_DIR_DUMP_WORK_FILE[] = "/tmp/dir_listing.tmp"; static constexpr dur_millis_t INIT_SD_CARD_CHECK_TIMEOUT = 5000; static constexpr dur_millis_t DEFAULT_SD_CARD_CHECK_TIMEOUT = 60000; @@ -250,6 +251,12 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe ReturnValue_t actionListDirectoryIntoFile(ActionId_t actionId, MessageQueueId_t commandedBy, const uint8_t* data, size_t size); + ReturnValue_t actionListDirectoryDumpDirectly(ActionId_t actionId, MessageQueueId_t commandedBy, + const uint8_t* data, size_t size); + + ReturnValue_t actionListDirectoryCommonCommandCreator(const uint8_t* data, size_t size, + std::ostringstream& oss); + ReturnValue_t actionXscReboot(const uint8_t* data, size_t size); ReturnValue_t actionReboot(const uint8_t* data, size_t size); diff --git a/mission/sysDefs.h b/mission/sysDefs.h index 424b5752..3ec3f7e0 100644 --- a/mission/sysDefs.h +++ b/mission/sysDefs.h @@ -32,7 +32,6 @@ static constexpr char VERSION_FILE_NAME[] = "version.txt"; static constexpr char REBOOT_FILE_NAME[] = "reboot.txt"; static constexpr char TIME_FILE_NAME[] = "time_backup.txt"; -static constexpr ActionId_t LIST_DIRECTORY_INTO_FILE = 0; static constexpr ActionId_t ANNOUNCE_VERSION = 1; static constexpr ActionId_t ANNOUNCE_CURRENT_IMAGE = 2; static constexpr ActionId_t ANNOUNCE_BOOT_COUNTS = 3; @@ -59,6 +58,12 @@ static constexpr ActionId_t EXECUTE_SHELL_CMD_BLOCKING = 40; static constexpr ActionId_t EXECUTE_SHELL_CMD_NON_BLOCKING = 41; static constexpr ActionId_t SYSTEMCTL_CMD_EXECUTOR = 42; +static constexpr ActionId_t LIST_DIRECTORY_INTO_FILE = 50; +static constexpr ActionId_t LIST_DIRECTORY_DUMP_DIRECTLY = 51; +static constexpr ActionId_t CP_HELPER = 52; +static constexpr ActionId_t MV_HELPER = 53; +static constexpr ActionId_t RM_HELPER = 54; + static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::CORE; static constexpr Event ALLOC_FAILURE = event::makeEvent(SUBSYSTEM_ID, 0, severity::MEDIUM); @@ -96,6 +101,198 @@ static constexpr Event I2C_REBOOT = event::makeEvent(SUBSYSTEM_ID, 11, severity: //! [EXPORT] : [COMMENT] PDEC recovery through reset was not possible, performing full reboot. static constexpr Event PDEC_REBOOT = event::makeEvent(SUBSYSTEM_ID, 12, severity::HIGH); +class ListDirectoryCmdBase { + public: // TODO: Packet definition for clean deserialization + // 3 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with + // null termination + static constexpr size_t MIN_DATA_LEN = 8; + + ListDirectoryCmdBase(const uint8_t* data, size_t maxSize) : data(data), maxSize(maxSize) {} + virtual ~ListDirectoryCmdBase() = default; + + virtual ReturnValue_t parse() { + if (maxSize < MIN_DATA_LEN) { + return SerializeIF::STREAM_TOO_SHORT; + } + aFlag = data[0]; + rFlag = data[1]; + compressOption = data[2]; + + repoNameLen = strnlen(reinterpret_cast(data + 3), maxSize - 3); + // Last byte MUST be null terminated! + if (repoNameLen >= maxSize - 3) { + return HasActionsIF::INVALID_PARAMETERS; + } + repoName = reinterpret_cast(data + 3); + return returnvalue::OK; + } + + bool aFlagSet() const { return this->aFlag; } + bool rFlagSet() const { return this->rFlag; } + + bool compressionOptionSet() const { return this->compressOption; } + + const char* getRepoName(size_t& repoNameLen) const { + return this->repoName; + repoNameLen = this->repoNameLen; + } + + size_t getBaseSize() { + // Include NULL termination + if (repoName != nullptr) { + return 3 + repoNameLen + 1; + } + return 0; + } + + protected: + const uint8_t* data; + size_t maxSize; + + bool aFlag = false; + bool rFlag = false; + bool compressOption = false; + const char* repoName = nullptr; + size_t repoNameLen = 0; +}; + +class ListDirectoryIntoFile : public ListDirectoryCmdBase { + public: + // TODO: Packet definition for clean deserialization + // 3 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with + // null termination, at least 7 bytes for minimum target file name /tmp/a with + // null termination. + static constexpr size_t MIN_DATA_LEN = 15; + + ListDirectoryIntoFile(const uint8_t* data, size_t maxSize) + : ListDirectoryCmdBase(data, maxSize) {} + ReturnValue_t parse() override { + if (maxSize < MIN_DATA_LEN) { + return SerializeIF::STREAM_TOO_SHORT; + } + ReturnValue_t result = ListDirectoryCmdBase::parse(); + if (result != returnvalue::OK) { + return result; + } + + targetNameLen = + strnlen(reinterpret_cast(data + getBaseSize()), maxSize - getBaseSize()); + if (targetNameLen >= maxSize - getBaseSize()) { + // Again: String MUST be null terminated. + return HasActionsIF::INVALID_PARAMETERS; + } + targetName = reinterpret_cast(data + getBaseSize()); + return result; + } + const char* getTargetName(size_t& targetNameLen) const { + return this->targetName; + targetNameLen = this->targetNameLen; + } + + private: + const char* targetName = nullptr; + size_t targetNameLen = 0; +}; + +struct SourceTargetPair { + const char* sourceName = nullptr; + size_t sourceNameSize = 0; + const char* targetName = nullptr; + size_t targetNameSize = 0; +}; + +static ReturnValue_t parseDestTargetString(const uint8_t* data, size_t maxLen, + SourceTargetPair& destTgt) { + if (maxLen < 4) { + return SerializeIF::STREAM_TOO_SHORT; + } + destTgt.sourceNameSize = strnlen(reinterpret_cast(data), maxLen); + if (destTgt.sourceNameSize >= maxLen) { + return HasActionsIF::INVALID_PARAMETERS; + } + destTgt.sourceName = reinterpret_cast(data); + size_t remainingLen = maxLen - destTgt.sourceNameSize + 1; + if (remainingLen == 0) { + return HasActionsIF::INVALID_PARAMETERS; + } + destTgt.targetNameSize = + strnlen(reinterpret_cast(data + destTgt.sourceNameSize + 1), remainingLen); + if (destTgt.targetNameSize >= remainingLen) { + return HasActionsIF::INVALID_PARAMETERS; + } + destTgt.targetName = reinterpret_cast(data + destTgt.targetNameSize + 1); + return returnvalue::OK; +} + +class CpHelperParser { + public: + CpHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {} + + ReturnValue_t parse() { + if (maxLen < 1) { + return SerializeIF::STREAM_TOO_SHORT; + } + recursiveOpt = data[0]; + return parseDestTargetString(data + 1, maxLen - 1, destTgt); + } + const SourceTargetPair& destTgtPair() const { return destTgt; } + bool isRecursiveOptSet() const { return recursiveOpt; } + + private: + const uint8_t* data; + size_t maxLen; + bool recursiveOpt = false; + SourceTargetPair destTgt; +}; + +class MvHelperParser { + public: + MvHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {} + + ReturnValue_t parse() { return parseDestTargetString(data, maxLen, destTgt); } + const SourceTargetPair& destTgtPair() const { return destTgt; } + + private: + const uint8_t* data; + size_t maxLen; + SourceTargetPair destTgt; +}; + +class RmHelperParser { + public: + RmHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {} + + ReturnValue_t parse() { + if (maxLen < 2) { + return SerializeIF::STREAM_TOO_SHORT; + } + recursiveOpt = data[0]; + forceOpt = data[1]; + removeTargetSize = strnlen(reinterpret_cast(data + 2), maxLen - 2); + // Must be null-terminated + if (removeTargetSize >= maxLen - 2) { + return HasActionsIF::INVALID_PARAMETERS; + } + removeTarget = reinterpret_cast(data + 2); + return returnvalue::OK; + } + bool isRecursiveOptSet() const { return recursiveOpt; } + bool isForceOptSet() const { return forceOpt; } + + const char* getRemoveTarget(size_t& removeTargetSize) { + removeTargetSize = this->removeTargetSize; + return removeTarget; + } + + private: + const uint8_t* data; + size_t maxLen; + bool recursiveOpt = false; + bool forceOpt = false; + const char* removeTarget = nullptr; + size_t removeTargetSize = 0; +}; + } // namespace core #endif /* MISSION_SYSDEFS_H_ */