diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c3e4e50..fde054ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,20 @@ will consitute of a breaking change warranting a new major release: # [unreleased] +## Added + +- Added `mv`, `cp` and `rm` action helpers for the core controller for common filesystem operations. +- Extended directory listing helpers. There is now a directory listing helper which dumps the + directory listing as an action data reply immediately. For smaller directory listings, this + allows a listing without requiring a separate file downlink (which also has not been implemented + yet) + +## Changed + +- The directory listing action commands now allow compressing of either the target output file + for the directory listing into file action command, or compression in the helper which dumps + the directory listing directly. + # [v1.45.0] 2023-04-14 - q7s-package: v2.5.0 diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 49145f9f..5c4c4d62 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -233,6 +233,84 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_ case (LIST_DIRECTORY_INTO_FILE): { return actionListDirectoryIntoFile(actionId, commandedBy, data, size); } + case (LIST_DIRECTORY_DUMP_DIRECTLY): { + return actionListDirectoryDumpDirectly(actionId, commandedBy, data, size); + } + case (CP_HELPER): { + CpHelperParser parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; + } + std::ostringstream oss("cp ", std::ostringstream::ate); + if (parser.isRecursiveOptSet()) { + oss << "-r "; + } + auto &sourceTgt = parser.destTgtPair(); + oss << sourceTgt.sourceName << " " << sourceTgt.targetName; + sif::info << "CoreController: Performing copy command: " << oss.str() << std::endl; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + return returnvalue::FAILED; + } + return EXECUTION_FINISHED; + } + case (MV_HELPER): { + MvHelperParser parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; + } + std::ostringstream oss("mv ", std::ostringstream::ate); + auto &sourceTgt = parser.destTgtPair(); + oss << sourceTgt.sourceName << " " << sourceTgt.targetName; + sif::info << "CoreController: Performing move command: " << oss.str() << std::endl; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + return returnvalue::FAILED; + } + return EXECUTION_FINISHED; + } + case (RM_HELPER): { + RmHelperParser parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; + } + std::ostringstream oss("rm ", std::ostringstream::ate); + if (parser.isRecursiveOptSet() or parser.isForceOptSet()) { + oss << "-"; + } + if (parser.isRecursiveOptSet()) { + oss << "r"; + } + if (parser.isForceOptSet()) { + oss << "f"; + } + size_t removeTargetSize = 0; + const char *removeTgt = parser.getRemoveTarget(removeTargetSize); + oss << " " << removeTgt; + sif::info << "CoreController: Performing remove command: " << oss.str() << std::endl; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + return returnvalue::FAILED; + } + return EXECUTION_FINISHED; + } + case (MKDIR_HELPER): { + if (size < 1) { + return HasActionsIF::INVALID_PARAMETERS; + } + std::string createdDir = std::string(reinterpret_cast(data), size); + std::ostringstream oss("mkdir ", std::ostringstream::ate); + oss << createdDir; + sif::info << "CoreController: Performing directory creation: " << oss.str() << std::endl; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + return returnvalue::FAILED; + } + return EXECUTION_FINISHED; + } case (SWITCH_REBOOT_FILE_HANDLING): { if (size < 1) { return HasActionsIF::INVALID_PARAMETERS; @@ -911,59 +989,144 @@ ReturnValue_t CoreController::initVersionFile() { return returnvalue::OK; } -ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId, - MessageQueueId_t commandedBy, - const uint8_t *data, size_t size) { - // TODO: Packet definition for clean deserialization - // 2 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with - // null termination, at least 7 bytes for minimum target file name /tmp/a with - // null termination. - if (size < 14) { - return HasActionsIF::INVALID_PARAMETERS; +ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionId, + MessageQueueId_t commandedBy, + const uint8_t *data, size_t size) { + core::ListDirectoryCmdBase parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; } - // We could also make -l optional, but I can't think of a reason why to not use -l.. - // This flag specifies to run ls with -a - bool aFlag = data[0]; - data += 1; - // This flag specifies to run ls with -R - bool RFlag = data[1]; - data += 1; - - size_t remainingSize = size - 2; - // One larger for null termination, which prevents undefined behaviour if the sent - // strings are not 0 terminated properly - std::vector repoAndTargetFileBuffer(remainingSize + 1, 0); - std::memcpy(repoAndTargetFileBuffer.data(), data, remainingSize); - const char *currentCharPtr = reinterpret_cast(repoAndTargetFileBuffer.data()); - // Full target file name - std::string repoName(currentCharPtr); - size_t repoLength = repoName.length(); - // The other string needs to be at least one letter plus NULL termination to be valid at all - // The first string also needs to be NULL terminated, but the termination is not included - // in the string length, so this is subtracted from the remaining size as well - if (repoLength > remainingSize - 3) { - return HasActionsIF::INVALID_PARAMETERS; - } - // The file length will not include the NULL termination, so we skip it - currentCharPtr += repoLength + 1; - std::string targetFileName(currentCharPtr); - std::ostringstream oss; - oss << "ls -l"; - if (aFlag) { + std::ostringstream oss("ls -l", std::ostringstream::ate); + if (parser.aFlagSet()) { oss << "a"; } - if (RFlag) { + if (parser.rFlagSet()) { oss << "R"; } - oss << " " << repoName << " > " << targetFileName; - int result = std::system(oss.str().c_str()); - if (result != 0) { - utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile"); - actionHelper.finish(false, commandedBy, actionId); + size_t repoNameLen = 0; + const char *repoName = parser.getRepoName(repoNameLen); + + oss << " " << repoName << " > " << LIST_DIR_DUMP_WORK_FILE; + sif::info << "Executing " << oss.str() << " for direct dump"; + if (parser.compressionOptionSet()) { + sif::info << " with compression"; } - return returnvalue::OK; + sif::info << std::endl; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); + return returnvalue::FAILED; + } + if (parser.compressionOptionSet()) { + std::string compressedName = LIST_DIR_DUMP_WORK_FILE + std::string(".gz"); + oss.str(""); + oss << "gzip " << LIST_DIR_DUMP_WORK_FILE; + ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); + return returnvalue::FAILED; + } + oss.str(""); + // Overwrite the work file with the compressed archive. + oss << "mv " << compressedName << " " << LIST_DIR_DUMP_WORK_FILE; + ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly"); + return returnvalue::FAILED; + } + } + std::array dirListingBuf{}; + dirListingBuf[8] = parser.compressionOptionSet(); + // First four bytes reserved for segment index. One byte for compression option information + std::strcpy(reinterpret_cast(dirListingBuf.data() + 2 * sizeof(uint32_t) + 1), repoName); + std::ifstream ifile(LIST_DIR_DUMP_WORK_FILE, std::ios::binary); + if (ifile.bad()) { + return returnvalue::FAILED; + } + std::error_code e; + size_t totalFileSize = std::filesystem::file_size(LIST_DIR_DUMP_WORK_FILE, e); + uint32_t segmentIdx = 0; + size_t dumpedBytes = 0; + size_t nextDumpLen = 0; + size_t dummy = 0; + size_t maxDumpLen = dirListingBuf.size() - 2 * sizeof(uint32_t) - 1 - repoNameLen - 1; + size_t listingDataOffset = 2 * sizeof(uint32_t) + 1 + repoNameLen + 1; + uint32_t chunks = totalFileSize / maxDumpLen; + if (totalFileSize % maxDumpLen != 0) { + chunks++; + } + SerializeAdapter::serialize(&chunks, dirListingBuf.data() + sizeof(uint32_t), &dummy, + dirListingBuf.size() - sizeof(uint32_t), + SerializeIF::Endianness::NETWORK); + while (dumpedBytes < totalFileSize) { + ifile.seekg(dumpedBytes, std::ios::beg); + nextDumpLen = maxDumpLen; + if (totalFileSize - dumpedBytes < maxDumpLen) { + nextDumpLen = totalFileSize - dumpedBytes; + } + SerializeAdapter::serialize(&segmentIdx, dirListingBuf.data(), &dummy, dirListingBuf.size(), + SerializeIF::Endianness::NETWORK); + ifile.read(reinterpret_cast(dirListingBuf.data() + listingDataOffset), nextDumpLen); + result = actionHelper.reportData(commandedBy, actionId, dirListingBuf.data(), + listingDataOffset + nextDumpLen); + if (result != returnvalue::OK) { + // Remove work file when we are done + std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); + return result; + } + segmentIdx++; + dumpedBytes += nextDumpLen; + } + // Remove work file when we are done + std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); + return EXECUTION_FINISHED; +} + +ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId, + MessageQueueId_t commandedBy, + const uint8_t *data, size_t size) { + core::ListDirectoryIntoFile parser(data, size); + ReturnValue_t result = parser.parse(); + if (result != returnvalue::OK) { + return result; + } + + std::ostringstream oss("ls -l", std::ostringstream::ate); + if (parser.aFlagSet()) { + oss << "a"; + } + if (parser.rFlagSet()) { + oss << "R"; + } + + size_t repoNameLen = 0; + const char *repoName = parser.getRepoName(repoNameLen); + size_t targetFileNameLen = 0; + const char *targetFileName = parser.getTargetName(targetFileNameLen); + oss << " " << repoName << " > " << targetFileName; + sif::info << "Executing list directory request, command: " << oss.str() << std::endl; + int ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile"); + return returnvalue::FAILED; + } + + // Compression will add a .gz ending. I don't have any issue with this, it makes it explicit + // that this is a compressed file. + if (parser.compressionOptionSet()) { + oss.str(""); + oss << "gzip " << targetFileName; + sif::info << "Compressing directory listing: " << oss.str() << std::endl; + ret = std::system(oss.str().c_str()); + if (ret != 0) { + utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile"); + return returnvalue::FAILED; + } + } + return EXECUTION_FINISHED; } ReturnValue_t CoreController::initBootCopyFile() { @@ -2049,6 +2212,8 @@ ReturnValue_t CoreController::executeSwUpdate(SwUpdateSources sourceDir, const u if (not exists(archivePath, e)) { return HasFileSystemIF::FILE_DOES_NOT_EXIST; } + // TODO: Decompressing without limiting memory usage with xz is actually a bit risky.. + // But has not been an issue so far. ostringstream cmd("tar -xJf", ios::app); cmd << " " << archivePath << " -C " << prefixPath; int result = system(cmd.str().c_str()); diff --git a/bsp_q7s/core/CoreController.h b/bsp_q7s/core/CoreController.h index c1f5e40d..d44907e8 100644 --- a/bsp_q7s/core/CoreController.h +++ b/bsp_q7s/core/CoreController.h @@ -66,6 +66,7 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe static constexpr char CHIP_0_COPY_1_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi0-gold-rootfs"; static constexpr char CHIP_1_COPY_0_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi1-nom-rootfs"; static constexpr char CHIP_1_COPY_1_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi1-gold-rootfs"; + static constexpr char LIST_DIR_DUMP_WORK_FILE[] = "/tmp/dir_listing.tmp"; static constexpr dur_millis_t INIT_SD_CARD_CHECK_TIMEOUT = 5000; static constexpr dur_millis_t DEFAULT_SD_CARD_CHECK_TIMEOUT = 60000; @@ -250,6 +251,12 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe ReturnValue_t actionListDirectoryIntoFile(ActionId_t actionId, MessageQueueId_t commandedBy, const uint8_t* data, size_t size); + ReturnValue_t actionListDirectoryDumpDirectly(ActionId_t actionId, MessageQueueId_t commandedBy, + const uint8_t* data, size_t size); + + ReturnValue_t actionListDirectoryCommonCommandCreator(const uint8_t* data, size_t size, + std::ostringstream& oss); + ReturnValue_t actionXscReboot(const uint8_t* data, size_t size); ReturnValue_t actionReboot(const uint8_t* data, size_t size); diff --git a/mission/sysDefs.h b/mission/sysDefs.h index 424b5752..e8303524 100644 --- a/mission/sysDefs.h +++ b/mission/sysDefs.h @@ -32,7 +32,6 @@ static constexpr char VERSION_FILE_NAME[] = "version.txt"; static constexpr char REBOOT_FILE_NAME[] = "reboot.txt"; static constexpr char TIME_FILE_NAME[] = "time_backup.txt"; -static constexpr ActionId_t LIST_DIRECTORY_INTO_FILE = 0; static constexpr ActionId_t ANNOUNCE_VERSION = 1; static constexpr ActionId_t ANNOUNCE_CURRENT_IMAGE = 2; static constexpr ActionId_t ANNOUNCE_BOOT_COUNTS = 3; @@ -59,6 +58,13 @@ static constexpr ActionId_t EXECUTE_SHELL_CMD_BLOCKING = 40; static constexpr ActionId_t EXECUTE_SHELL_CMD_NON_BLOCKING = 41; static constexpr ActionId_t SYSTEMCTL_CMD_EXECUTOR = 42; +static constexpr ActionId_t LIST_DIRECTORY_INTO_FILE = 50; +static constexpr ActionId_t LIST_DIRECTORY_DUMP_DIRECTLY = 51; +static constexpr ActionId_t CP_HELPER = 52; +static constexpr ActionId_t MV_HELPER = 53; +static constexpr ActionId_t RM_HELPER = 54; +static constexpr ActionId_t MKDIR_HELPER = 55; + static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::CORE; static constexpr Event ALLOC_FAILURE = event::makeEvent(SUBSYSTEM_ID, 0, severity::MEDIUM); @@ -96,6 +102,198 @@ static constexpr Event I2C_REBOOT = event::makeEvent(SUBSYSTEM_ID, 11, severity: //! [EXPORT] : [COMMENT] PDEC recovery through reset was not possible, performing full reboot. static constexpr Event PDEC_REBOOT = event::makeEvent(SUBSYSTEM_ID, 12, severity::HIGH); +class ListDirectoryCmdBase { + public: // TODO: Packet definition for clean deserialization + // 3 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with + // null termination + static constexpr size_t MIN_DATA_LEN = 8; + + ListDirectoryCmdBase(const uint8_t* data, size_t maxSize) : data(data), maxSize(maxSize) {} + virtual ~ListDirectoryCmdBase() = default; + + virtual ReturnValue_t parse() { + if (maxSize < MIN_DATA_LEN) { + return SerializeIF::STREAM_TOO_SHORT; + } + aFlag = data[0]; + rFlag = data[1]; + compressOption = data[2]; + + repoNameLen = strnlen(reinterpret_cast(data + 3), maxSize - 3); + // Last byte MUST be null terminated! + if (repoNameLen >= maxSize - 3) { + return HasActionsIF::INVALID_PARAMETERS; + } + repoName = reinterpret_cast(data + 3); + return returnvalue::OK; + } + + bool aFlagSet() const { return this->aFlag; } + bool rFlagSet() const { return this->rFlag; } + + bool compressionOptionSet() const { return this->compressOption; } + + const char* getRepoName(size_t& repoNameLen) const { + repoNameLen = this->repoNameLen; + return this->repoName; + } + + size_t getBaseSize() { + // Include NULL termination + if (repoName != nullptr) { + return 3 + repoNameLen + 1; + } + return 0; + } + + protected: + const uint8_t* data; + size_t maxSize; + + bool aFlag = false; + bool rFlag = false; + bool compressOption = false; + const char* repoName = nullptr; + size_t repoNameLen = 0; +}; + +class ListDirectoryIntoFile : public ListDirectoryCmdBase { + public: + // TODO: Packet definition for clean deserialization + // 3 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with + // null termination, at least 7 bytes for minimum target file name /tmp/a with + // null termination. + static constexpr size_t MIN_DATA_LEN = 15; + + ListDirectoryIntoFile(const uint8_t* data, size_t maxSize) + : ListDirectoryCmdBase(data, maxSize) {} + ReturnValue_t parse() override { + if (maxSize < MIN_DATA_LEN) { + return SerializeIF::STREAM_TOO_SHORT; + } + ReturnValue_t result = ListDirectoryCmdBase::parse(); + if (result != returnvalue::OK) { + return result; + } + + targetNameLen = + strnlen(reinterpret_cast(data + getBaseSize()), maxSize - getBaseSize()); + if (targetNameLen >= maxSize - getBaseSize()) { + // Again: String MUST be null terminated. + return HasActionsIF::INVALID_PARAMETERS; + } + targetName = reinterpret_cast(data + getBaseSize()); + return result; + } + const char* getTargetName(size_t& targetNameLen) const { + targetNameLen = this->targetNameLen; + return this->targetName; + } + + private: + const char* targetName = nullptr; + size_t targetNameLen = 0; +}; + +struct SourceTargetPair { + const char* sourceName = nullptr; + size_t sourceNameSize = 0; + const char* targetName = nullptr; + size_t targetNameSize = 0; +}; + +static ReturnValue_t parseDestTargetString(const uint8_t* data, size_t maxLen, + SourceTargetPair& destTgt) { + if (maxLen < 4) { + return SerializeIF::STREAM_TOO_SHORT; + } + destTgt.sourceNameSize = strnlen(reinterpret_cast(data), maxLen); + if (destTgt.sourceNameSize >= maxLen) { + return HasActionsIF::INVALID_PARAMETERS; + } + destTgt.sourceName = reinterpret_cast(data); + size_t remainingLen = maxLen - destTgt.sourceNameSize - 1; + if (remainingLen == 0) { + return HasActionsIF::INVALID_PARAMETERS; + } + destTgt.targetNameSize = + strnlen(reinterpret_cast(data + destTgt.sourceNameSize + 1), remainingLen); + if (destTgt.targetNameSize >= remainingLen) { + return HasActionsIF::INVALID_PARAMETERS; + } + destTgt.targetName = reinterpret_cast(data + destTgt.sourceNameSize + 1); + return returnvalue::OK; +} + +class CpHelperParser { + public: + CpHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {} + + ReturnValue_t parse() { + if (maxLen < 1) { + return SerializeIF::STREAM_TOO_SHORT; + } + recursiveOpt = data[0]; + return parseDestTargetString(data + 1, maxLen - 1, destTgt); + } + const SourceTargetPair& destTgtPair() const { return destTgt; } + bool isRecursiveOptSet() const { return recursiveOpt; } + + private: + const uint8_t* data; + size_t maxLen; + bool recursiveOpt = false; + SourceTargetPair destTgt; +}; + +class MvHelperParser { + public: + MvHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {} + + ReturnValue_t parse() { return parseDestTargetString(data, maxLen, destTgt); } + const SourceTargetPair& destTgtPair() const { return destTgt; } + + private: + const uint8_t* data; + size_t maxLen; + SourceTargetPair destTgt; +}; + +class RmHelperParser { + public: + RmHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {} + + ReturnValue_t parse() { + if (maxLen < 2) { + return SerializeIF::STREAM_TOO_SHORT; + } + recursiveOpt = data[0]; + forceOpt = data[1]; + removeTargetSize = strnlen(reinterpret_cast(data + 2), maxLen - 2); + // Must be null-terminated + if (removeTargetSize >= maxLen - 2) { + return HasActionsIF::INVALID_PARAMETERS; + } + removeTarget = reinterpret_cast(data + 2); + return returnvalue::OK; + } + bool isRecursiveOptSet() const { return recursiveOpt; } + bool isForceOptSet() const { return forceOpt; } + + const char* getRemoveTarget(size_t& removeTargetSize) { + removeTargetSize = this->removeTargetSize; + return removeTarget; + } + + private: + const uint8_t* data; + size_t maxLen; + bool recursiveOpt = false; + bool forceOpt = false; + const char* removeTarget = nullptr; + size_t removeTargetSize = 0; +}; + } // namespace core #endif /* MISSION_SYSDEFS_H_ */ diff --git a/tmtc b/tmtc index d00e4247..d623e83b 160000 --- a/tmtc +++ b/tmtc @@ -1 +1 @@ -Subproject commit d00e4247f66eb2f010f1fe53ee7f59b7fb992481 +Subproject commit d623e83be8536f8887f1da00a1f0c4be191db1e8