Merge pull request 'FS helpers' (#604) from core_ctrl_fs_helpers into develop

Reviewed-on: #604
Reviewed-by: Marius Eggert <eggertm@irs.uni-stuttgart.de>

CHANGELOG.md

@@ -16,6 +16,20 @@ will consitute of a breaking change warranting a new major release:
 
 # [unreleased]
 
+## Added
+
+- Added `mv`, `cp` and `rm` action helpers for the core controller for common filesystem operations.
+- Extended directory listing helpers. There is now a directory listing helper which dumps the
+  directory listing as an action data reply immediately. For smaller directory listings, this
+  allows a listing without requiring a separate file downlink (which also has not been implemented
+  yet)
+
+## Changed
+
+- The directory listing action commands now allow compressing of either the target output file
+  for the directory listing into file action command, or compression in the helper which dumps
+  the directory listing directly.
+
 # [v1.45.0] 2023-04-14
 
 - q7s-package: v2.5.0

bsp_q7s/core/CoreController.cpp

@@ -233,6 +233,84 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_
     case (LIST_DIRECTORY_INTO_FILE): {
       return actionListDirectoryIntoFile(actionId, commandedBy, data, size);
     }
+    case (LIST_DIRECTORY_DUMP_DIRECTLY): {
+      return actionListDirectoryDumpDirectly(actionId, commandedBy, data, size);
+    }
+    case (CP_HELPER): {
+      CpHelperParser parser(data, size);
+      ReturnValue_t result = parser.parse();
+      if (result != returnvalue::OK) {
+        return result;
+      }
+      std::ostringstream oss("cp ", std::ostringstream::ate);
+      if (parser.isRecursiveOptSet()) {
+        oss << "-r ";
+      }
+      auto &sourceTgt = parser.destTgtPair();
+      oss << sourceTgt.sourceName << " " << sourceTgt.targetName;
+      sif::info << "CoreController: Performing copy command: " << oss.str() << std::endl;
+      int ret = std::system(oss.str().c_str());
+      if (ret != 0) {
+        return returnvalue::FAILED;
+      }
+      return EXECUTION_FINISHED;
+    }
+    case (MV_HELPER): {
+      MvHelperParser parser(data, size);
+      ReturnValue_t result = parser.parse();
+      if (result != returnvalue::OK) {
+        return result;
+      }
+      std::ostringstream oss("mv ", std::ostringstream::ate);
+      auto &sourceTgt = parser.destTgtPair();
+      oss << sourceTgt.sourceName << " " << sourceTgt.targetName;
+      sif::info << "CoreController: Performing move command: " << oss.str() << std::endl;
+      int ret = std::system(oss.str().c_str());
+      if (ret != 0) {
+        return returnvalue::FAILED;
+      }
+      return EXECUTION_FINISHED;
+    }
+    case (RM_HELPER): {
+      RmHelperParser parser(data, size);
+      ReturnValue_t result = parser.parse();
+      if (result != returnvalue::OK) {
+        return result;
+      }
+      std::ostringstream oss("rm ", std::ostringstream::ate);
+      if (parser.isRecursiveOptSet() or parser.isForceOptSet()) {
+        oss << "-";
+      }
+      if (parser.isRecursiveOptSet()) {
+        oss << "r";
+      }
+      if (parser.isForceOptSet()) {
+        oss << "f";
+      }
+      size_t removeTargetSize = 0;
+      const char *removeTgt = parser.getRemoveTarget(removeTargetSize);
+      oss << " " << removeTgt;
+      sif::info << "CoreController: Performing remove command: " << oss.str() << std::endl;
+      int ret = std::system(oss.str().c_str());
+      if (ret != 0) {
+        return returnvalue::FAILED;
+      }
+      return EXECUTION_FINISHED;
+    }
+    case (MKDIR_HELPER): {
+      if (size < 1) {
+        return HasActionsIF::INVALID_PARAMETERS;
+      }
+      std::string createdDir = std::string(reinterpret_cast<const char *>(data), size);
+      std::ostringstream oss("mkdir ", std::ostringstream::ate);
+      oss << createdDir;
+      sif::info << "CoreController: Performing directory creation: " << oss.str() << std::endl;
+      int ret = std::system(oss.str().c_str());
+      if (ret != 0) {
+        return returnvalue::FAILED;
+      }
+      return EXECUTION_FINISHED;
+    }
     case (SWITCH_REBOOT_FILE_HANDLING): {
       if (size < 1) {
         return HasActionsIF::INVALID_PARAMETERS;
@@ -911,59 +989,144 @@ ReturnValue_t CoreController::initVersionFile() {
   return returnvalue::OK;
 }
 
-ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId,
+ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionId,
                                                               MessageQueueId_t commandedBy,
                                                               const uint8_t *data, size_t size) {
-  // TODO: Packet definition for clean deserialization
+  core::ListDirectoryCmdBase parser(data, size);
-  // 2 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with
+  ReturnValue_t result = parser.parse();
-  // null termination, at least 7 bytes for minimum target file name /tmp/a with
+  if (result != returnvalue::OK) {
-  // null termination.
+    return result;
-  if (size < 14) {
-    return HasActionsIF::INVALID_PARAMETERS;
   }
-  // We could also make -l optional, but I can't think of a reason why to not use -l..
 
-  // This flag specifies to run ls with -a
+  std::ostringstream oss("ls -l", std::ostringstream::ate);
-  bool aFlag = data[0];
+  if (parser.aFlagSet()) {
-  data += 1;
-  // This flag specifies to run ls with -R
-  bool RFlag = data[1];
-  data += 1;
-
-  size_t remainingSize = size - 2;
-  // One larger for null termination, which prevents undefined behaviour if the sent
-  // strings are not 0 terminated properly
-  std::vector<uint8_t> repoAndTargetFileBuffer(remainingSize + 1, 0);
-  std::memcpy(repoAndTargetFileBuffer.data(), data, remainingSize);
-  const char *currentCharPtr = reinterpret_cast<const char *>(repoAndTargetFileBuffer.data());
-  // Full target file name
-  std::string repoName(currentCharPtr);
-  size_t repoLength = repoName.length();
-  // The other string needs to be at least one letter plus NULL termination to be valid at all
-  // The first string also needs to be NULL terminated, but the termination is not included
-  // in the string length, so this is subtracted from the remaining size as well
-  if (repoLength > remainingSize - 3) {
-    return HasActionsIF::INVALID_PARAMETERS;
-  }
-  // The file length will not include the NULL termination, so we skip it
-  currentCharPtr += repoLength + 1;
-  std::string targetFileName(currentCharPtr);
-  std::ostringstream oss;
-  oss << "ls -l";
-  if (aFlag) {
     oss << "a";
   }
-  if (RFlag) {
+  if (parser.rFlagSet()) {
     oss << "R";
   }
 
-  oss << " " << repoName << " > " << targetFileName;
+  size_t repoNameLen = 0;
-  int result = std::system(oss.str().c_str());
+  const char *repoName = parser.getRepoName(repoNameLen);
-  if (result != 0) {
+
-    utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile");
+  oss << " " << repoName << " > " << LIST_DIR_DUMP_WORK_FILE;
-    actionHelper.finish(false, commandedBy, actionId);
+  sif::info << "Executing " << oss.str() << " for direct dump";
+  if (parser.compressionOptionSet()) {
+    sif::info << " with compression";
   }
-  return returnvalue::OK;
+  sif::info << std::endl;
+  int ret = std::system(oss.str().c_str());
+  if (ret != 0) {
+    utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly");
+    return returnvalue::FAILED;
+  }
+  if (parser.compressionOptionSet()) {
+    std::string compressedName = LIST_DIR_DUMP_WORK_FILE + std::string(".gz");
+    oss.str("");
+    oss << "gzip " << LIST_DIR_DUMP_WORK_FILE;
+    ret = std::system(oss.str().c_str());
+    if (ret != 0) {
+      utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly");
+      return returnvalue::FAILED;
+    }
+    oss.str("");
+    // Overwrite the work file with the compressed archive.
+    oss << "mv " << compressedName << " " << LIST_DIR_DUMP_WORK_FILE;
+    ret = std::system(oss.str().c_str());
+    if (ret != 0) {
+      utility::handleSystemError(result, "CoreController::actionListDirectoryDumpDirectly");
+      return returnvalue::FAILED;
+    }
+  }
+  std::array<uint8_t, 1024> dirListingBuf{};
+  dirListingBuf[8] = parser.compressionOptionSet();
+  // First four bytes reserved for segment index. One byte for compression option information
+  std::strcpy(reinterpret_cast<char *>(dirListingBuf.data() + 2 * sizeof(uint32_t) + 1), repoName);
+  std::ifstream ifile(LIST_DIR_DUMP_WORK_FILE, std::ios::binary);
+  if (ifile.bad()) {
+    return returnvalue::FAILED;
+  }
+  std::error_code e;
+  size_t totalFileSize = std::filesystem::file_size(LIST_DIR_DUMP_WORK_FILE, e);
+  uint32_t segmentIdx = 0;
+  size_t dumpedBytes = 0;
+  size_t nextDumpLen = 0;
+  size_t dummy = 0;
+  size_t maxDumpLen = dirListingBuf.size() - 2 * sizeof(uint32_t) - 1 - repoNameLen - 1;
+  size_t listingDataOffset = 2 * sizeof(uint32_t) + 1 + repoNameLen + 1;
+  uint32_t chunks = totalFileSize / maxDumpLen;
+  if (totalFileSize % maxDumpLen != 0) {
+    chunks++;
+  }
+  SerializeAdapter::serialize(&chunks, dirListingBuf.data() + sizeof(uint32_t), &dummy,
+                              dirListingBuf.size() - sizeof(uint32_t),
+                              SerializeIF::Endianness::NETWORK);
+  while (dumpedBytes < totalFileSize) {
+    ifile.seekg(dumpedBytes, std::ios::beg);
+    nextDumpLen = maxDumpLen;
+    if (totalFileSize - dumpedBytes < maxDumpLen) {
+      nextDumpLen = totalFileSize - dumpedBytes;
+    }
+    SerializeAdapter::serialize(&segmentIdx, dirListingBuf.data(), &dummy, dirListingBuf.size(),
+                                SerializeIF::Endianness::NETWORK);
+    ifile.read(reinterpret_cast<char *>(dirListingBuf.data() + listingDataOffset), nextDumpLen);
+    result = actionHelper.reportData(commandedBy, actionId, dirListingBuf.data(),
+                                     listingDataOffset + nextDumpLen);
+    if (result != returnvalue::OK) {
+      // Remove work file when we are done
+      std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e);
+      return result;
+    }
+    segmentIdx++;
+    dumpedBytes += nextDumpLen;
+  }
+  // Remove work file when we are done
+  std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e);
+  return EXECUTION_FINISHED;
+}
+
+ReturnValue_t CoreController::actionListDirectoryIntoFile(ActionId_t actionId,
+                                                          MessageQueueId_t commandedBy,
+                                                          const uint8_t *data, size_t size) {
+  core::ListDirectoryIntoFile parser(data, size);
+  ReturnValue_t result = parser.parse();
+  if (result != returnvalue::OK) {
+    return result;
+  }
+
+  std::ostringstream oss("ls -l", std::ostringstream::ate);
+  if (parser.aFlagSet()) {
+    oss << "a";
+  }
+  if (parser.rFlagSet()) {
+    oss << "R";
+  }
+
+  size_t repoNameLen = 0;
+  const char *repoName = parser.getRepoName(repoNameLen);
+  size_t targetFileNameLen = 0;
+  const char *targetFileName = parser.getTargetName(targetFileNameLen);
+  oss << " " << repoName << " > " << targetFileName;
+  sif::info << "Executing list directory request, command: " << oss.str() << std::endl;
+  int ret = std::system(oss.str().c_str());
+  if (ret != 0) {
+    utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile");
+    return returnvalue::FAILED;
+  }
+
+  // Compression will add a .gz ending. I don't have any issue with this, it makes it explicit
+  // that this is a compressed file.
+  if (parser.compressionOptionSet()) {
+    oss.str("");
+    oss << "gzip " << targetFileName;
+    sif::info << "Compressing directory listing: " << oss.str() << std::endl;
+    ret = std::system(oss.str().c_str());
+    if (ret != 0) {
+      utility::handleSystemError(result, "CoreController::actionListDirectoryIntoFile");
+      return returnvalue::FAILED;
+    }
+  }
+  return EXECUTION_FINISHED;
 }
 
 ReturnValue_t CoreController::initBootCopyFile() {
@@ -2049,6 +2212,8 @@ ReturnValue_t CoreController::executeSwUpdate(SwUpdateSources sourceDir, const u
   if (not exists(archivePath, e)) {
     return HasFileSystemIF::FILE_DOES_NOT_EXIST;
   }
+  // TODO: Decompressing without limiting memory usage with xz is actually a bit risky..
+  // But has not been an issue so far.
   ostringstream cmd("tar -xJf", ios::app);
   cmd << " " << archivePath << " -C " << prefixPath;
   int result = system(cmd.str().c_str());

bsp_q7s/core/CoreController.h

@@ -66,6 +66,7 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe
   static constexpr char CHIP_0_COPY_1_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi0-gold-rootfs";
   static constexpr char CHIP_1_COPY_0_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi1-nom-rootfs";
   static constexpr char CHIP_1_COPY_1_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi1-gold-rootfs";
+  static constexpr char LIST_DIR_DUMP_WORK_FILE[] = "/tmp/dir_listing.tmp";
 
   static constexpr dur_millis_t INIT_SD_CARD_CHECK_TIMEOUT = 5000;
   static constexpr dur_millis_t DEFAULT_SD_CARD_CHECK_TIMEOUT = 60000;
@@ -250,6 +251,12 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe
 
   ReturnValue_t actionListDirectoryIntoFile(ActionId_t actionId, MessageQueueId_t commandedBy,
                                             const uint8_t* data, size_t size);
+  ReturnValue_t actionListDirectoryDumpDirectly(ActionId_t actionId, MessageQueueId_t commandedBy,
+                                                const uint8_t* data, size_t size);
+
+  ReturnValue_t actionListDirectoryCommonCommandCreator(const uint8_t* data, size_t size,
+                                                        std::ostringstream& oss);
+
   ReturnValue_t actionXscReboot(const uint8_t* data, size_t size);
   ReturnValue_t actionReboot(const uint8_t* data, size_t size);
 

mission/sysDefs.h

@@ -32,7 +32,6 @@ static constexpr char VERSION_FILE_NAME[] = "version.txt";
 static constexpr char REBOOT_FILE_NAME[] = "reboot.txt";
 static constexpr char TIME_FILE_NAME[] = "time_backup.txt";
 
-static constexpr ActionId_t LIST_DIRECTORY_INTO_FILE = 0;
 static constexpr ActionId_t ANNOUNCE_VERSION = 1;
 static constexpr ActionId_t ANNOUNCE_CURRENT_IMAGE = 2;
 static constexpr ActionId_t ANNOUNCE_BOOT_COUNTS = 3;
@@ -59,6 +58,13 @@ static constexpr ActionId_t EXECUTE_SHELL_CMD_BLOCKING = 40;
 static constexpr ActionId_t EXECUTE_SHELL_CMD_NON_BLOCKING = 41;
 static constexpr ActionId_t SYSTEMCTL_CMD_EXECUTOR = 42;
 
+static constexpr ActionId_t LIST_DIRECTORY_INTO_FILE = 50;
+static constexpr ActionId_t LIST_DIRECTORY_DUMP_DIRECTLY = 51;
+static constexpr ActionId_t CP_HELPER = 52;
+static constexpr ActionId_t MV_HELPER = 53;
+static constexpr ActionId_t RM_HELPER = 54;
+static constexpr ActionId_t MKDIR_HELPER = 55;
+
 static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::CORE;
 
 static constexpr Event ALLOC_FAILURE = event::makeEvent(SUBSYSTEM_ID, 0, severity::MEDIUM);
@@ -96,6 +102,198 @@ static constexpr Event I2C_REBOOT = event::makeEvent(SUBSYSTEM_ID, 11, severity:
 //! [EXPORT] : [COMMENT] PDEC recovery through reset was not possible, performing full reboot.
 static constexpr Event PDEC_REBOOT = event::makeEvent(SUBSYSTEM_ID, 12, severity::HIGH);
 
+class ListDirectoryCmdBase {
+ public:  // TODO: Packet definition for clean deserialization
+  // 3 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with
+  // null termination
+  static constexpr size_t MIN_DATA_LEN = 8;
+
+  ListDirectoryCmdBase(const uint8_t* data, size_t maxSize) : data(data), maxSize(maxSize) {}
+  virtual ~ListDirectoryCmdBase() = default;
+
+  virtual ReturnValue_t parse() {
+    if (maxSize < MIN_DATA_LEN) {
+      return SerializeIF::STREAM_TOO_SHORT;
+    }
+    aFlag = data[0];
+    rFlag = data[1];
+    compressOption = data[2];
+
+    repoNameLen = strnlen(reinterpret_cast<const char*>(data + 3), maxSize - 3);
+    // Last byte MUST be null terminated!
+    if (repoNameLen >= maxSize - 3) {
+      return HasActionsIF::INVALID_PARAMETERS;
+    }
+    repoName = reinterpret_cast<const char*>(data + 3);
+    return returnvalue::OK;
+  }
+
+  bool aFlagSet() const { return this->aFlag; }
+  bool rFlagSet() const { return this->rFlag; }
+
+  bool compressionOptionSet() const { return this->compressOption; }
+
+  const char* getRepoName(size_t& repoNameLen) const {
+    repoNameLen = this->repoNameLen;
+    return this->repoName;
+  }
+
+  size_t getBaseSize() {
+    // Include NULL termination
+    if (repoName != nullptr) {
+      return 3 + repoNameLen + 1;
+    }
+    return 0;
+  }
+
+ protected:
+  const uint8_t* data;
+  size_t maxSize;
+
+  bool aFlag = false;
+  bool rFlag = false;
+  bool compressOption = false;
+  const char* repoName = nullptr;
+  size_t repoNameLen = 0;
+};
+
+class ListDirectoryIntoFile : public ListDirectoryCmdBase {
+ public:
+  // TODO: Packet definition for clean deserialization
+  // 3 bytes for a and R flag, at least 5 bytes for minimum valid path /tmp with
+  // null termination, at least 7 bytes for minimum target file name /tmp/a with
+  // null termination.
+  static constexpr size_t MIN_DATA_LEN = 15;
+
+  ListDirectoryIntoFile(const uint8_t* data, size_t maxSize)
+      : ListDirectoryCmdBase(data, maxSize) {}
+  ReturnValue_t parse() override {
+    if (maxSize < MIN_DATA_LEN) {
+      return SerializeIF::STREAM_TOO_SHORT;
+    }
+    ReturnValue_t result = ListDirectoryCmdBase::parse();
+    if (result != returnvalue::OK) {
+      return result;
+    }
+
+    targetNameLen =
+        strnlen(reinterpret_cast<const char*>(data + getBaseSize()), maxSize - getBaseSize());
+    if (targetNameLen >= maxSize - getBaseSize()) {
+      // Again: String MUST be null terminated.
+      return HasActionsIF::INVALID_PARAMETERS;
+    }
+    targetName = reinterpret_cast<const char*>(data + getBaseSize());
+    return result;
+  }
+  const char* getTargetName(size_t& targetNameLen) const {
+    targetNameLen = this->targetNameLen;
+    return this->targetName;
+  }
+
+ private:
+  const char* targetName = nullptr;
+  size_t targetNameLen = 0;
+};
+
+struct SourceTargetPair {
+  const char* sourceName = nullptr;
+  size_t sourceNameSize = 0;
+  const char* targetName = nullptr;
+  size_t targetNameSize = 0;
+};
+
+static ReturnValue_t parseDestTargetString(const uint8_t* data, size_t maxLen,
+                                           SourceTargetPair& destTgt) {
+  if (maxLen < 4) {
+    return SerializeIF::STREAM_TOO_SHORT;
+  }
+  destTgt.sourceNameSize = strnlen(reinterpret_cast<const char*>(data), maxLen);
+  if (destTgt.sourceNameSize >= maxLen) {
+    return HasActionsIF::INVALID_PARAMETERS;
+  }
+  destTgt.sourceName = reinterpret_cast<const char*>(data);
+  size_t remainingLen = maxLen - destTgt.sourceNameSize - 1;
+  if (remainingLen == 0) {
+    return HasActionsIF::INVALID_PARAMETERS;
+  }
+  destTgt.targetNameSize =
+      strnlen(reinterpret_cast<const char*>(data + destTgt.sourceNameSize + 1), remainingLen);
+  if (destTgt.targetNameSize >= remainingLen) {
+    return HasActionsIF::INVALID_PARAMETERS;
+  }
+  destTgt.targetName = reinterpret_cast<const char*>(data + destTgt.sourceNameSize + 1);
+  return returnvalue::OK;
+}
+
+class CpHelperParser {
+ public:
+  CpHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {}
+
+  ReturnValue_t parse() {
+    if (maxLen < 1) {
+      return SerializeIF::STREAM_TOO_SHORT;
+    }
+    recursiveOpt = data[0];
+    return parseDestTargetString(data + 1, maxLen - 1, destTgt);
+  }
+  const SourceTargetPair& destTgtPair() const { return destTgt; }
+  bool isRecursiveOptSet() const { return recursiveOpt; }
+
+ private:
+  const uint8_t* data;
+  size_t maxLen;
+  bool recursiveOpt = false;
+  SourceTargetPair destTgt;
+};
+
+class MvHelperParser {
+ public:
+  MvHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {}
+
+  ReturnValue_t parse() { return parseDestTargetString(data, maxLen, destTgt); }
+  const SourceTargetPair& destTgtPair() const { return destTgt; }
+
+ private:
+  const uint8_t* data;
+  size_t maxLen;
+  SourceTargetPair destTgt;
+};
+
+class RmHelperParser {
+ public:
+  RmHelperParser(const uint8_t* data, size_t maxLen) : data(data), maxLen(maxLen) {}
+
+  ReturnValue_t parse() {
+    if (maxLen < 2) {
+      return SerializeIF::STREAM_TOO_SHORT;
+    }
+    recursiveOpt = data[0];
+    forceOpt = data[1];
+    removeTargetSize = strnlen(reinterpret_cast<const char*>(data + 2), maxLen - 2);
+    // Must be null-terminated
+    if (removeTargetSize >= maxLen - 2) {
+      return HasActionsIF::INVALID_PARAMETERS;
+    }
+    removeTarget = reinterpret_cast<const char*>(data + 2);
+    return returnvalue::OK;
+  }
+  bool isRecursiveOptSet() const { return recursiveOpt; }
+  bool isForceOptSet() const { return forceOpt; }
+
+  const char* getRemoveTarget(size_t& removeTargetSize) {
+    removeTargetSize = this->removeTargetSize;
+    return removeTarget;
+  }
+
+ private:
+  const uint8_t* data;
+  size_t maxLen;
+  bool recursiveOpt = false;
+  bool forceOpt = false;
+  const char* removeTarget = nullptr;
+  size_t removeTargetSize = 0;
+};
+
 }  // namespace core
 
 #endif /* MISSION_SYSDEFS_H_ */

tmtc

@@ -1 +1 @@
-Subproject commit d00e4247f66eb2f010f1fe53ee7f59b7fb992481
+Subproject commit d623e83be8536f8887f1da00a1f0c4be191db1e8