Merge pull request 'Core Controller Dir Listing FSM' (#669) from core-ctrl-dir-listing-fsm into v3.0.0-dev

Reviewed-on: #669

CHANGELOG.md

@@ -55,6 +55,10 @@ will consitute of a breaking change warranting a new major release:
   only be used to cancel a transfer.
 - Handling of multiple RWs in the ACS Controller is improved and can be changed by parameter
   commands.
+- The Directory Listing direct dumper now has a state machine to stagger the directory listing dump.
+  This is required because very large dumps will overload the queue capacities in the framework.
+- The PUS Service 8 now has larger queue sizes to handle more action replies. The PUS Service 1
+  also has a larger queue size to handle a lot of step replies now.
 - Moved PDU `vcc` and `vbat` variable from auxiliary dataset to core dataset.
 - Tweak TCP/IP configuration: Only the TCP server will be included on the EM. For the FM, no
   TCP/IP servers will be included by default.

bsp_q7s/core/CoreController.cpp

@@ -112,6 +112,9 @@ void CoreController::performControlOperation() {
   sdStateMachine();
   performMountedSdCardOperations();
   readHkData();
+  if (dumpContext.active) {
+    dirListingDumpHandler();
+  }
 
   if (shellCmdIsExecuting) {
     bool replyReceived = false;
@@ -1038,7 +1041,6 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI
       return returnvalue::FAILED;
     }
   }
-  std::array<uint8_t, 1024> dirListingBuf{};
   dirListingBuf[8] = parser.compressionOptionSet();
   // First four bytes reserved for segment index. One byte for compression option information
   std::strcpy(reinterpret_cast<char *>(dirListingBuf.data() + 2 * sizeof(uint32_t) + 1), repoName);
@@ -1047,38 +1049,47 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI
     return returnvalue::FAILED;
   }
   std::error_code e;
-  size_t totalFileSize = std::filesystem::file_size(LIST_DIR_DUMP_WORK_FILE, e);
+  dumpContext.totalFileSize = std::filesystem::file_size(LIST_DIR_DUMP_WORK_FILE, e);
-  uint32_t segmentIdx = 0;
+  dumpContext.segmentIdx = 0;
-  size_t dumpedBytes = 0;
+  dumpContext.dumpedBytes = 0;
   size_t nextDumpLen = 0;
   size_t dummy = 0;
-  size_t maxDumpLen = dirListingBuf.size() - 2 * sizeof(uint32_t) - 1 - repoNameLen - 1;
+  dumpContext.maxDumpLen = dirListingBuf.size() - 2 * sizeof(uint32_t) - 1 - repoNameLen - 1;
-  size_t listingDataOffset = 2 * sizeof(uint32_t) + 1 + repoNameLen + 1;
+  dumpContext.listingDataOffset = 2 * sizeof(uint32_t) + 1 + repoNameLen + 1;
-  uint32_t chunks = totalFileSize / maxDumpLen;
+  uint32_t chunks = dumpContext.totalFileSize / dumpContext.maxDumpLen;
-  if (totalFileSize % maxDumpLen != 0) {
+  if (dumpContext.totalFileSize % dumpContext.maxDumpLen != 0) {
     chunks++;
   }
   SerializeAdapter::serialize(&chunks, dirListingBuf.data() + sizeof(uint32_t), &dummy,
                               dirListingBuf.size() - sizeof(uint32_t),
                               SerializeIF::Endianness::NETWORK);
-  while (dumpedBytes < totalFileSize) {
+  while (dumpContext.dumpedBytes < dumpContext.totalFileSize) {
-    ifile.seekg(dumpedBytes, std::ios::beg);
+    ifile.seekg(dumpContext.dumpedBytes, std::ios::beg);
-    nextDumpLen = maxDumpLen;
+    nextDumpLen = dumpContext.maxDumpLen;
-    if (totalFileSize - dumpedBytes < maxDumpLen) {
+    if (dumpContext.totalFileSize - dumpContext.dumpedBytes < dumpContext.maxDumpLen) {
-      nextDumpLen = totalFileSize - dumpedBytes;
+      nextDumpLen = dumpContext.totalFileSize - dumpContext.dumpedBytes;
     }
-    SerializeAdapter::serialize(&segmentIdx, dirListingBuf.data(), &dummy, dirListingBuf.size(),
+    SerializeAdapter::serialize(&dumpContext.segmentIdx, dirListingBuf.data(), &dummy,
-                                SerializeIF::Endianness::NETWORK);
+                                dirListingBuf.size(), SerializeIF::Endianness::NETWORK);
-    ifile.read(reinterpret_cast<char *>(dirListingBuf.data() + listingDataOffset), nextDumpLen);
+    ifile.read(reinterpret_cast<char *>(dirListingBuf.data() + dumpContext.listingDataOffset),
+               nextDumpLen);
     result = actionHelper.reportData(commandedBy, actionId, dirListingBuf.data(),
-                                     listingDataOffset + nextDumpLen);
+                                     dumpContext.listingDataOffset + nextDumpLen);
     if (result != returnvalue::OK) {
       // Remove work file when we are done
       std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e);
       return result;
     }
-    segmentIdx++;
+    dumpContext.segmentIdx++;
-    dumpedBytes += nextDumpLen;
+    dumpContext.dumpedBytes += nextDumpLen;
+    // Dump takes multiple task cycles, so cache the dump state and continue dump the next cycles.
+    if (dumpContext.segmentIdx == 10) {
+      dumpContext.active = true;
+      dumpContext.firstDump = true;
+      dumpContext.commander = commandedBy;
+      dumpContext.actionId = actionId;
+      return returnvalue::OK;
+    }
   }
   // Remove work file when we are done
   std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e);
@@ -2346,6 +2357,54 @@ MessageQueueId_t CoreController::getCommandQueue() const {
   return ExtendedControllerBase::getCommandQueue();
 }
 
+void CoreController::dirListingDumpHandler() {
+  if (dumpContext.firstDump) {
+    dumpContext.firstDump = false;
+    return;
+  }
+  size_t nextDumpLen = 0;
+  size_t dummy = 0;
+  ReturnValue_t result;
+  std::error_code e;
+  std::ifstream ifile(LIST_DIR_DUMP_WORK_FILE, std::ios::binary);
+  if (ifile.bad()) {
+    return;
+  }
+  while (dumpContext.dumpedBytes < dumpContext.totalFileSize) {
+    ifile.seekg(dumpContext.dumpedBytes, std::ios::beg);
+    nextDumpLen = dumpContext.maxDumpLen;
+    if (dumpContext.totalFileSize - dumpContext.dumpedBytes < dumpContext.maxDumpLen) {
+      nextDumpLen = dumpContext.totalFileSize - dumpContext.dumpedBytes;
+    }
+    SerializeAdapter::serialize(&dumpContext.segmentIdx, dirListingBuf.data(), &dummy,
+                                dirListingBuf.size(), SerializeIF::Endianness::NETWORK);
+    ifile.read(reinterpret_cast<char *>(dirListingBuf.data() + dumpContext.listingDataOffset),
+               nextDumpLen);
+    result =
+        actionHelper.reportData(dumpContext.commander, dumpContext.actionId, dirListingBuf.data(),
+                                dumpContext.listingDataOffset + nextDumpLen);
+    if (result != returnvalue::OK) {
+      // Remove work file when we are done
+      std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e);
+      dumpContext.active = false;
+      actionHelper.finish(false, dumpContext.commander, dumpContext.actionId, result);
+      return;
+    }
+    dumpContext.segmentIdx++;
+    dumpContext.dumpedBytes += nextDumpLen;
+    // Dump takes multiple task cycles, so cache the dump state and continue dump the next cycles.
+    if (dumpContext.segmentIdx == 10) {
+      break;
+    }
+  }
+  if (dumpContext.dumpedBytes >= dumpContext.totalFileSize) {
+    actionHelper.finish(true, dumpContext.commander, dumpContext.actionId, result);
+    dumpContext.active = false;
+    // Remove work file when we are done
+    std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e);
+  }
+}
+
 bool CoreController::isNumber(const std::string &s) {
   return !s.empty() && std::find_if(s.begin(), s.end(),
                                     [](unsigned char c) { return !std::isdigit(c); }) == s.end();

bsp_q7s/core/CoreController.h

@@ -177,6 +177,20 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe
     DeviceCommandId_t actionId;
   } sdCommandingInfo;
 
+  struct DirListingDumpContext {
+    bool active;
+    bool firstDump;
+    size_t dumpedBytes;
+    size_t totalFileSize;
+    size_t listingDataOffset;
+    size_t maxDumpLen;
+    uint32_t segmentIdx;
+    MessageQueueId_t commander = MessageQueueIF::NO_QUEUE;
+    DeviceCommandId_t actionId;
+  };
+  std::array<uint8_t, 1024> dirListingBuf{};
+  DirListingDumpContext dumpContext{};
+
   RebootFile rebootFile = {};
 
   CommandExecutor cmdExecutor;
@@ -274,6 +288,7 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe
   void rewriteRebootFile(RebootFile file);
   void announceBootCounts();
   void readHkData();
+  void dirListingDumpHandler();
   bool isNumber(const std::string& s);
 };
 

common/config/eive/definitions.h

@@ -58,7 +58,9 @@ static constexpr uint32_t CFDP_STORE_QUEUE_SIZE = 300;
 
 static constexpr uint32_t MAX_PUS_FUNNEL_QUEUE_DEPTH = 100;
 static constexpr uint32_t MAX_CFDP_FUNNEL_QUEUE_DEPTH = 80;
+static constexpr uint32_t VERIFICATION_SERVICE_QUEUE_DEPTH = 120;
 static constexpr uint32_t HK_SERVICE_QUEUE_DEPTH = 60;
+static constexpr uint32_t ACTION_SERVICE_QUEUE_DEPTH = 60;
 
 static constexpr uint32_t MAX_STORED_CMDS_UDP = 150;
 static constexpr uint32_t MAX_STORED_CMDS_TCP = 180;

mission/genericFactory.cpp

@@ -233,7 +233,8 @@ void ObjectFactory::produceGenericObjects(HealthTableIF** healthTable_, PusTmFun
 
   // PUS service stack
   new Service1TelecommandVerification(objects::PUS_SERVICE_1_VERIFICATION, config::EIVE_PUS_APID,
-                                      pus::PUS_SERVICE_1, objects::PUS_TM_FUNNEL, 40);
+                                      pus::PUS_SERVICE_1, objects::PUS_TM_FUNNEL,
+                                      config::VERIFICATION_SERVICE_QUEUE_DEPTH);
   new Service2DeviceAccess(objects::PUS_SERVICE_2_DEVICE_ACCESS, config::EIVE_PUS_APID,
                            pus::PUS_SERVICE_2, 3, 10);
   new Service3Housekeeping(objects::PUS_SERVICE_3_HOUSEKEEPING, config::EIVE_PUS_APID,
@@ -242,7 +243,7 @@ void ObjectFactory::produceGenericObjects(HealthTableIF** healthTable_, PusTmFun
       PsbParams(objects::PUS_SERVICE_5_EVENT_REPORTING, config::EIVE_PUS_APID, pus::PUS_SERVICE_5),
       80, 160);
   new Service8FunctionManagement(objects::PUS_SERVICE_8_FUNCTION_MGMT, config::EIVE_PUS_APID,
-                                 pus::PUS_SERVICE_8, 16, 60);
+                                 pus::PUS_SERVICE_8, config::ACTION_SERVICE_QUEUE_DEPTH, 16, 60);
   new Service9TimeManagement(
       PsbParams(objects::PUS_SERVICE_9_TIME_MGMT, config::EIVE_PUS_APID, pus::PUS_SERVICE_9));