From 9aec8960b980502b5d9a1c75eaf9a6c1f66f2188 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Wed, 7 Jun 2023 13:43:23 +0200 Subject: [PATCH 1/8] hi --- fsfw | 2 +- mission/genericFactory.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fsfw b/fsfw index 5eb9ee8b..9903371a 160000 --- a/fsfw +++ b/fsfw @@ -1 +1 @@ -Subproject commit 5eb9ee8bc1e6f8640cbea46febd11e4b92241881 +Subproject commit 9903371ae9040a8171ead9cda9c8c73177d03d71 diff --git a/mission/genericFactory.cpp b/mission/genericFactory.cpp index 2dd00ee6..4ec4adec 100644 --- a/mission/genericFactory.cpp +++ b/mission/genericFactory.cpp @@ -243,7 +243,7 @@ void ObjectFactory::produceGenericObjects(HealthTableIF** healthTable_, PusTmFun PsbParams(objects::PUS_SERVICE_5_EVENT_REPORTING, config::EIVE_PUS_APID, pus::PUS_SERVICE_5), 80, 160); new Service8FunctionManagement(objects::PUS_SERVICE_8_FUNCTION_MGMT, config::EIVE_PUS_APID, - pus::PUS_SERVICE_8, 16, 60); + pus::PUS_SERVICE_8, 16, 60, 50); new Service9TimeManagement( PsbParams(objects::PUS_SERVICE_9_TIME_MGMT, config::EIVE_PUS_APID, pus::PUS_SERVICE_9)); -- 2.43.0 From 5ca96b2dd359bdd967fd2c82c23e75028b135507 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Wed, 7 Jun 2023 13:49:12 +0200 Subject: [PATCH 2/8] bump fsfw --- fsfw | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fsfw b/fsfw index 9903371a..b442ca09 160000 --- a/fsfw +++ b/fsfw @@ -1 +1 @@ -Subproject commit 9903371ae9040a8171ead9cda9c8c73177d03d71 +Subproject commit b442ca09b91d0cfdc6acd5a30349a25c4051972c -- 2.43.0 From 06b381d965eae30e0a96203bcfe807d96a0d7c1d Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Wed, 7 Jun 2023 14:20:58 +0200 Subject: [PATCH 3/8] dir listing dump state machine --- bsp_q7s/core/CoreController.cpp | 92 ++++++++++++++++++++++------ bsp_q7s/core/CoreController.h | 14 +++++ mission/controller/acs/ActuatorCmd.h | 1 - 3 files changed, 87 insertions(+), 20 deletions(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 5c4c4d62..9b6a22b0 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -112,6 +112,9 @@ void CoreController::performControlOperation() { sdStateMachine(); performMountedSdCardOperations(); readHkData(); + if (dumpContext.active) { + dirListingDumpHandler(); + } if (shellCmdIsExecuting) { bool replyReceived = false; @@ -1038,7 +1041,6 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI return returnvalue::FAILED; } } - std::array dirListingBuf{}; dirListingBuf[8] = parser.compressionOptionSet(); // First four bytes reserved for segment index. One byte for compression option information std::strcpy(reinterpret_cast(dirListingBuf.data() + 2 * sizeof(uint32_t) + 1), repoName); @@ -1047,38 +1049,46 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI return returnvalue::FAILED; } std::error_code e; - size_t totalFileSize = std::filesystem::file_size(LIST_DIR_DUMP_WORK_FILE, e); - uint32_t segmentIdx = 0; - size_t dumpedBytes = 0; + dumpContext.totalFileSize = std::filesystem::file_size(LIST_DIR_DUMP_WORK_FILE, e); + dumpContext.segmentIdx = 0; + dumpContext.dumpedBytes = 0; size_t nextDumpLen = 0; size_t dummy = 0; - size_t maxDumpLen = dirListingBuf.size() - 2 * sizeof(uint32_t) - 1 - repoNameLen - 1; - size_t listingDataOffset = 2 * sizeof(uint32_t) + 1 + repoNameLen + 1; - uint32_t chunks = totalFileSize / maxDumpLen; - if (totalFileSize % maxDumpLen != 0) { + dumpContext.maxDumpLen = dirListingBuf.size() - 2 * sizeof(uint32_t) - 1 - repoNameLen - 1; + dumpContext.listingDataOffset = 2 * sizeof(uint32_t) + 1 + repoNameLen + 1; + uint32_t chunks = dumpContext.totalFileSize / dumpContext.maxDumpLen; + if (dumpContext.totalFileSize % dumpContext.maxDumpLen != 0) { chunks++; } SerializeAdapter::serialize(&chunks, dirListingBuf.data() + sizeof(uint32_t), &dummy, dirListingBuf.size() - sizeof(uint32_t), SerializeIF::Endianness::NETWORK); - while (dumpedBytes < totalFileSize) { - ifile.seekg(dumpedBytes, std::ios::beg); - nextDumpLen = maxDumpLen; - if (totalFileSize - dumpedBytes < maxDumpLen) { - nextDumpLen = totalFileSize - dumpedBytes; + while (dumpContext.dumpedBytes < dumpContext.totalFileSize) { + ifile.seekg(dumpContext.dumpedBytes, std::ios::beg); + nextDumpLen = dumpContext.maxDumpLen; + if (dumpContext.totalFileSize - dumpContext.dumpedBytes < dumpContext.maxDumpLen) { + nextDumpLen = dumpContext.totalFileSize - dumpContext.dumpedBytes; } - SerializeAdapter::serialize(&segmentIdx, dirListingBuf.data(), &dummy, dirListingBuf.size(), - SerializeIF::Endianness::NETWORK); - ifile.read(reinterpret_cast(dirListingBuf.data() + listingDataOffset), nextDumpLen); + SerializeAdapter::serialize(&dumpContext.segmentIdx, dirListingBuf.data(), &dummy, + dirListingBuf.size(), SerializeIF::Endianness::NETWORK); + ifile.read(reinterpret_cast(dirListingBuf.data() + dumpContext.listingDataOffset), + nextDumpLen); result = actionHelper.reportData(commandedBy, actionId, dirListingBuf.data(), - listingDataOffset + nextDumpLen); + dumpContext.listingDataOffset + nextDumpLen); if (result != returnvalue::OK) { // Remove work file when we are done std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); return result; } - segmentIdx++; - dumpedBytes += nextDumpLen; + dumpContext.segmentIdx++; + dumpContext.dumpedBytes += nextDumpLen; + // Dump takes multiple task cycles, so cache the dump state and continue dump the next cycles. + if (dumpContext.segmentIdx == 25) { + dumpContext.active = true; + dumpContext.commander = commandedBy; + dumpContext.actionId = actionId; + return returnvalue::OK; + } } // Remove work file when we are done std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); @@ -2346,6 +2356,50 @@ MessageQueueId_t CoreController::getCommandQueue() const { return ExtendedControllerBase::getCommandQueue(); } +void CoreController::dirListingDumpHandler() { + size_t nextDumpLen = 0; + size_t dummy = 0; + ReturnValue_t result; + std::error_code e; + std::ifstream ifile(LIST_DIR_DUMP_WORK_FILE, std::ios::binary); + if (ifile.bad()) { + return; + } + while (dumpContext.dumpedBytes < dumpContext.totalFileSize) { + ifile.seekg(dumpContext.dumpedBytes, std::ios::beg); + nextDumpLen = dumpContext.maxDumpLen; + if (dumpContext.totalFileSize - dumpContext.dumpedBytes < dumpContext.maxDumpLen) { + nextDumpLen = dumpContext.totalFileSize - dumpContext.dumpedBytes; + } + SerializeAdapter::serialize(&dumpContext.segmentIdx, dirListingBuf.data(), &dummy, + dirListingBuf.size(), SerializeIF::Endianness::NETWORK); + ifile.read(reinterpret_cast(dirListingBuf.data() + dumpContext.listingDataOffset), + nextDumpLen); + result = + actionHelper.reportData(dumpContext.commander, dumpContext.actionId, dirListingBuf.data(), + dumpContext.listingDataOffset + nextDumpLen); + if (result != returnvalue::OK) { + // Remove work file when we are done + std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); + dumpContext.active = false; + actionHelper.finish(false, dumpContext.commander, dumpContext.actionId, result); + return; + } + dumpContext.segmentIdx++; + dumpContext.dumpedBytes += nextDumpLen; + // Dump takes multiple task cycles, so cache the dump state and continue dump the next cycles. + if (dumpContext.segmentIdx == 25) { + break; + } + } + if (dumpContext.dumpedBytes >= dumpContext.totalFileSize) { + actionHelper.finish(true, dumpContext.commander, dumpContext.actionId, result); + dumpContext.active = false; + // Remove work file when we are done + std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); + } +} + bool CoreController::isNumber(const std::string &s) { return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end(); diff --git a/bsp_q7s/core/CoreController.h b/bsp_q7s/core/CoreController.h index d44907e8..a58faf86 100644 --- a/bsp_q7s/core/CoreController.h +++ b/bsp_q7s/core/CoreController.h @@ -177,6 +177,19 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe DeviceCommandId_t actionId; } sdCommandingInfo; + struct DirListingDumpContext { + bool active; + size_t dumpedBytes; + size_t totalFileSize; + size_t listingDataOffset; + size_t maxDumpLen; + uint32_t segmentIdx; + MessageQueueId_t commander = MessageQueueIF::NO_QUEUE; + DeviceCommandId_t actionId; + }; + std::array dirListingBuf{}; + DirListingDumpContext dumpContext{}; + RebootFile rebootFile = {}; CommandExecutor cmdExecutor; @@ -274,6 +287,7 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe void rewriteRebootFile(RebootFile file); void announceBootCounts(); void readHkData(); + void dirListingDumpHandler(); bool isNumber(const std::string& s); }; diff --git a/mission/controller/acs/ActuatorCmd.h b/mission/controller/acs/ActuatorCmd.h index 6a1b3dbb..b14a4a25 100644 --- a/mission/controller/acs/ActuatorCmd.h +++ b/mission/controller/acs/ActuatorCmd.h @@ -3,7 +3,6 @@ #include - class ActuatorCmd { public: ActuatorCmd(); -- 2.43.0 From 26ed774806767952cd3ba058312210a5841e634c Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Wed, 7 Jun 2023 14:25:13 +0200 Subject: [PATCH 4/8] changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17ac5d5d..d686359d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,9 @@ will consitute of a breaking change warranting a new major release: only be used to cancel a transfer. - Handling of multiple RWs in the ACS Controller is improved and can be changed by parameter commands. +- The Directory Listing direct dumper now has a state machine to stagger the directory listing dump. + This is required because very large dumps will overload the queue capacities in the framework. +- The PUS Service 8 now has larger queue sizes to handle more action replies. ## Added -- 2.43.0 From dcf3b1b1fee30d09beb40359af7536fdad8b60d8 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Wed, 7 Jun 2023 15:55:26 +0200 Subject: [PATCH 5/8] added debug statement for testing --- bsp_q7s/core/CoreController.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 9b6a22b0..3c86915f 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -1080,6 +1080,7 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); return result; } + sif::debug << "dumped segment " << dumpContext.segmentIdx << std::endl; dumpContext.segmentIdx++; dumpContext.dumpedBytes += nextDumpLen; // Dump takes multiple task cycles, so cache the dump state and continue dump the next cycles. @@ -2385,6 +2386,7 @@ void CoreController::dirListingDumpHandler() { actionHelper.finish(false, dumpContext.commander, dumpContext.actionId, result); return; } + sif::debug << "dumped segment " << dumpContext.segmentIdx << std::endl; dumpContext.segmentIdx++; dumpContext.dumpedBytes += nextDumpLen; // Dump takes multiple task cycles, so cache the dump state and continue dump the next cycles. -- 2.43.0 From deb154770d0d5b7cdbaa90cf4e71e82d9ebac242 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Wed, 7 Jun 2023 16:21:51 +0200 Subject: [PATCH 6/8] increased service 1 queue depth --- bsp_q7s/core/CoreController.cpp | 9 +++++++-- bsp_q7s/core/CoreController.h | 1 + common/config/eive/definitions.h | 1 + mission/genericFactory.cpp | 4 ++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 3c86915f..3304fce8 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -1084,8 +1084,9 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI dumpContext.segmentIdx++; dumpContext.dumpedBytes += nextDumpLen; // Dump takes multiple task cycles, so cache the dump state and continue dump the next cycles. - if (dumpContext.segmentIdx == 25) { + if (dumpContext.segmentIdx == 10) { dumpContext.active = true; + dumpContext.firstDump = true; dumpContext.commander = commandedBy; dumpContext.actionId = actionId; return returnvalue::OK; @@ -2358,6 +2359,10 @@ MessageQueueId_t CoreController::getCommandQueue() const { } void CoreController::dirListingDumpHandler() { + if (dumpContext.firstDump) { + dumpContext.firstDump = false; + return; + } size_t nextDumpLen = 0; size_t dummy = 0; ReturnValue_t result; @@ -2390,7 +2395,7 @@ void CoreController::dirListingDumpHandler() { dumpContext.segmentIdx++; dumpContext.dumpedBytes += nextDumpLen; // Dump takes multiple task cycles, so cache the dump state and continue dump the next cycles. - if (dumpContext.segmentIdx == 25) { + if (dumpContext.segmentIdx == 10) { break; } } diff --git a/bsp_q7s/core/CoreController.h b/bsp_q7s/core/CoreController.h index a58faf86..05878d6d 100644 --- a/bsp_q7s/core/CoreController.h +++ b/bsp_q7s/core/CoreController.h @@ -179,6 +179,7 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe struct DirListingDumpContext { bool active; + bool firstDump; size_t dumpedBytes; size_t totalFileSize; size_t listingDataOffset; diff --git a/common/config/eive/definitions.h b/common/config/eive/definitions.h index 78ba52e4..bd93586c 100644 --- a/common/config/eive/definitions.h +++ b/common/config/eive/definitions.h @@ -59,6 +59,7 @@ static constexpr uint32_t CFDP_STORE_QUEUE_SIZE = 300; static constexpr uint32_t MAX_PUS_FUNNEL_QUEUE_DEPTH = 100; static constexpr uint32_t MAX_CFDP_FUNNEL_QUEUE_DEPTH = 80; static constexpr uint32_t HK_SERVICE_QUEUE_DEPTH = 60; +static constexpr uint32_t ACTION_SERVICE_QUEUE_DEPTH = 60; static constexpr uint32_t MAX_STORED_CMDS_UDP = 150; static constexpr uint32_t MAX_STORED_CMDS_TCP = 180; diff --git a/mission/genericFactory.cpp b/mission/genericFactory.cpp index 4ec4adec..36c75f0e 100644 --- a/mission/genericFactory.cpp +++ b/mission/genericFactory.cpp @@ -234,7 +234,7 @@ void ObjectFactory::produceGenericObjects(HealthTableIF** healthTable_, PusTmFun // PUS service stack new Service1TelecommandVerification(objects::PUS_SERVICE_1_VERIFICATION, config::EIVE_PUS_APID, - pus::PUS_SERVICE_1, objects::PUS_TM_FUNNEL, 40); + pus::PUS_SERVICE_1, objects::PUS_TM_FUNNEL, 120); new Service2DeviceAccess(objects::PUS_SERVICE_2_DEVICE_ACCESS, config::EIVE_PUS_APID, pus::PUS_SERVICE_2, 3, 10); new Service3Housekeeping(objects::PUS_SERVICE_3_HOUSEKEEPING, config::EIVE_PUS_APID, @@ -243,7 +243,7 @@ void ObjectFactory::produceGenericObjects(HealthTableIF** healthTable_, PusTmFun PsbParams(objects::PUS_SERVICE_5_EVENT_REPORTING, config::EIVE_PUS_APID, pus::PUS_SERVICE_5), 80, 160); new Service8FunctionManagement(objects::PUS_SERVICE_8_FUNCTION_MGMT, config::EIVE_PUS_APID, - pus::PUS_SERVICE_8, 16, 60, 50); + pus::PUS_SERVICE_8, config::ACTION_SERVICE_QUEUE_DEPTH, 16, 60); new Service9TimeManagement( PsbParams(objects::PUS_SERVICE_9_TIME_MGMT, config::EIVE_PUS_APID, pus::PUS_SERVICE_9)); -- 2.43.0 From cd6b7d90be4ece727e21d8792ff6407f8dcec862 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Fri, 9 Jun 2023 14:25:59 +0200 Subject: [PATCH 7/8] add one more cfg constant --- common/config/eive/definitions.h | 1 + mission/genericFactory.cpp | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/common/config/eive/definitions.h b/common/config/eive/definitions.h index bd93586c..2083962a 100644 --- a/common/config/eive/definitions.h +++ b/common/config/eive/definitions.h @@ -58,6 +58,7 @@ static constexpr uint32_t CFDP_STORE_QUEUE_SIZE = 300; static constexpr uint32_t MAX_PUS_FUNNEL_QUEUE_DEPTH = 100; static constexpr uint32_t MAX_CFDP_FUNNEL_QUEUE_DEPTH = 80; +static constexpr uint32_t VERIFICATION_SERVICE_QUEUE_DEPTH = 120; static constexpr uint32_t HK_SERVICE_QUEUE_DEPTH = 60; static constexpr uint32_t ACTION_SERVICE_QUEUE_DEPTH = 60; diff --git a/mission/genericFactory.cpp b/mission/genericFactory.cpp index 36c75f0e..466219b3 100644 --- a/mission/genericFactory.cpp +++ b/mission/genericFactory.cpp @@ -234,7 +234,8 @@ void ObjectFactory::produceGenericObjects(HealthTableIF** healthTable_, PusTmFun // PUS service stack new Service1TelecommandVerification(objects::PUS_SERVICE_1_VERIFICATION, config::EIVE_PUS_APID, - pus::PUS_SERVICE_1, objects::PUS_TM_FUNNEL, 120); + pus::PUS_SERVICE_1, objects::PUS_TM_FUNNEL, + config::VERIFICATION_SERVICE_QUEUE_DEPTH); new Service2DeviceAccess(objects::PUS_SERVICE_2_DEVICE_ACCESS, config::EIVE_PUS_APID, pus::PUS_SERVICE_2, 3, 10); new Service3Housekeeping(objects::PUS_SERVICE_3_HOUSEKEEPING, config::EIVE_PUS_APID, -- 2.43.0 From 171d4976c3956a9f4609104086c82db68b33985b Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Fri, 9 Jun 2023 14:40:47 +0200 Subject: [PATCH 8/8] removing printout --- bsp_q7s/core/CoreController.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 3304fce8..76460aa5 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -1080,7 +1080,6 @@ ReturnValue_t CoreController::actionListDirectoryDumpDirectly(ActionId_t actionI std::filesystem::remove(LIST_DIR_DUMP_WORK_FILE, e); return result; } - sif::debug << "dumped segment " << dumpContext.segmentIdx << std::endl; dumpContext.segmentIdx++; dumpContext.dumpedBytes += nextDumpLen; // Dump takes multiple task cycles, so cache the dump state and continue dump the next cycles. @@ -2391,7 +2390,6 @@ void CoreController::dirListingDumpHandler() { actionHelper.finish(false, dumpContext.commander, dumpContext.actionId, result); return; } - sif::debug << "dumped segment " << dumpContext.segmentIdx << std::endl; dumpContext.segmentIdx++; dumpContext.dumpedBytes += nextDumpLen; // Dump takes multiple task cycles, so cache the dump state and continue dump the next cycles. -- 2.43.0