From aa43912279a254f4e347bf14bd1a53cab2a2c3a3 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 8 Apr 2023 17:13:42 +0200 Subject: [PATCH 1/5] everything except SDC switch looking good.. --- CHANGELOG.md | 7 ++++ bsp_q7s/core/CoreController.cpp | 57 +++++++++++++++++++++++++++++++-- bsp_q7s/core/CoreController.h | 8 +++++ bsp_q7s/fs/SdCardManager.cpp | 5 +++ bsp_q7s/fs/SdCardManager.h | 1 + 5 files changed, 76 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf88bce3..0eb5fe6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,13 @@ will consitute of a breaking change warranting a new major release: - Bugfixes and improvements for SDC state machine. Internal state was not updated correctly due to a regression, so commanding the SDC state machine externally lead to confusing results. +## Changed + +- Added additional logic for SDC state machine so that the SD cards are marked unusable when + the active SD card is switched or there is a transition from hot redundant to cold redundant mode. + This gives other tasks some time to register the SD cards being unusable, and therefore prevents + a way for them to perform any re-initialization tasks necessary after SD card switches. + # [v1.44.0] 2023-04-07 - eive-tmtc: v2.22.0 diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 08135b7b..ec7213a5 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -440,6 +440,19 @@ ReturnValue_t CoreController::sdStateMachine() { } result = sdcMan->getSdCardsStatus(sdInfo.currentState); updateInternalSdInfo(); + auto currentlyActiveSdc = sdcMan->getActiveSdCard(); + // Used/active SD card switches, so mark SD card unusable so other tasks have some time + // registering the unavailable SD card. + if (not currentlyActiveSdc.has_value() or + ((currentlyActiveSdc.value() == sd::SdCard::SLOT_0) and + (sdInfo.active == sd::SdCard::SLOT_1)) or + ((currentlyActiveSdc.value() == sd::SdCard::SLOT_1) and + (sdInfo.active == sd::SdCard::SLOT_0))) { + sdInfo.lockSdCardUsage = true; + } + if (sdInfo.lockSdCardUsage) { + sdcMan->markUnusable(); + } if (sdInfo.active != sd::SdCard::SLOT_0 and sdInfo.active != sd::SdCard::SLOT_1) { sif::warning << "Preferred SD card invalid. Setting to card 0.." << std::endl; sdInfo.active = sd::SdCard::SLOT_0; @@ -451,7 +464,11 @@ ReturnValue_t CoreController::sdStateMachine() { sif::info << "Cold redundant SD card configuration, target SD card: " << static_cast(sdInfo.active) << std::endl; } + SdStates tgtState = SdStates::IDLE; + bool skipCycles = sdInfo.lockSdCardUsage; + // Need to do different things depending on state of SD card which will be active. if (sdInfo.activeState == sd::SdState::MOUNTED) { + // Already mounted, so we can perform handling of the other side. #if OBSW_VERBOSE_LEVEL >= 1 std::string mountString; if (sdInfo.active == sd::SdCard::SLOT_0) { @@ -464,21 +481,50 @@ ReturnValue_t CoreController::sdStateMachine() { #endif sdcMan->setActiveSdCard(sdInfo.active); currMntPrefix = sdcMan->getCurrentMountPrefix(); - sdFsmState = SdStates::DETERMINE_OTHER; + tgtState = SdStates::DETERMINE_OTHER; } else if (sdInfo.activeState == sd::SdState::OFF) { + // It's okay to do the delay after swichting active SD on, no one can use it anyway.. sdCardSetup(sdInfo.active, sd::SdState::ON, sdInfo.activeChar, false); sdInfo.commandPending = true; + // Do not skip cycles here, would mess up the state machine. We skip the cycles after + // the SD card was switched on. + skipCycles = false; } else if (sdInfo.activeState == sd::SdState::ON) { - sdFsmState = SdStates::MOUNT_SELF; + // We can do the delay before mounting where applicable. + tgtState = SdStates::MOUNT_SELF; + } + if (skipCycles) { + sdFsmState = SdStates::SKIP_TWO_CYCLES_IF_SD_LOCKED; + fsmStateAfterDelay = tgtState; + sdInfo.skippedCyclesCount = 0; + } else { + sdFsmState = tgtState; } } else { if (nonBlockingSdcOpChecking(SdStates::MOUNT_SELF, 10, "Setting SDC state")) { sdInfo.activeState = sd::SdState::ON; currentStateSetter(sdInfo.active, sd::SdState::ON); + // Skip the two cycles now. + if (sdInfo.lockSdCardUsage) { + sdFsmState = SdStates::SKIP_TWO_CYCLES_IF_SD_LOCKED; + fsmStateAfterDelay = SdStates::MOUNT_SELF; + sdInfo.skippedCyclesCount = 0; + } } } } + if (sdFsmState == SdStates::SKIP_TWO_CYCLES_IF_SD_LOCKED) { + sif::debug << "skipping cycle.." << std::endl; + sdInfo.skippedCyclesCount++; + // Count to three because this branch will run in the same FSM cycle. + if (sdInfo.skippedCyclesCount == 3) { + sdFsmState = fsmStateAfterDelay; + fsmStateAfterDelay = SdStates::IDLE; + sdInfo.skippedCyclesCount = 0; + } + } + if (sdFsmState == SdStates::MOUNT_SELF) { if (not sdInfo.commandPending) { result = sdCardSetup(sdInfo.active, sd::SdState::MOUNTED, sdInfo.activeChar); @@ -583,6 +629,8 @@ ReturnValue_t CoreController::sdStateMachine() { sif::warning << "CoreController: Updating SD card state file failed" << std::endl; } updateInternalSdInfo(); + // Mark usable again in any case. + sdcMan->markUsable(); sdInfo.commandPending = false; sdFsmState = SdStates::IDLE; sdInfo.cycleCount = 0; @@ -2032,6 +2080,11 @@ bool CoreController::startSdStateMachine(sd::SdCard targetActiveSd, SdCfgMode mo } sdFsmState = SdStates::START; sdInfo.active = targetActiveSd; + // If we are going from 2 SD cards to one, lock SD card usage in any case because 1 SD card is + // going off. + if (sdInfo.cfgMode == SdCfgMode::HOT_REDUNDANT and mode == SdCfgMode::COLD_REDUNDANT) { + sdInfo.lockSdCardUsage = true; + } sdInfo.cfgMode = mode; sdCommandingInfo.actionId = actionId; sdCommandingInfo.commander = commander; diff --git a/bsp_q7s/core/CoreController.h b/bsp_q7s/core/CoreController.h index 8d6e802c..22b1f6ff 100644 --- a/bsp_q7s/core/CoreController.h +++ b/bsp_q7s/core/CoreController.h @@ -130,6 +130,7 @@ class CoreController : public ExtendedControllerBase { NONE, START, UPDATE_SD_INFO_START, + SKIP_TWO_CYCLES_IF_SD_LOCKED, MOUNT_SELF, // Determine operations for other SD card, depending on redundancy configuration DETERMINE_OTHER, @@ -152,6 +153,7 @@ class CoreController : public ExtendedControllerBase { MessageQueueIF* eventQueue = nullptr; SdStates sdFsmState = SdStates::START; + SdStates fsmStateAfterDelay = SdStates::IDLE; enum SdCfgMode { PASSIVE, COLD_REDUNDANT, HOT_REDUNDANT }; struct SdFsmParams { @@ -163,10 +165,16 @@ class CoreController : public ExtendedControllerBase { sd::SdState activeState = sd::SdState::OFF; sd::SdState otherState = sd::SdState::OFF; std::pair mountSwitch = {true, true}; + // This flag denotes that the SD card usage is locked. This is relevant if SD cards go off + // to leave appliation using the SD cards some time to detect the SD card is not usable anymore. + // This is relevant if the active SD card is being switched. The SD card will also be locked + // when going from hot-redundant mode to cold-redundant mode. + bool lockSdCardUsage = false; bool commandPending = true; bool initFinished = false; SdCardManager::SdStatePair currentState; uint16_t cycleCount = 0; + uint16_t skippedCyclesCount = 0; } sdInfo; struct SdCommanding { diff --git a/bsp_q7s/fs/SdCardManager.cpp b/bsp_q7s/fs/SdCardManager.cpp index 0375a71d..14e3e6aa 100644 --- a/bsp_q7s/fs/SdCardManager.cpp +++ b/bsp_q7s/fs/SdCardManager.cpp @@ -581,3 +581,8 @@ void SdCardManager::markUnusable() { MutexGuard mg(defaultLock, LOCK_TYPE, OTHER_TIMEOUT, LOCK_CTX); markedUnusable = true; } + +void SdCardManager::markUsable() { + MutexGuard mg(defaultLock, LOCK_TYPE, OTHER_TIMEOUT, LOCK_CTX); + markedUnusable = false; +} diff --git a/bsp_q7s/fs/SdCardManager.h b/bsp_q7s/fs/SdCardManager.h index 74bf6e4b..7a4a7cbe 100644 --- a/bsp_q7s/fs/SdCardManager.h +++ b/bsp_q7s/fs/SdCardManager.h @@ -205,6 +205,7 @@ class SdCardManager : public SystemObject, public SdCardMountedIF { ReturnValue_t performFsck(sd::SdCard sdcard, bool printOutput, int& linuxError); void markUnusable(); + void markUsable(); private: CommandExecutor cmdExecutor; From 2df556c5bed4ac506b1aef8c8c2928c2877119fb Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 8 Apr 2023 17:46:54 +0200 Subject: [PATCH 2/5] some bugfixes, seems to work now --- bsp_q7s/core/CoreController.cpp | 13 ++++++++++++- mission/tmtc/PersistentTmStore.cpp | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index ec7213a5..9e40b03c 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -489,6 +489,8 @@ ReturnValue_t CoreController::sdStateMachine() { // Do not skip cycles here, would mess up the state machine. We skip the cycles after // the SD card was switched on. skipCycles = false; + // Remain on the current state. + tgtState = sdFsmState; } else if (sdInfo.activeState == sd::SdState::ON) { // We can do the delay before mounting where applicable. tgtState = SdStates::MOUNT_SELF; @@ -515,7 +517,6 @@ ReturnValue_t CoreController::sdStateMachine() { } if (sdFsmState == SdStates::SKIP_TWO_CYCLES_IF_SD_LOCKED) { - sif::debug << "skipping cycle.." << std::endl; sdInfo.skippedCyclesCount++; // Count to three because this branch will run in the same FSM cycle. if (sdInfo.skippedCyclesCount == 3) { @@ -575,6 +576,11 @@ ReturnValue_t CoreController::sdStateMachine() { "Switching off other SD card")) { sdInfo.otherState = sd::SdState::OFF; currentStateSetter(sdInfo.other, sd::SdState::OFF); + } else { + // Continue.. avoid being stuck here.. + sdFsmState = SdStates::SKIP_CYCLE_BEFORE_INFO_UPDATE; + sdInfo.otherState = sd::SdState::OFF; + currentStateSetter(sdInfo.other, sd::SdState::OFF); } } } else if (sdInfo.cfgMode == SdCfgMode::HOT_REDUNDANT) { @@ -586,6 +592,11 @@ ReturnValue_t CoreController::sdStateMachine() { "Switching on other SD card")) { sdInfo.otherState = sd::SdState::ON; currentStateSetter(sdInfo.other, sd::SdState::ON); + } else { + // Contnue.. avoid being stuck here. + sdFsmState = SdStates::MOUNT_UNMOUNT_OTHER; + sdInfo.otherState = sd::SdState::ON; + currentStateSetter(sdInfo.other, sd::SdState::ON); } } } diff --git a/mission/tmtc/PersistentTmStore.cpp b/mission/tmtc/PersistentTmStore.cpp index bace6025..7c3738fc 100644 --- a/mission/tmtc/PersistentTmStore.cpp +++ b/mission/tmtc/PersistentTmStore.cpp @@ -338,6 +338,8 @@ ReturnValue_t PersistentTmStore::createMostRecentFile(std::optional suf ReturnValue_t PersistentTmStore::initializeTmStore() { Clock::getClock_timeval(¤tTv); updateBaseDir(); + // Reset active file, base directory might have changed. + activeFile = std::nullopt; return assignAndOrCreateMostRecentFile(); } From cc39acd4369e60e5a3161564461b0c98676bfbdd Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 8 Apr 2023 17:47:56 +0200 Subject: [PATCH 3/5] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0eb5fe6d..a1288786 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ will consitute of a breaking change warranting a new major release: - Bugfixes and improvements for SDC state machine. Internal state was not updated correctly due to a regression, so commanding the SDC state machine externally lead to confusing results. +- Fixed a bug in persistent TM store, where the active file was not reset of SD card switches. + SD card switch from 0 to 1 and vice-versa works without errors from persistent TM stores now. ## Changed From 4afddad503cf525713fd95d5bdd80afd6c423bb8 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Sat, 8 Apr 2023 17:48:45 +0200 Subject: [PATCH 4/5] typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1288786..e3995873 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ will consitute of a breaking change warranting a new major release: - Added additional logic for SDC state machine so that the SD cards are marked unusable when the active SD card is switched or there is a transition from hot redundant to cold redundant mode. - This gives other tasks some time to register the SD cards being unusable, and therefore prevents + This gives other tasks some time to register the SD cards being unusable, and therefore provides a way for them to perform any re-initialization tasks necessary after SD card switches. # [v1.44.0] 2023-04-07 From e01fe19d53936f15be37b5d194142019ae662a0c Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Tue, 11 Apr 2023 15:34:46 +0200 Subject: [PATCH 5/5] EM bugfix --- bsp_q7s/em/emObjectFactory.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/bsp_q7s/em/emObjectFactory.cpp b/bsp_q7s/em/emObjectFactory.cpp index e43d34f2..2d06e697 100644 --- a/bsp_q7s/em/emObjectFactory.cpp +++ b/bsp_q7s/em/emObjectFactory.cpp @@ -68,6 +68,7 @@ void ObjectFactory::produce(void* args) { #else createPcduComponents(gpioComIF, &pwrSwitcher, enableHkSets); #endif + satsystem::EIVE_SYSTEM.setI2cRecoveryParams(pwrSwitcher); dummy::createDummies(dummyCfg, *pwrSwitcher, gpioComIF);