Merge pull request 'Feature and Bugfixes: Better SDC state machine handling' (#584) from feature_sdc_fsm_mark_sd_unusbale into develop

Reviewed-on: #584
Reviewed-by: Marius Eggert <eggertm@irs.uni-stuttgart.de>

CHANGELOG.md

@@ -20,6 +20,15 @@ will consitute of a breaking change warranting a new major release:
 
 - Bugfixes and improvements for SDC state machine. Internal state was not updated correctly due
   to a regression, so commanding the SDC state machine externally lead to confusing results.
+- Fixed a bug in persistent TM store, where the active file was not reset of SD card switches.
+  SD card switch from 0 to 1 and vice-versa works without errors from persistent TM stores now.
+
+## Changed
+
+- Added additional logic for SDC state machine so that the SD cards are marked unusable when
+  the active SD card is switched or there is a transition from hot redundant to cold redundant mode.
+  This gives other tasks some time to register the SD cards being unusable, and therefore provides
+  a way for them to perform any re-initialization tasks necessary after SD card switches.
 
 # [v1.44.0] 2023-04-07
 

bsp_q7s/core/CoreController.cpp

@@ -440,6 +440,19 @@ ReturnValue_t CoreController::sdStateMachine() {
       }
       result = sdcMan->getSdCardsStatus(sdInfo.currentState);
       updateInternalSdInfo();
+      auto currentlyActiveSdc = sdcMan->getActiveSdCard();
+      // Used/active SD card switches, so mark SD card unusable so other tasks have some time
+      // registering the unavailable SD card.
+      if (not currentlyActiveSdc.has_value() or
+          ((currentlyActiveSdc.value() == sd::SdCard::SLOT_0) and
+           (sdInfo.active == sd::SdCard::SLOT_1)) or
+          ((currentlyActiveSdc.value() == sd::SdCard::SLOT_1) and
+           (sdInfo.active == sd::SdCard::SLOT_0))) {
+        sdInfo.lockSdCardUsage = true;
+      }
+      if (sdInfo.lockSdCardUsage) {
+        sdcMan->markUnusable();
+      }
       if (sdInfo.active != sd::SdCard::SLOT_0 and sdInfo.active != sd::SdCard::SLOT_1) {
         sif::warning << "Preferred SD card invalid. Setting to card 0.." << std::endl;
         sdInfo.active = sd::SdCard::SLOT_0;
@@ -451,7 +464,11 @@ ReturnValue_t CoreController::sdStateMachine() {
         sif::info << "Cold redundant SD card configuration, target SD card: "
                   << static_cast<int>(sdInfo.active) << std::endl;
       }
+      SdStates tgtState = SdStates::IDLE;
+      bool skipCycles = sdInfo.lockSdCardUsage;
+      // Need to do different things depending on state of SD card which will be active.
       if (sdInfo.activeState == sd::SdState::MOUNTED) {
+        // Already mounted, so we can perform handling of the other side.
 #if OBSW_VERBOSE_LEVEL >= 1
         std::string mountString;
         if (sdInfo.active == sd::SdCard::SLOT_0) {
@@ -464,21 +481,51 @@ ReturnValue_t CoreController::sdStateMachine() {
 #endif
         sdcMan->setActiveSdCard(sdInfo.active);
         currMntPrefix = sdcMan->getCurrentMountPrefix();
-        sdFsmState = SdStates::DETERMINE_OTHER;
+        tgtState = SdStates::DETERMINE_OTHER;
       } else if (sdInfo.activeState == sd::SdState::OFF) {
+        // It's okay to do the delay after swichting active SD on, no one can use it anyway..
         sdCardSetup(sdInfo.active, sd::SdState::ON, sdInfo.activeChar, false);
         sdInfo.commandPending = true;
+        // Do not skip cycles here, would mess up the state machine. We skip the cycles after
+        // the SD card was switched on.
+        skipCycles = false;
+        // Remain on the current state.
+        tgtState = sdFsmState;
       } else if (sdInfo.activeState == sd::SdState::ON) {
-        sdFsmState = SdStates::MOUNT_SELF;
+        // We can do the delay before mounting where applicable.
+        tgtState = SdStates::MOUNT_SELF;
+      }
+      if (skipCycles) {
+        sdFsmState = SdStates::SKIP_TWO_CYCLES_IF_SD_LOCKED;
+        fsmStateAfterDelay = tgtState;
+        sdInfo.skippedCyclesCount = 0;
+      } else {
+        sdFsmState = tgtState;
       }
     } else {
       if (nonBlockingSdcOpChecking(SdStates::MOUNT_SELF, 10, "Setting SDC state")) {
         sdInfo.activeState = sd::SdState::ON;
         currentStateSetter(sdInfo.active, sd::SdState::ON);
+        // Skip the two cycles now.
+        if (sdInfo.lockSdCardUsage) {
+          sdFsmState = SdStates::SKIP_TWO_CYCLES_IF_SD_LOCKED;
+          fsmStateAfterDelay = SdStates::MOUNT_SELF;
+          sdInfo.skippedCyclesCount = 0;
+        }
       }
     }
   }
 
+  if (sdFsmState == SdStates::SKIP_TWO_CYCLES_IF_SD_LOCKED) {
+    sdInfo.skippedCyclesCount++;
+    // Count to three because this branch will run in the same FSM cycle.
+    if (sdInfo.skippedCyclesCount == 3) {
+      sdFsmState = fsmStateAfterDelay;
+      fsmStateAfterDelay = SdStates::IDLE;
+      sdInfo.skippedCyclesCount = 0;
+    }
+  }
+
   if (sdFsmState == SdStates::MOUNT_SELF) {
     if (not sdInfo.commandPending) {
       result = sdCardSetup(sdInfo.active, sd::SdState::MOUNTED, sdInfo.activeChar);
@@ -529,6 +576,11 @@ ReturnValue_t CoreController::sdStateMachine() {
                                      "Switching off other SD card")) {
           sdInfo.otherState = sd::SdState::OFF;
           currentStateSetter(sdInfo.other, sd::SdState::OFF);
+        } else {
+          // Continue.. avoid being stuck here..
+          sdFsmState = SdStates::SKIP_CYCLE_BEFORE_INFO_UPDATE;
+          sdInfo.otherState = sd::SdState::OFF;
+          currentStateSetter(sdInfo.other, sd::SdState::OFF);
         }
       }
     } else if (sdInfo.cfgMode == SdCfgMode::HOT_REDUNDANT) {
@@ -540,6 +592,11 @@ ReturnValue_t CoreController::sdStateMachine() {
                                      "Switching on other SD card")) {
           sdInfo.otherState = sd::SdState::ON;
           currentStateSetter(sdInfo.other, sd::SdState::ON);
+        } else {
+          // Contnue.. avoid being stuck here.
+          sdFsmState = SdStates::MOUNT_UNMOUNT_OTHER;
+          sdInfo.otherState = sd::SdState::ON;
+          currentStateSetter(sdInfo.other, sd::SdState::ON);
         }
       }
     }
@@ -583,6 +640,8 @@ ReturnValue_t CoreController::sdStateMachine() {
       sif::warning << "CoreController: Updating SD card state file failed" << std::endl;
     }
     updateInternalSdInfo();
+    // Mark usable again in any case.
+    sdcMan->markUsable();
     sdInfo.commandPending = false;
     sdFsmState = SdStates::IDLE;
     sdInfo.cycleCount = 0;
@@ -2032,6 +2091,11 @@ bool CoreController::startSdStateMachine(sd::SdCard targetActiveSd, SdCfgMode mo
   }
   sdFsmState = SdStates::START;
   sdInfo.active = targetActiveSd;
+  // If we are going from 2 SD cards to one, lock SD card usage in any case because 1 SD card is
+  // going off.
+  if (sdInfo.cfgMode == SdCfgMode::HOT_REDUNDANT and mode == SdCfgMode::COLD_REDUNDANT) {
+    sdInfo.lockSdCardUsage = true;
+  }
   sdInfo.cfgMode = mode;
   sdCommandingInfo.actionId = actionId;
   sdCommandingInfo.commander = commander;

bsp_q7s/core/CoreController.h

@@ -130,6 +130,7 @@ class CoreController : public ExtendedControllerBase {
     NONE,
     START,
     UPDATE_SD_INFO_START,
+    SKIP_TWO_CYCLES_IF_SD_LOCKED,
     MOUNT_SELF,
     // Determine operations for other SD card, depending on redundancy configuration
     DETERMINE_OTHER,
@@ -152,6 +153,7 @@ class CoreController : public ExtendedControllerBase {
   MessageQueueIF* eventQueue = nullptr;
 
   SdStates sdFsmState = SdStates::START;
+  SdStates fsmStateAfterDelay = SdStates::IDLE;
   enum SdCfgMode { PASSIVE, COLD_REDUNDANT, HOT_REDUNDANT };
 
   struct SdFsmParams {
@@ -163,10 +165,16 @@ class CoreController : public ExtendedControllerBase {
     sd::SdState activeState = sd::SdState::OFF;
     sd::SdState otherState = sd::SdState::OFF;
     std::pair<bool, bool> mountSwitch = {true, true};
+    // This flag denotes that the SD card usage is locked. This is relevant if SD cards go off
+    // to leave appliation using the SD cards some time to detect the SD card is not usable anymore.
+    // This is relevant if the active SD card is being switched. The SD card will also be locked
+    // when going from hot-redundant mode to cold-redundant mode.
+    bool lockSdCardUsage = false;
     bool commandPending = true;
     bool initFinished = false;
     SdCardManager::SdStatePair currentState;
     uint16_t cycleCount = 0;
+    uint16_t skippedCyclesCount = 0;
   } sdInfo;
 
   struct SdCommanding {

bsp_q7s/fs/SdCardManager.cpp

@@ -581,3 +581,8 @@ void SdCardManager::markUnusable() {
   MutexGuard mg(defaultLock, LOCK_TYPE, OTHER_TIMEOUT, LOCK_CTX);
   markedUnusable = true;
 }
+
+void SdCardManager::markUsable() {
+  MutexGuard mg(defaultLock, LOCK_TYPE, OTHER_TIMEOUT, LOCK_CTX);
+  markedUnusable = false;
+}

bsp_q7s/fs/SdCardManager.h

@@ -205,6 +205,7 @@ class SdCardManager : public SystemObject, public SdCardMountedIF {
   ReturnValue_t performFsck(sd::SdCard sdcard, bool printOutput, int& linuxError);
 
   void markUnusable();
+  void markUsable();
 
  private:
   CommandExecutor cmdExecutor;

mission/tmtc/PersistentTmStore.cpp

@@ -338,6 +338,8 @@ ReturnValue_t PersistentTmStore::createMostRecentFile(std::optional<uint8_t> suf
 ReturnValue_t PersistentTmStore::initializeTmStore() {
   Clock::getClock_timeval(&currentTv);
   updateBaseDir();
+  // Reset active file, base directory might have changed.
+  activeFile = std::nullopt;
   return assignAndOrCreateMostRecentFile();
 }