Merge pull request 'Feature and Bugfixes: Better SDC state machine handling' (#584) from feature_sdc_fsm_mark_sd_unusbale into develop
All checks were successful
EIVE/eive-obsw/pipeline/head This commit looks good

Reviewed-on: #584
Reviewed-by: Marius Eggert <eggertm@irs.uni-stuttgart.de>
This commit is contained in:
Marius Eggert 2023-04-11 09:52:16 +02:00
commit abef9da9f9
6 changed files with 91 additions and 2 deletions

View File

@ -20,6 +20,15 @@ will consitute of a breaking change warranting a new major release:
- Bugfixes and improvements for SDC state machine. Internal state was not updated correctly due
to a regression, so commanding the SDC state machine externally lead to confusing results.
- Fixed a bug in persistent TM store, where the active file was not reset of SD card switches.
SD card switch from 0 to 1 and vice-versa works without errors from persistent TM stores now.
## Changed
- Added additional logic for SDC state machine so that the SD cards are marked unusable when
the active SD card is switched or there is a transition from hot redundant to cold redundant mode.
This gives other tasks some time to register the SD cards being unusable, and therefore provides
a way for them to perform any re-initialization tasks necessary after SD card switches.
# [v1.44.0] 2023-04-07

View File

@ -440,6 +440,19 @@ ReturnValue_t CoreController::sdStateMachine() {
}
result = sdcMan->getSdCardsStatus(sdInfo.currentState);
updateInternalSdInfo();
auto currentlyActiveSdc = sdcMan->getActiveSdCard();
// Used/active SD card switches, so mark SD card unusable so other tasks have some time
// registering the unavailable SD card.
if (not currentlyActiveSdc.has_value() or
((currentlyActiveSdc.value() == sd::SdCard::SLOT_0) and
(sdInfo.active == sd::SdCard::SLOT_1)) or
((currentlyActiveSdc.value() == sd::SdCard::SLOT_1) and
(sdInfo.active == sd::SdCard::SLOT_0))) {
sdInfo.lockSdCardUsage = true;
}
if (sdInfo.lockSdCardUsage) {
sdcMan->markUnusable();
}
if (sdInfo.active != sd::SdCard::SLOT_0 and sdInfo.active != sd::SdCard::SLOT_1) {
sif::warning << "Preferred SD card invalid. Setting to card 0.." << std::endl;
sdInfo.active = sd::SdCard::SLOT_0;
@ -451,7 +464,11 @@ ReturnValue_t CoreController::sdStateMachine() {
sif::info << "Cold redundant SD card configuration, target SD card: "
<< static_cast<int>(sdInfo.active) << std::endl;
}
SdStates tgtState = SdStates::IDLE;
bool skipCycles = sdInfo.lockSdCardUsage;
// Need to do different things depending on state of SD card which will be active.
if (sdInfo.activeState == sd::SdState::MOUNTED) {
// Already mounted, so we can perform handling of the other side.
#if OBSW_VERBOSE_LEVEL >= 1
std::string mountString;
if (sdInfo.active == sd::SdCard::SLOT_0) {
@ -464,21 +481,51 @@ ReturnValue_t CoreController::sdStateMachine() {
#endif
sdcMan->setActiveSdCard(sdInfo.active);
currMntPrefix = sdcMan->getCurrentMountPrefix();
sdFsmState = SdStates::DETERMINE_OTHER;
tgtState = SdStates::DETERMINE_OTHER;
} else if (sdInfo.activeState == sd::SdState::OFF) {
// It's okay to do the delay after swichting active SD on, no one can use it anyway..
sdCardSetup(sdInfo.active, sd::SdState::ON, sdInfo.activeChar, false);
sdInfo.commandPending = true;
// Do not skip cycles here, would mess up the state machine. We skip the cycles after
// the SD card was switched on.
skipCycles = false;
// Remain on the current state.
tgtState = sdFsmState;
} else if (sdInfo.activeState == sd::SdState::ON) {
sdFsmState = SdStates::MOUNT_SELF;
// We can do the delay before mounting where applicable.
tgtState = SdStates::MOUNT_SELF;
}
if (skipCycles) {
sdFsmState = SdStates::SKIP_TWO_CYCLES_IF_SD_LOCKED;
fsmStateAfterDelay = tgtState;
sdInfo.skippedCyclesCount = 0;
} else {
sdFsmState = tgtState;
}
} else {
if (nonBlockingSdcOpChecking(SdStates::MOUNT_SELF, 10, "Setting SDC state")) {
sdInfo.activeState = sd::SdState::ON;
currentStateSetter(sdInfo.active, sd::SdState::ON);
// Skip the two cycles now.
if (sdInfo.lockSdCardUsage) {
sdFsmState = SdStates::SKIP_TWO_CYCLES_IF_SD_LOCKED;
fsmStateAfterDelay = SdStates::MOUNT_SELF;
sdInfo.skippedCyclesCount = 0;
}
}
}
}
if (sdFsmState == SdStates::SKIP_TWO_CYCLES_IF_SD_LOCKED) {
sdInfo.skippedCyclesCount++;
// Count to three because this branch will run in the same FSM cycle.
if (sdInfo.skippedCyclesCount == 3) {
sdFsmState = fsmStateAfterDelay;
fsmStateAfterDelay = SdStates::IDLE;
sdInfo.skippedCyclesCount = 0;
}
}
if (sdFsmState == SdStates::MOUNT_SELF) {
if (not sdInfo.commandPending) {
result = sdCardSetup(sdInfo.active, sd::SdState::MOUNTED, sdInfo.activeChar);
@ -529,6 +576,11 @@ ReturnValue_t CoreController::sdStateMachine() {
"Switching off other SD card")) {
sdInfo.otherState = sd::SdState::OFF;
currentStateSetter(sdInfo.other, sd::SdState::OFF);
} else {
// Continue.. avoid being stuck here..
sdFsmState = SdStates::SKIP_CYCLE_BEFORE_INFO_UPDATE;
sdInfo.otherState = sd::SdState::OFF;
currentStateSetter(sdInfo.other, sd::SdState::OFF);
}
}
} else if (sdInfo.cfgMode == SdCfgMode::HOT_REDUNDANT) {
@ -540,6 +592,11 @@ ReturnValue_t CoreController::sdStateMachine() {
"Switching on other SD card")) {
sdInfo.otherState = sd::SdState::ON;
currentStateSetter(sdInfo.other, sd::SdState::ON);
} else {
// Contnue.. avoid being stuck here.
sdFsmState = SdStates::MOUNT_UNMOUNT_OTHER;
sdInfo.otherState = sd::SdState::ON;
currentStateSetter(sdInfo.other, sd::SdState::ON);
}
}
}
@ -583,6 +640,8 @@ ReturnValue_t CoreController::sdStateMachine() {
sif::warning << "CoreController: Updating SD card state file failed" << std::endl;
}
updateInternalSdInfo();
// Mark usable again in any case.
sdcMan->markUsable();
sdInfo.commandPending = false;
sdFsmState = SdStates::IDLE;
sdInfo.cycleCount = 0;
@ -2032,6 +2091,11 @@ bool CoreController::startSdStateMachine(sd::SdCard targetActiveSd, SdCfgMode mo
}
sdFsmState = SdStates::START;
sdInfo.active = targetActiveSd;
// If we are going from 2 SD cards to one, lock SD card usage in any case because 1 SD card is
// going off.
if (sdInfo.cfgMode == SdCfgMode::HOT_REDUNDANT and mode == SdCfgMode::COLD_REDUNDANT) {
sdInfo.lockSdCardUsage = true;
}
sdInfo.cfgMode = mode;
sdCommandingInfo.actionId = actionId;
sdCommandingInfo.commander = commander;

View File

@ -130,6 +130,7 @@ class CoreController : public ExtendedControllerBase {
NONE,
START,
UPDATE_SD_INFO_START,
SKIP_TWO_CYCLES_IF_SD_LOCKED,
MOUNT_SELF,
// Determine operations for other SD card, depending on redundancy configuration
DETERMINE_OTHER,
@ -152,6 +153,7 @@ class CoreController : public ExtendedControllerBase {
MessageQueueIF* eventQueue = nullptr;
SdStates sdFsmState = SdStates::START;
SdStates fsmStateAfterDelay = SdStates::IDLE;
enum SdCfgMode { PASSIVE, COLD_REDUNDANT, HOT_REDUNDANT };
struct SdFsmParams {
@ -163,10 +165,16 @@ class CoreController : public ExtendedControllerBase {
sd::SdState activeState = sd::SdState::OFF;
sd::SdState otherState = sd::SdState::OFF;
std::pair<bool, bool> mountSwitch = {true, true};
// This flag denotes that the SD card usage is locked. This is relevant if SD cards go off
// to leave appliation using the SD cards some time to detect the SD card is not usable anymore.
// This is relevant if the active SD card is being switched. The SD card will also be locked
// when going from hot-redundant mode to cold-redundant mode.
bool lockSdCardUsage = false;
bool commandPending = true;
bool initFinished = false;
SdCardManager::SdStatePair currentState;
uint16_t cycleCount = 0;
uint16_t skippedCyclesCount = 0;
} sdInfo;
struct SdCommanding {

View File

@ -581,3 +581,8 @@ void SdCardManager::markUnusable() {
MutexGuard mg(defaultLock, LOCK_TYPE, OTHER_TIMEOUT, LOCK_CTX);
markedUnusable = true;
}
void SdCardManager::markUsable() {
MutexGuard mg(defaultLock, LOCK_TYPE, OTHER_TIMEOUT, LOCK_CTX);
markedUnusable = false;
}

View File

@ -205,6 +205,7 @@ class SdCardManager : public SystemObject, public SdCardMountedIF {
ReturnValue_t performFsck(sd::SdCard sdcard, bool printOutput, int& linuxError);
void markUnusable();
void markUsable();
private:
CommandExecutor cmdExecutor;

View File

@ -338,6 +338,8 @@ ReturnValue_t PersistentTmStore::createMostRecentFile(std::optional<uint8_t> suf
ReturnValue_t PersistentTmStore::initializeTmStore() {
Clock::getClock_timeval(&currentTv);
updateBaseDir();
// Reset active file, base directory might have changed.
activeFile = std::nullopt;
return assignAndOrCreateMostRecentFile();
}