theoretically done

mission/system/EiveSystem.cpp

@@ -13,9 +13,10 @@
 #include "mission/sysDefs.h"
 
 EiveSystem::EiveSystem(object_id_t setObjectId, uint32_t maxNumberOfSequences,
-                       uint32_t maxNumberOfTables)
+                       uint32_t maxNumberOfTables, std::atomic_uint16_t& i2cErrors)
     : Subsystem(setObjectId, maxNumberOfSequences, maxNumberOfTables),
-      actionHelper(this, commandQueue) {
+      actionHelper(this, commandQueue),
+      i2cErrors(i2cErrors) {
   auto mqArgs = MqArgs(SubsystemBase::getObjectId(), static_cast<void*>(this));
   eventQueue =
       QueueFactory::instance()->createMessageQueue(10, EventMessage::EVENT_MESSAGE_SIZE, &mqArgs);
@@ -97,6 +98,7 @@ ReturnValue_t EiveSystem::initialize() {
                             event::getEventId(tcsCtrl::PCDU_SYSTEM_OVERHEATING));
   manager->subscribeToEvent(eventQueue->getId(), event::getEventId(tcsCtrl::OBC_OVERHEATING));
 
+  // manager->subscribeToEvent(eventQueue->getId(), event::getEventId(CoreController::))
   return Subsystem::initialize();
 }
 
@@ -133,6 +135,9 @@ ReturnValue_t EiveSystem::executeAction(ActionId_t actionId, MessageQueueId_t co
   switch (actionId) {
     case (EXECUTE_I2C_REBOOT): {
       performI2cReboot = true;
+      // This flag is more related to autonomous recovery handling, so we reset it here if this
+      // reboot sequence is commanded manually.
+      alreadyTriedI2cRecovery = false;
       i2cRebootState = I2cRebootState::SYSTEM_MODE_BOOT;
       this->actionCommandedBy = commandedBy;
       return returnvalue::OK;
@@ -144,36 +149,58 @@ ReturnValue_t EiveSystem::executeAction(ActionId_t actionId, MessageQueueId_t co
   return returnvalue::OK;
 }
 
-void EiveSystem::setPowerSwitcher(PowerSwitchIF* pwrSwitcher) { this->powerSwitcher = pwrSwitcher; }
+void EiveSystem::setI2cRecoveryParams(PowerSwitchIF* pwrSwitcher) {
+  this->powerSwitcher = pwrSwitcher;
+}
 
 void EiveSystem::i2cRecoveryLogic() {
   ReturnValue_t result;
-  if (alreadyTriedI2cRecovery and i2cRecoveryClearCountdown.hasTimedOut()) {
-    alreadyTriedI2cRecovery = false;
+  if (not performI2cReboot) {
+    // If a recovery worked, need to reset these flags and the error count after some time.
+    if (i2cRecoveryClearCountdown.hasTimedOut()) {
+      i2cErrors = 0;
+      alreadyTriedI2cRecovery = false;
+    }
+    // If an I2C recovery is not ongoing and the I2C error counter is above a threshold, try
+    // recovery or reboot if recovery was already attempted.
+    if (i2cErrors >= 5) {
+      if (not alreadyTriedI2cRecovery) {
+        // Try recovery.
+        executeAction(EXECUTE_I2C_REBOOT, MessageQueueIF::NO_QUEUE, nullptr, 0);
+      } else {
+        // We already tried an I2C recovery but the bus is still broken.
+        // Send full reboot request to core controller.
+        CommandMessage msg;
+        ActionMessage::setCommand(&msg, core::REBOOT_OBC, store_address_t());
+        result = commandQueue->sendMessage(coreCtrlQueueId, &msg);
+      }
+    }
   }
   if (not isInTransition and performI2cReboot) {
     if (i2cRebootState == I2cRebootState::SYSTEM_MODE_BOOT) {
       startTransition(satsystem::Mode::BOOT, 0);
       i2cRebootState = I2cRebootState::SWITCH_3V3_STACK_OFF_AND_BATT_REBOOT;
-      i2cRebootCountdown.resetTimer();
+      i2cRebootHandlingCountdown.resetTimer();
     } else if (i2cRebootState == I2cRebootState::SWITCH_3V3_STACK_OFF_AND_BATT_REBOOT) {
       if (mode == satsystem::Mode::BOOT) {
         result = powerSwitcher->sendSwitchCommand(power::Switches::P60_DOCK_3V3_STACK,
                                                   PowerSwitchIF::SWITCH_OFF);
         if (result != returnvalue::OK) {
           actionHelper.finish(false, actionCommandedBy, EXECUTE_I2C_REBOOT, result);
-          performI2cReboot = false;
+          commonI2cRecoverySequenceFinish();
+          return;
         }
         CommandMessage msg;
         ActionMessage::setCommand(&msg, BpxBattery::REBOOT, store_address_t());
         result = commandQueue->sendMessage(bpxBattQueueId, &msg);
         if (result != returnvalue::OK) {
           actionHelper.finish(false, actionCommandedBy, EXECUTE_I2C_REBOOT, result);
-          performI2cReboot = false;
+          commonI2cRecoverySequenceFinish();
+          return;
         }
         i2cRebootState = I2cRebootState::SWITCH_3V3_STACK_ON;
       }
-      if (i2cRebootCountdown.hasTimedOut()) {
+      if (i2cRebootHandlingCountdown.hasTimedOut()) {
         actionHelper.finish(false, actionCommandedBy, EXECUTE_I2C_REBOOT, returnvalue::FAILED);
         performI2cReboot = false;
       }
@@ -182,33 +209,40 @@ void EiveSystem::i2cRecoveryLogic() {
                                                 PowerSwitchIF::SWITCH_ON);
       if (result != returnvalue::OK) {
         actionHelper.finish(false, actionCommandedBy, EXECUTE_I2C_REBOOT, result);
-        performI2cReboot = false;
+        commonI2cRecoverySequenceFinish();
+        return;
       }
       i2cRebootState = I2cRebootState::SYSTEM_MODE_SAFE;
     } else if (i2cRebootState == I2cRebootState::SYSTEM_MODE_SAFE) {
       if (powerSwitcher->getSwitchState(power::Switches::P60_DOCK_3V3_STACK) ==
           PowerSwitchIF::SWITCH_ON) {
         // This should always be accepted
-        commandSelfToSafe();
-        i2cRebootState = I2cRebootState::NONE;
-        alreadyTriedI2cRecovery = true;
-        i2cRecoveryClearCountdown.resetTimer();
-        performI2cReboot = false;
+        commonI2cRecoverySequenceFinish();
         actionHelper.finish(true, actionCommandedBy, EXECUTE_I2C_REBOOT);
       }
     }
 
     // Timeout handling for the internal procedure.
-    if (i2cRebootState != I2cRebootState::NONE and i2cRebootCountdown.hasTimedOut()) {
+    if (i2cRebootState != I2cRebootState::NONE and i2cRebootHandlingCountdown.hasTimedOut()) {
       actionHelper.finish(false, actionCommandedBy, EXECUTE_I2C_REBOOT, returnvalue::FAILED);
+      // Command stack back on in any case.
       powerSwitcher->sendSwitchCommand(power::Switches::P60_DOCK_3V3_STACK,
                                        PowerSwitchIF::SWITCH_ON);
-      alreadyTriedI2cRecovery = true;
-      i2cRecoveryClearCountdown.resetTimer();
-      // This should always be accepted
-      commandSelfToSafe();
+      commonI2cRecoverySequenceFinish();
     }
   }
 }
 
 void EiveSystem::commandSelfToSafe() { startTransition(satsystem::Mode::SAFE, 0); }
+
+void EiveSystem::commonI2cRecoverySequenceFinish() {
+  alreadyTriedI2cRecovery = true;
+  performI2cReboot = false;
+  i2cRecoveryClearCountdown.resetTimer();
+  i2cRebootState = I2cRebootState::NONE;
+  // Reset this counter. If I2C devices are still problematic, we will get a full reboot
+  // next time this count goes above 5.
+  i2cErrors = 0;
+  // This should always be accepted
+  commandSelfToSafe();
+}