From 58aef99bbc93d013f4286abde25dfb60a490ca1a Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Thu, 6 Apr 2023 17:49:34 +0200 Subject: [PATCH] theoretically done --- bsp_q7s/core/CoreController.cpp | 28 ++++----- bsp_q7s/core/CoreController.h | 60 +----------------- bsp_q7s/core/ObjectFactory.h | 1 - bsp_q7s/em/emObjectFactory.cpp | 2 +- bsp_q7s/fmObjectFactory.cpp | 2 +- mission/sysDefs.h | 64 ++++++++++++++++++++ mission/system/EiveSystem.cpp | 74 +++++++++++++++++------ mission/system/EiveSystem.h | 16 +++-- mission/system/systemTree.cpp | 4 +- unittest/rebootLogic/src/CoreController.h | 7 +-- 10 files changed, 149 insertions(+), 109 deletions(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index c895106b..5f3333f4 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -9,6 +9,7 @@ #include "fsfw/serviceinterface/ServiceInterface.h" #include "fsfw/timemanager/Stopwatch.h" #include "fsfw/version.h" +#include "mission/sysDefs.h" #include "watchdog/definitions.h" #if OBSW_ADD_TMTC_UDP_SERVER == 1 #include "fsfw/osal/common/UdpTmTcBridge.h" @@ -31,8 +32,7 @@ xsc::Chip CoreController::CURRENT_CHIP = xsc::Chip::NO_CHIP; xsc::Copy CoreController::CURRENT_COPY = xsc::Copy::NO_COPY; -CoreController::CoreController(object_id_t objectId, const std::atomic_uint16_t &i2cErrors, - bool enableHkSet) +CoreController::CoreController(object_id_t objectId, bool enableHkSet) : ExtendedControllerBase(objectId, 5), enableHkSet(enableHkSet), cmdExecutor(4096), @@ -40,8 +40,7 @@ CoreController::CoreController(object_id_t objectId, const std::atomic_uint16_t cmdRepliesSizes(128), opDivider5(5), opDivider10(10), - hkSet(this), - i2cErrors(i2cErrors) { + hkSet(this) { cmdExecutor.setRingBuffer(&cmdReplyBuf, &cmdRepliesSizes); try { sdcMan = SdCardManager::instance(); @@ -110,17 +109,12 @@ void CoreController::performControlOperation() { sdStateMachine(); performMountedSdCardOperations(); readHkData(); - if (i2cErrors >= 5) { - bool protOpPerformed = false; - triggerEvent(I2C_UNAVAILABLE_REBOOT); - gracefulShutdownTasks(CURRENT_CHIP, CURRENT_COPY, protOpPerformed); - std::system("xsc_boot_copy -r"); - } + if (shellCmdIsExecuting) { bool replyReceived = false; // TODO: We could read the data in the ring buffer and send it as an action data reply. if (cmdExecutor.check(replyReceived) == CommandExecutor::EXECUTION_FINISHED) { - actionHelper.finish(true, successRecipient, EXECUTE_SHELL_CMD); + actionHelper.finish(true, successRecipient, core::EXECUTE_SHELL_CMD); shellCmdIsExecuting = false; cmdReplyBuf.clear(); while (not cmdRepliesSizes.empty()) { @@ -163,7 +157,7 @@ ReturnValue_t CoreController::initialize() { sdStateMachine(); - triggerEvent(REBOOT_SW, CURRENT_CHIP, CURRENT_COPY); + triggerEvent(core::REBOOT_SW, CURRENT_CHIP, CURRENT_COPY); EventManagerIF *eventManager = ObjectManager::instance()->get(objects::EVENT_MANAGER); if (eventManager == nullptr or eventQueue == nullptr) { @@ -202,6 +196,7 @@ ReturnValue_t CoreController::initializeAfterTaskCreation() { ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_t commandedBy, const uint8_t *data, size_t size) { + using namespace core; switch (actionId) { case (ANNOUNCE_VERSION): { uint32_t p1 = (common::OBSW_VERSION_MAJOR << 24) | (common::OBSW_VERSION_MINOR << 16) | @@ -1324,7 +1319,7 @@ ReturnValue_t CoreController::performSdCardCheck() { someSdCardActive = true; } if (not someSdCardActive and remountAttemptFlag) { - triggerEvent(NO_SD_CARD_ACTIVE); + triggerEvent(core::NO_SD_CARD_ACTIVE); initSdCardBlocking(); remountAttemptFlag = false; } @@ -1378,7 +1373,7 @@ void CoreController::performRebootFileHandling(bool recreateFile) { if (rebootFile.bootFlag) { // Trigger event to inform ground that a reboot was triggered uint32_t p1 = rebootFile.lastChip << 16 | rebootFile.lastCopy; - triggerEvent(REBOOT_MECHANISM_TRIGGERED, p1, 0); + triggerEvent(core::REBOOT_MECHANISM_TRIGGERED, p1, 0); // Clear the boot flag rebootFile.bootFlag = false; } @@ -2047,8 +2042,9 @@ void CoreController::announceBootCounts() { rebootFile.img00Cnt + rebootFile.img01Cnt + rebootFile.img10Cnt + rebootFile.img11Cnt; uint32_t individualBootCountsP1 = (rebootFile.img00Cnt << 16) | rebootFile.img01Cnt; uint32_t individualBootCountsP2 = (rebootFile.img10Cnt << 16) | rebootFile.img11Cnt; - triggerEvent(INDIVIDUAL_BOOT_COUNTS, individualBootCountsP1, individualBootCountsP2); - triggerEvent(REBOOT_COUNTER, (totalBootCount >> 32) & 0xffffffff, totalBootCount & 0xffffffff); + triggerEvent(core::INDIVIDUAL_BOOT_COUNTS, individualBootCountsP1, individualBootCountsP2); + triggerEvent(core::REBOOT_COUNTER, (totalBootCount >> 32) & 0xffffffff, + totalBootCount & 0xffffffff); } bool CoreController::isNumber(const std::string &s) { diff --git a/bsp_q7s/core/CoreController.h b/bsp_q7s/core/CoreController.h index 72d63b15..53a96cc3 100644 --- a/bsp_q7s/core/CoreController.h +++ b/bsp_q7s/core/CoreController.h @@ -78,64 +78,7 @@ class CoreController : public ExtendedControllerBase { static constexpr dur_millis_t INIT_SD_CARD_CHECK_TIMEOUT = 5000; static constexpr dur_millis_t DEFAULT_SD_CARD_CHECK_TIMEOUT = 60000; - static constexpr ActionId_t LIST_DIRECTORY_INTO_FILE = 0; - static constexpr ActionId_t ANNOUNCE_VERSION = 1; - static constexpr ActionId_t ANNOUNCE_CURRENT_IMAGE = 2; - static constexpr ActionId_t ANNOUNCE_BOOT_COUNTS = 3; - static constexpr ActionId_t SWITCH_REBOOT_FILE_HANDLING = 5; - static constexpr ActionId_t RESET_REBOOT_COUNTERS = 6; - static constexpr ActionId_t SWITCH_IMG_LOCK = 7; - static constexpr ActionId_t SET_MAX_REBOOT_CNT = 8; - - static constexpr ActionId_t OBSW_UPDATE_FROM_SD_0 = 10; - static constexpr ActionId_t OBSW_UPDATE_FROM_SD_1 = 11; - static constexpr ActionId_t OBSW_UPDATE_FROM_TMP = 12; - - static constexpr ActionId_t SWITCH_TO_SD_0 = 16; - static constexpr ActionId_t SWITCH_TO_SD_1 = 17; - static constexpr ActionId_t SWITCH_TO_BOTH_SD_CARDS = 18; - - //! Reboot using the xsc_boot_copy command - static constexpr ActionId_t XSC_REBOOT_OBC = 32; - static constexpr ActionId_t MOUNT_OTHER_COPY = 33; - //! Reboot using the reboot command - static constexpr ActionId_t REBOOT_OBC = 34; - - static constexpr ActionId_t EXECUTE_SHELL_CMD = 40; - - static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::CORE; - - static constexpr Event ALLOC_FAILURE = event::makeEvent(SUBSYSTEM_ID, 0, severity::MEDIUM); - //! [EXPORT] : [COMMENT] Software reboot occurred. Can also be a systemd reboot. - //! P1: Current Chip, P2: Current Copy - static constexpr Event REBOOT_SW = event::makeEvent(SUBSYSTEM_ID, 1, severity::LOW); - //! [EXPORT] : [COMMENT] The reboot mechanism was triggered. - //! P1: First 16 bits: Last Chip, Last 16 bits: Last Copy, - //! P2: Each byte is the respective reboot count for the slots - static constexpr Event REBOOT_MECHANISM_TRIGGERED = - event::makeEvent(SUBSYSTEM_ID, 2, severity::MEDIUM); - //! Trying to find a way how to determine that the reboot came from ProASIC3 or PCDU.. - static constexpr Event REBOOT_HW = event::makeEvent(SUBSYSTEM_ID, 3, severity::MEDIUM); - //! [EXPORT] : [COMMENT] No SD card was active. Core controller will attempt to re-initialize - //! a SD card. - static constexpr Event NO_SD_CARD_ACTIVE = event::makeEvent(SUBSYSTEM_ID, 4, severity::HIGH); - //! [EXPORT] : [COMMENT] - //! P1: Byte 0: Major, Byte 1: Minor, Byte 2: Patch, Byte 3: Has Git Hash - //! P2: First four letters of Git SHA is the last byte of P1 is set. - static constexpr Event VERSION_INFO = event::makeEvent(SUBSYSTEM_ID, 5, severity::INFO); - //! [EXPORT] : [COMMENT] P1: Current Chip, P2: Current Copy - static constexpr Event CURRENT_IMAGE_INFO = event::makeEvent(SUBSYSTEM_ID, 6, severity::INFO); - //! [EXPORT] : [COMMENT] Total reboot counter, which is the sum of the boot count of all - //! individual images. - static constexpr Event REBOOT_COUNTER = event::makeEvent(SUBSYSTEM_ID, 7, severity::INFO); - //! [EXPORT] : [COMMENT] Get the boot count of the individual images. - //! P1: First 16 bits boot count of image 0 0, last 16 bits boot count of image 0 1. - //! P2: First 16 bits boot count of image 1 0, last 16 bits boot count of image 1 1. - static constexpr Event INDIVIDUAL_BOOT_COUNTS = event::makeEvent(SUBSYSTEM_ID, 8, severity::INFO); - static constexpr Event I2C_UNAVAILABLE_REBOOT = - event::makeEvent(SUBSYSTEM_ID, 10, severity::MEDIUM); - - CoreController(object_id_t objectId, const std::atomic_uint16_t& i2cErrors, bool enableHkSet); + CoreController(object_id_t objectId, bool enableHkSet); virtual ~CoreController(); ReturnValue_t initialize() override; @@ -266,7 +209,6 @@ class CoreController : public ExtendedControllerBase { PoolEntry plVoltageEntry = PoolEntry(0.0); core::HkSet hkSet; - const std::atomic_uint16_t& i2cErrors; #if OBSW_SD_CARD_MUST_BE_ON == 1 bool remountAttemptFlag = true; diff --git a/bsp_q7s/core/ObjectFactory.h b/bsp_q7s/core/ObjectFactory.h index 9daa4d01..b5685389 100644 --- a/bsp_q7s/core/ObjectFactory.h +++ b/bsp_q7s/core/ObjectFactory.h @@ -23,7 +23,6 @@ class HealthTableIF; class AcsBoardAssembly; class GpioIF; -extern std::atomic_uint16_t I2C_FATAL_ERRORS; extern std::atomic_bool PTME_LOCKED; namespace ObjectFactory { diff --git a/bsp_q7s/em/emObjectFactory.cpp b/bsp_q7s/em/emObjectFactory.cpp index ca1a54ae..e43d34f2 100644 --- a/bsp_q7s/em/emObjectFactory.cpp +++ b/bsp_q7s/em/emObjectFactory.cpp @@ -71,7 +71,7 @@ void ObjectFactory::produce(void* args) { dummy::createDummies(dummyCfg, *pwrSwitcher, gpioComIF); - new CoreController(objects::CORE_CONTROLLER, I2C_FATAL_ERRORS, enableHkSets); + new CoreController(objects::CORE_CONTROLLER, enableHkSets); // Regular FM code, does not work for EM if the hardware is not connected // createPcduComponents(gpioComIF, &pwrSwitcher); diff --git a/bsp_q7s/fmObjectFactory.cpp b/bsp_q7s/fmObjectFactory.cpp index 699c2569..050dc9d3 100644 --- a/bsp_q7s/fmObjectFactory.cpp +++ b/bsp_q7s/fmObjectFactory.cpp @@ -46,7 +46,7 @@ void ObjectFactory::produce(void* args) { new CoreController(objects::CORE_CONTROLLER, I2C_FATAL_ERRORS, enableHkSets); createPcduComponents(gpioComIF, &pwrSwitcher, enableHkSets); - satsystem::EIVE_SYSTEM.setPowerSwitcher(pwrSwitcher); + satsystem::EIVE_SYSTEM.setI2cRecoveryParams(I2C_FATAL_ERRORS, pwrSwitcher); auto* stackHandler = new Stack5VHandler(*pwrSwitcher); diff --git a/mission/sysDefs.h b/mission/sysDefs.h index 70ff8138..2b542c82 100644 --- a/mission/sysDefs.h +++ b/mission/sysDefs.h @@ -1,12 +1,76 @@ #ifndef MISSION_SYSDEFS_H_ #define MISSION_SYSDEFS_H_ +#include + #include "acs/defs.h" +extern std::atomic_uint16_t I2C_FATAL_ERRORS; + namespace satsystem { enum Mode : Mode_t { BOOT = 5, SAFE = acs::AcsMode::SAFE, PTG_IDLE = acs::AcsMode::PTG_IDLE }; } +namespace core { +static constexpr ActionId_t LIST_DIRECTORY_INTO_FILE = 0; +static constexpr ActionId_t ANNOUNCE_VERSION = 1; +static constexpr ActionId_t ANNOUNCE_CURRENT_IMAGE = 2; +static constexpr ActionId_t ANNOUNCE_BOOT_COUNTS = 3; +static constexpr ActionId_t SWITCH_REBOOT_FILE_HANDLING = 5; +static constexpr ActionId_t RESET_REBOOT_COUNTERS = 6; +static constexpr ActionId_t SWITCH_IMG_LOCK = 7; +static constexpr ActionId_t SET_MAX_REBOOT_CNT = 8; + +static constexpr ActionId_t OBSW_UPDATE_FROM_SD_0 = 10; +static constexpr ActionId_t OBSW_UPDATE_FROM_SD_1 = 11; +static constexpr ActionId_t OBSW_UPDATE_FROM_TMP = 12; + +static constexpr ActionId_t SWITCH_TO_SD_0 = 16; +static constexpr ActionId_t SWITCH_TO_SD_1 = 17; +static constexpr ActionId_t SWITCH_TO_BOTH_SD_CARDS = 18; + +//! Reboot using the xsc_boot_copy command +static constexpr ActionId_t XSC_REBOOT_OBC = 32; +static constexpr ActionId_t MOUNT_OTHER_COPY = 33; +//! Reboot using the reboot command +static constexpr ActionId_t REBOOT_OBC = 34; + +static constexpr ActionId_t EXECUTE_SHELL_CMD = 40; + +static constexpr uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::CORE; + +static constexpr Event ALLOC_FAILURE = event::makeEvent(SUBSYSTEM_ID, 0, severity::MEDIUM); +//! [EXPORT] : [COMMENT] Software reboot occurred. Can also be a systemd reboot. +//! P1: Current Chip, P2: Current Copy +static constexpr Event REBOOT_SW = event::makeEvent(SUBSYSTEM_ID, 1, severity::LOW); +//! [EXPORT] : [COMMENT] The reboot mechanism was triggered. +//! P1: First 16 bits: Last Chip, Last 16 bits: Last Copy, +//! P2: Each byte is the respective reboot count for the slots +static constexpr Event REBOOT_MECHANISM_TRIGGERED = + event::makeEvent(SUBSYSTEM_ID, 2, severity::MEDIUM); +//! Trying to find a way how to determine that the reboot came from ProASIC3 or PCDU.. +static constexpr Event REBOOT_HW = event::makeEvent(SUBSYSTEM_ID, 3, severity::MEDIUM); +//! [EXPORT] : [COMMENT] No SD card was active. Core controller will attempt to re-initialize +//! a SD card. +static constexpr Event NO_SD_CARD_ACTIVE = event::makeEvent(SUBSYSTEM_ID, 4, severity::HIGH); +//! [EXPORT] : [COMMENT] +//! P1: Byte 0: Major, Byte 1: Minor, Byte 2: Patch, Byte 3: Has Git Hash +//! P2: First four letters of Git SHA is the last byte of P1 is set. +static constexpr Event VERSION_INFO = event::makeEvent(SUBSYSTEM_ID, 5, severity::INFO); +//! [EXPORT] : [COMMENT] P1: Current Chip, P2: Current Copy +static constexpr Event CURRENT_IMAGE_INFO = event::makeEvent(SUBSYSTEM_ID, 6, severity::INFO); +//! [EXPORT] : [COMMENT] Total reboot counter, which is the sum of the boot count of all +//! individual images. +static constexpr Event REBOOT_COUNTER = event::makeEvent(SUBSYSTEM_ID, 7, severity::INFO); +//! [EXPORT] : [COMMENT] Get the boot count of the individual images. +//! P1: First 16 bits boot count of image 0 0, last 16 bits boot count of image 0 1. +//! P2: First 16 bits boot count of image 1 0, last 16 bits boot count of image 1 1. +static constexpr Event INDIVIDUAL_BOOT_COUNTS = event::makeEvent(SUBSYSTEM_ID, 8, severity::INFO); +//! [EXPORT] : [COMMENT] I2C is unavailable. Triggered to system object can take further steps +//! like I2C reboot or reboot. +static constexpr Event I2C_UNAVAILABLE = event::makeEvent(SUBSYSTEM_ID, 10, severity::MEDIUM); +} // namespace core + #endif /* MISSION_SYSDEFS_H_ */ diff --git a/mission/system/EiveSystem.cpp b/mission/system/EiveSystem.cpp index 1a29507c..c7a6f021 100644 --- a/mission/system/EiveSystem.cpp +++ b/mission/system/EiveSystem.cpp @@ -13,9 +13,10 @@ #include "mission/sysDefs.h" EiveSystem::EiveSystem(object_id_t setObjectId, uint32_t maxNumberOfSequences, - uint32_t maxNumberOfTables) + uint32_t maxNumberOfTables, std::atomic_uint16_t& i2cErrors) : Subsystem(setObjectId, maxNumberOfSequences, maxNumberOfTables), - actionHelper(this, commandQueue) { + actionHelper(this, commandQueue), + i2cErrors(i2cErrors) { auto mqArgs = MqArgs(SubsystemBase::getObjectId(), static_cast(this)); eventQueue = QueueFactory::instance()->createMessageQueue(10, EventMessage::EVENT_MESSAGE_SIZE, &mqArgs); @@ -97,6 +98,7 @@ ReturnValue_t EiveSystem::initialize() { event::getEventId(tcsCtrl::PCDU_SYSTEM_OVERHEATING)); manager->subscribeToEvent(eventQueue->getId(), event::getEventId(tcsCtrl::OBC_OVERHEATING)); + // manager->subscribeToEvent(eventQueue->getId(), event::getEventId(CoreController::)) return Subsystem::initialize(); } @@ -133,6 +135,9 @@ ReturnValue_t EiveSystem::executeAction(ActionId_t actionId, MessageQueueId_t co switch (actionId) { case (EXECUTE_I2C_REBOOT): { performI2cReboot = true; + // This flag is more related to autonomous recovery handling, so we reset it here if this + // reboot sequence is commanded manually. + alreadyTriedI2cRecovery = false; i2cRebootState = I2cRebootState::SYSTEM_MODE_BOOT; this->actionCommandedBy = commandedBy; return returnvalue::OK; @@ -144,36 +149,58 @@ ReturnValue_t EiveSystem::executeAction(ActionId_t actionId, MessageQueueId_t co return returnvalue::OK; } -void EiveSystem::setPowerSwitcher(PowerSwitchIF* pwrSwitcher) { this->powerSwitcher = pwrSwitcher; } +void EiveSystem::setI2cRecoveryParams(PowerSwitchIF* pwrSwitcher) { + this->powerSwitcher = pwrSwitcher; +} void EiveSystem::i2cRecoveryLogic() { ReturnValue_t result; - if (alreadyTriedI2cRecovery and i2cRecoveryClearCountdown.hasTimedOut()) { - alreadyTriedI2cRecovery = false; + if (not performI2cReboot) { + // If a recovery worked, need to reset these flags and the error count after some time. + if (i2cRecoveryClearCountdown.hasTimedOut()) { + i2cErrors = 0; + alreadyTriedI2cRecovery = false; + } + // If an I2C recovery is not ongoing and the I2C error counter is above a threshold, try + // recovery or reboot if recovery was already attempted. + if (i2cErrors >= 5) { + if (not alreadyTriedI2cRecovery) { + // Try recovery. + executeAction(EXECUTE_I2C_REBOOT, MessageQueueIF::NO_QUEUE, nullptr, 0); + } else { + // We already tried an I2C recovery but the bus is still broken. + // Send full reboot request to core controller. + CommandMessage msg; + ActionMessage::setCommand(&msg, core::REBOOT_OBC, store_address_t()); + result = commandQueue->sendMessage(coreCtrlQueueId, &msg); + } + } } if (not isInTransition and performI2cReboot) { if (i2cRebootState == I2cRebootState::SYSTEM_MODE_BOOT) { startTransition(satsystem::Mode::BOOT, 0); i2cRebootState = I2cRebootState::SWITCH_3V3_STACK_OFF_AND_BATT_REBOOT; - i2cRebootCountdown.resetTimer(); + i2cRebootHandlingCountdown.resetTimer(); } else if (i2cRebootState == I2cRebootState::SWITCH_3V3_STACK_OFF_AND_BATT_REBOOT) { if (mode == satsystem::Mode::BOOT) { result = powerSwitcher->sendSwitchCommand(power::Switches::P60_DOCK_3V3_STACK, PowerSwitchIF::SWITCH_OFF); if (result != returnvalue::OK) { actionHelper.finish(false, actionCommandedBy, EXECUTE_I2C_REBOOT, result); - performI2cReboot = false; + commonI2cRecoverySequenceFinish(); + return; } CommandMessage msg; ActionMessage::setCommand(&msg, BpxBattery::REBOOT, store_address_t()); result = commandQueue->sendMessage(bpxBattQueueId, &msg); if (result != returnvalue::OK) { actionHelper.finish(false, actionCommandedBy, EXECUTE_I2C_REBOOT, result); - performI2cReboot = false; + commonI2cRecoverySequenceFinish(); + return; } i2cRebootState = I2cRebootState::SWITCH_3V3_STACK_ON; } - if (i2cRebootCountdown.hasTimedOut()) { + if (i2cRebootHandlingCountdown.hasTimedOut()) { actionHelper.finish(false, actionCommandedBy, EXECUTE_I2C_REBOOT, returnvalue::FAILED); performI2cReboot = false; } @@ -182,33 +209,40 @@ void EiveSystem::i2cRecoveryLogic() { PowerSwitchIF::SWITCH_ON); if (result != returnvalue::OK) { actionHelper.finish(false, actionCommandedBy, EXECUTE_I2C_REBOOT, result); - performI2cReboot = false; + commonI2cRecoverySequenceFinish(); + return; } i2cRebootState = I2cRebootState::SYSTEM_MODE_SAFE; } else if (i2cRebootState == I2cRebootState::SYSTEM_MODE_SAFE) { if (powerSwitcher->getSwitchState(power::Switches::P60_DOCK_3V3_STACK) == PowerSwitchIF::SWITCH_ON) { // This should always be accepted - commandSelfToSafe(); - i2cRebootState = I2cRebootState::NONE; - alreadyTriedI2cRecovery = true; - i2cRecoveryClearCountdown.resetTimer(); - performI2cReboot = false; + commonI2cRecoverySequenceFinish(); actionHelper.finish(true, actionCommandedBy, EXECUTE_I2C_REBOOT); } } // Timeout handling for the internal procedure. - if (i2cRebootState != I2cRebootState::NONE and i2cRebootCountdown.hasTimedOut()) { + if (i2cRebootState != I2cRebootState::NONE and i2cRebootHandlingCountdown.hasTimedOut()) { actionHelper.finish(false, actionCommandedBy, EXECUTE_I2C_REBOOT, returnvalue::FAILED); + // Command stack back on in any case. powerSwitcher->sendSwitchCommand(power::Switches::P60_DOCK_3V3_STACK, PowerSwitchIF::SWITCH_ON); - alreadyTriedI2cRecovery = true; - i2cRecoveryClearCountdown.resetTimer(); - // This should always be accepted - commandSelfToSafe(); + commonI2cRecoverySequenceFinish(); } } } void EiveSystem::commandSelfToSafe() { startTransition(satsystem::Mode::SAFE, 0); } + +void EiveSystem::commonI2cRecoverySequenceFinish() { + alreadyTriedI2cRecovery = true; + performI2cReboot = false; + i2cRecoveryClearCountdown.resetTimer(); + i2cRebootState = I2cRebootState::NONE; + // Reset this counter. If I2C devices are still problematic, we will get a full reboot + // next time this count goes above 5. + i2cErrors = 0; + // This should always be accepted + commandSelfToSafe(); +} diff --git a/mission/system/EiveSystem.h b/mission/system/EiveSystem.h index c06fa910..d31db17d 100644 --- a/mission/system/EiveSystem.h +++ b/mission/system/EiveSystem.h @@ -5,13 +5,16 @@ #include #include +#include + class EiveSystem : public Subsystem, public HasActionsIF { public: static constexpr ActionId_t EXECUTE_I2C_REBOOT = 10; - EiveSystem(object_id_t setObjectId, uint32_t maxNumberOfSequences, uint32_t maxNumberOfTables); + EiveSystem(object_id_t setObjectId, uint32_t maxNumberOfSequences, uint32_t maxNumberOfTables, + std::atomic_uint16_t& i2cErrors); - void setPowerSwitcher(PowerSwitchIF* pwrSwitcher); + void setI2cRecoveryParams(PowerSwitchIF* pwrSwitcher); [[nodiscard]] MessageQueueId_t getCommandQueue() const override; @@ -31,10 +34,14 @@ class EiveSystem : public Subsystem, public HasActionsIF { ActionHelper actionHelper; PowerSwitchIF* powerSwitcher = nullptr; + std::atomic_uint16_t& i2cErrors; + MessageQueueId_t bpxBattQueueId = MessageQueueIF::NO_QUEUE; + MessageQueueId_t coreCtrlQueueId = MessageQueueIF::NO_QUEUE; MessageQueueId_t actionCommandedBy = MessageQueueIF::NO_QUEUE; - Countdown i2cRebootCountdown = Countdown(10000); - // After 1 minute, clear the flag to avoid full reboots on I2C issues. + Countdown i2cRebootHandlingCountdown = Countdown(10000); + // Countdown i2cUnavailableCountdown = Countdown(30000); + // After 1 minute, clear the flag to avoid full reboots on I2C issues. Countdown i2cRecoveryClearCountdown = Countdown(60000); ReturnValue_t initialize() override; void performChildOperation() override; @@ -45,6 +52,7 @@ class EiveSystem : public Subsystem, public HasActionsIF { void i2cRecoveryLogic(); void handleEventMessages(); void commandSelfToSafe(); + void commonI2cRecoverySequenceFinish(); }; #endif /* MISSION_SYSTEM_EIVESYSTEM_H_ */ diff --git a/mission/system/systemTree.cpp b/mission/system/systemTree.cpp index 2cecf244..73a6b4b7 100644 --- a/mission/system/systemTree.cpp +++ b/mission/system/systemTree.cpp @@ -6,6 +6,8 @@ #include #include +#include + #include "eive/objects.h" #include "mission/com/defs.h" #include "mission/system/acs/acsModeTree.h" @@ -41,7 +43,7 @@ void satsystem::init() { EIVE_SYSTEM.setInitialMode(satsystem::Mode::BOOT, 0); } -EiveSystem satsystem::EIVE_SYSTEM = EiveSystem(objects::EIVE_SYSTEM, 12, 24); +EiveSystem satsystem::EIVE_SYSTEM = EiveSystem(objects::EIVE_SYSTEM, 12, 24, I2C_FATAL_ERRORS); auto EIVE_SEQUENCE_BOOT = std::make_pair(satsystem::Mode::BOOT, FixedArrayList()); auto EIVE_TABLE_BOOT_TGT = diff --git a/unittest/rebootLogic/src/CoreController.h b/unittest/rebootLogic/src/CoreController.h index 1846c27f..0229a10e 100644 --- a/unittest/rebootLogic/src/CoreController.h +++ b/unittest/rebootLogic/src/CoreController.h @@ -47,11 +47,6 @@ class CoreController { static xsc::Chip CURRENT_CHIP; static xsc::Copy CURRENT_COPY; - static constexpr ActionId_t SWITCH_REBOOT_FILE_HANDLING = 5; - static constexpr ActionId_t RESET_REBOOT_COUNTERS = 6; - static constexpr ActionId_t SWITCH_IMG_LOCK = 7; - static constexpr ActionId_t SET_MAX_REBOOT_CNT = 8; - CoreController(); static void setCurrentBootCopy(xsc::Chip chip, xsc::Copy copy); @@ -76,4 +71,4 @@ class CoreController { SdCardManager* sdcMan = nullptr; RebootFile rebootFile = {}; bool doPerformRebootFileHandling = true; -}; \ No newline at end of file +};