From f62b312d3c0d6eab1bf8b576e0989b541f109159 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Mon, 13 Mar 2023 09:49:35 +0100 Subject: [PATCH 1/3] add i2c fatal error counter --- bsp_q7s/core/CoreController.cpp | 11 +++++++++-- bsp_q7s/core/CoreController.h | 5 ++++- bsp_q7s/core/ObjectFactory.cpp | 3 ++- bsp_q7s/core/ObjectFactory.h | 3 +++ bsp_q7s/fmObjectFactory.cpp | 2 +- linux/devices/ImtqPollingTask.cpp | 12 +++++++++++- linux/devices/ImtqPollingTask.h | 4 +++- tmtc | 2 +- 8 files changed, 34 insertions(+), 8 deletions(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 5c4710d4..e9ee9047 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -31,8 +31,9 @@ xsc::Chip CoreController::CURRENT_CHIP = xsc::Chip::NO_CHIP; xsc::Copy CoreController::CURRENT_COPY = xsc::Copy::NO_COPY; -CoreController::CoreController(object_id_t objectId) - : ExtendedControllerBase(objectId, 5), opDivider5(5), opDivider10(10), hkSet(this) { +CoreController::CoreController(object_id_t objectId, const std::atomic_uint16_t& i2cErrors) + : ExtendedControllerBase(objectId, 5), opDivider5(5), opDivider10(10), hkSet(this), + i2cErrors(i2cErrors) { try { sdcMan = SdCardManager::instance(); if (sdcMan == nullptr) { @@ -100,6 +101,12 @@ void CoreController::performControlOperation() { sdStateMachine(); performMountedSdCardOperations(); readHkData(); + if(i2cErrors >= 5) { + bool protOpPerformed = false; + triggerEvent(I2C_UNAVAILABLE_REBOOT); + gracefulShutdownTasks(CURRENT_CHIP, CURRENT_COPY, protOpPerformed); + std::system("xsc_boot_copy -r"); + } opDivider5.checkAndIncrement(); opDivider10.checkAndIncrement(); } diff --git a/bsp_q7s/core/CoreController.h b/bsp_q7s/core/CoreController.h index c5e23d48..39a002b7 100644 --- a/bsp_q7s/core/CoreController.h +++ b/bsp_q7s/core/CoreController.h @@ -13,6 +13,7 @@ #include "fsfw/controller/ExtendedControllerBase.h" #include "mission/devices/devicedefinitions/GPSDefinitions.h" #include "mission/trace.h" +#include class Timer; class SdCardManager; @@ -127,8 +128,9 @@ class CoreController : public ExtendedControllerBase { //! P1: First 16 bits boot count of image 0 0, last 16 bits boot count of image 0 1. //! P2: First 16 bits boot count of image 1 0, last 16 bits boot count of image 1 1. static constexpr Event INDIVIDUAL_BOOT_COUNTS = event::makeEvent(SUBSYSTEM_ID, 8, severity::INFO); + static constexpr Event I2C_UNAVAILABLE_REBOOT = event::makeEvent(SUBSYSTEM_ID, 10, severity::MEDIUM); - CoreController(object_id_t objectId); + CoreController(object_id_t objectId, const std::atomic_uint16_t& i2cErrors); virtual ~CoreController(); ReturnValue_t initialize() override; @@ -251,6 +253,7 @@ class CoreController : public ExtendedControllerBase { PoolEntry plVoltageEntry = PoolEntry(0.0); core::HkSet hkSet; + const std::atomic_uint16_t& i2cErrors; #if OBSW_SD_CARD_MUST_BE_ON == 1 bool remountAttemptFlag = true; diff --git a/bsp_q7s/core/ObjectFactory.cpp b/bsp_q7s/core/ObjectFactory.cpp index 3092e69d..156dd084 100644 --- a/bsp_q7s/core/ObjectFactory.cpp +++ b/bsp_q7s/core/ObjectFactory.cpp @@ -120,6 +120,7 @@ ResetArgs RESET_ARGS_GNSS; std::atomic_bool LINK_STATE = CcsdsIpCoreHandler::LINK_DOWN; +std::atomic_uint16_t I2C_FATAL_ERRORS = 0; void Factory::setStaticFrameworkObjectIds() { PusServiceBase::PUS_DISTRIBUTOR = objects::PUS_PACKET_DISTRIBUTOR; @@ -946,7 +947,7 @@ void ObjectFactory::createImtqComponents(PowerSwitchIF* pwrSwitcher) { auto* imtqAssy = new ImtqAssembly(objects::IMTQ_ASSY); imtqAssy->connectModeTreeParent(satsystem::acs::ACS_SUBSYSTEM); - new ImtqPollingTask(objects::IMTQ_POLLING); + new ImtqPollingTask(objects::IMTQ_POLLING, I2C_FATAL_ERRORS); I2cCookie* imtqI2cCookie = new I2cCookie(addresses::IMTQ, imtq::MAX_REPLY_SIZE, q7s::I2C_PL_EIVE); auto imtqHandler = new ImtqHandler(objects::IMTQ_HANDLER, objects::IMTQ_POLLING, imtqI2cCookie, pcdu::Switches::PDU1_CH3_MGT_5V); diff --git a/bsp_q7s/core/ObjectFactory.h b/bsp_q7s/core/ObjectFactory.h index 7b713005..5779f6c0 100644 --- a/bsp_q7s/core/ObjectFactory.h +++ b/bsp_q7s/core/ObjectFactory.h @@ -12,6 +12,7 @@ #include #include +#include class LinuxLibgpioIF; class SerialComIF; @@ -22,6 +23,8 @@ class HealthTableIF; class AcsBoardAssembly; class GpioIF; +extern std::atomic_uint16_t I2C_FATAL_ERRORS; + namespace ObjectFactory { struct CcsdsComponentArgs { diff --git a/bsp_q7s/fmObjectFactory.cpp b/bsp_q7s/fmObjectFactory.cpp index 34a7c9dc..8d157afa 100644 --- a/bsp_q7s/fmObjectFactory.cpp +++ b/bsp_q7s/fmObjectFactory.cpp @@ -38,7 +38,7 @@ void ObjectFactory::produce(void* args) { q7s::gpioCallbacks::initSpiCsDecoder(gpioComIF); gpioCallbacks::disableAllDecoder(gpioComIF); - new CoreController(objects::CORE_CONTROLLER); + new CoreController(objects::CORE_CONTROLLER, I2C_FATAL_ERRORS); createPcduComponents(gpioComIF, &pwrSwitcher); auto* stackHandler = new Stack5VHandler(*pwrSwitcher); diff --git a/linux/devices/ImtqPollingTask.cpp b/linux/devices/ImtqPollingTask.cpp index fbd5f847..a0e59412 100644 --- a/linux/devices/ImtqPollingTask.cpp +++ b/linux/devices/ImtqPollingTask.cpp @@ -10,7 +10,9 @@ #include "fsfw/FSFW.h" -ImtqPollingTask::ImtqPollingTask(object_id_t imtqPollingTask) : SystemObject(imtqPollingTask) { +ImtqPollingTask::ImtqPollingTask(object_id_t imtqPollingTask, + std::atomic_uint16_t& i2cFatalErrors): SystemObject(imtqPollingTask), + i2cFatalErrors(i2cFatalErrors) { semaphore = SemaphoreFactory::instance()->createBinarySemaphore(); semaphore->acquire(); ipcLock = MutexFactory::instance()->createMutex(); @@ -427,12 +429,20 @@ ReturnValue_t ImtqPollingTask::performI2cFullRequest(uint8_t* reply, size_t repl if (ioctl(fd, I2C_SLAVE, i2cAddr) < 0) { sif::warning << "Opening IMTQ slave device failed with code " << errno << ": " << strerror(errno) << std::endl; + if(errno == EBUSY) { + i2cFatalErrors++; + } } int written = write(fd, cmdBuf.data(), cmdLen); if (written < 0) { sif::error << "IMTQ: Failed to send with error code " << errno << ". Error description: " << strerror(errno) << std::endl; + // This is a weird issue which sometimes occurs on debug builds. All I2C buses are busy + // for all writes, + if(errno == EBUSY) { + i2cFatalErrors++; + } return returnvalue::FAILED; } else if (static_cast(written) != cmdLen) { sif::error << "IMTQ: Could not write all bytes" << std::endl; diff --git a/linux/devices/ImtqPollingTask.h b/linux/devices/ImtqPollingTask.h index cb2d3882..efe6a01b 100644 --- a/linux/devices/ImtqPollingTask.h +++ b/linux/devices/ImtqPollingTask.h @@ -8,12 +8,13 @@ #include "fsfw/objectmanager/SystemObject.h" #include "fsfw/tasks/ExecutableObjectIF.h" #include "mission/devices/devicedefinitions/imtqHelpers.h" +#include class ImtqPollingTask : public SystemObject, public ExecutableObjectIF, public DeviceCommunicationIF { public: - ImtqPollingTask(object_id_t imtqPollingTask); + ImtqPollingTask(object_id_t imtqPollingTask, std::atomic_uint16_t& i2cFatalErrors); ReturnValue_t performOperation(uint8_t operationCode) override; ReturnValue_t initialize() override; @@ -28,6 +29,7 @@ class ImtqPollingTask : public SystemObject, ReturnValue_t comStatus = returnvalue::OK; MutexIF* ipcLock; MutexIF* bufLock; + std::atomic_uint16_t& i2cFatalErrors; I2cCookie* i2cCookie = nullptr; const char* i2cDev = nullptr; address_t i2cAddr = 0; diff --git a/tmtc b/tmtc index a40c881b..c99a0701 160000 --- a/tmtc +++ b/tmtc @@ -1 +1 @@ -Subproject commit a40c881b9fc292fe598204280db38720a784b71f +Subproject commit c99a0701d29a5d5561bc08a95db23cdb696ade5b From 8843a0d8c587730386e1c3e586e9ce49a92c4a8e Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Mon, 20 Mar 2023 11:48:23 +0100 Subject: [PATCH 2/3] resolve merge conflict --- bsp_q7s/core/CoreController.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bsp_q7s/core/CoreController.cpp b/bsp_q7s/core/CoreController.cpp index 2f7722f2..61e7ee67 100644 --- a/bsp_q7s/core/CoreController.cpp +++ b/bsp_q7s/core/CoreController.cpp @@ -108,13 +108,12 @@ void CoreController::performControlOperation() { sdStateMachine(); performMountedSdCardOperations(); readHkData(); -<<<<<<< HEAD if(i2cErrors >= 5) { bool protOpPerformed = false; triggerEvent(I2C_UNAVAILABLE_REBOOT); gracefulShutdownTasks(CURRENT_CHIP, CURRENT_COPY, protOpPerformed); std::system("xsc_boot_copy -r"); -======= + } if (shellCmdIsExecuting) { bool replyReceived = false; // TODO: We could read the data in the ring buffer and send it as an action data reply. @@ -127,7 +126,6 @@ void CoreController::performControlOperation() { } successRecipient = MessageQueueIF::NO_QUEUE; } ->>>>>>> origin/develop } opDivider5.checkAndIncrement(); opDivider10.checkAndIncrement(); From bb27700cc2ad771e9d05df23f8483c7b5f523b28 Mon Sep 17 00:00:00 2001 From: Robin Mueller Date: Mon, 20 Mar 2023 18:02:59 +0100 Subject: [PATCH 3/3] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3727efef..b855f3fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ will consitute of a breaking change warranting a new major release: # [unreleased] +## Added + +- Contingency handling for non-working I2C bus bug. Reboot the system if the I2C is not working. + ## Fixed - Fixed transition for dual power lane assemblies: When going from dual side submode to single side