CHANGELOG.md

@@ -22,7 +22,7 @@ will consitute of a breaking change warranting a new major release:
   the `MEKF` on its own once. This way, there will not be an event spam and operators will have
   to look into the reason of wrong outputs. To restore the reset ability, an action command has
   been added.
-
+- Contingency handling for non-working I2C bus bug. Reboot the system if the I2C is not working.
 
 ## Fixed
 

bsp_q7s/core/CoreController.cpp

@@ -31,14 +31,15 @@
 xsc::Chip CoreController::CURRENT_CHIP = xsc::Chip::NO_CHIP;
 xsc::Copy CoreController::CURRENT_COPY = xsc::Copy::NO_COPY;
 
-CoreController::CoreController(object_id_t objectId)
+CoreController::CoreController(object_id_t objectId, const std::atomic_uint16_t& i2cErrors)
     : ExtendedControllerBase(objectId, 5),
       cmdExecutor(4096),
       cmdReplyBuf(4096, true),
       cmdRepliesSizes(128),
       opDivider5(5),
       opDivider10(10),
-      hkSet(this) {
+      hkSet(this),
+      i2cErrors(i2cErrors) {
   cmdExecutor.setRingBuffer(&cmdReplyBuf, &cmdRepliesSizes);
   try {
     sdcMan = SdCardManager::instance();
@@ -107,6 +108,12 @@ void CoreController::performControlOperation() {
   sdStateMachine();
   performMountedSdCardOperations();
   readHkData();
+  if(i2cErrors >= 5) {
+    bool protOpPerformed = false;
+    triggerEvent(I2C_UNAVAILABLE_REBOOT);
+    gracefulShutdownTasks(CURRENT_CHIP, CURRENT_COPY, protOpPerformed);
+    std::system("xsc_boot_copy -r");
+  }
   if (shellCmdIsExecuting) {
     bool replyReceived = false;
     // TODO: We could read the data in the ring buffer and send it as an action data reply.

bsp_q7s/core/CoreController.h

@@ -15,6 +15,7 @@
 #include "fsfw/controller/ExtendedControllerBase.h"
 #include "mission/devices/devicedefinitions/GPSDefinitions.h"
 #include "mission/trace.h"
+#include <atomic>
 
 class Timer;
 class SdCardManager;
@@ -131,8 +132,9 @@ class CoreController : public ExtendedControllerBase {
   //! P1: First 16 bits boot count of image 0 0, last 16 bits boot count of image 0 1.
   //! P2: First 16 bits boot count of image 1 0, last 16 bits boot count of image 1 1.
   static constexpr Event INDIVIDUAL_BOOT_COUNTS = event::makeEvent(SUBSYSTEM_ID, 8, severity::INFO);
+  static constexpr Event I2C_UNAVAILABLE_REBOOT = event::makeEvent(SUBSYSTEM_ID, 10, severity::MEDIUM);
 
-  CoreController(object_id_t objectId);
+  CoreController(object_id_t objectId, const std::atomic_uint16_t& i2cErrors);
   virtual ~CoreController();
 
   ReturnValue_t initialize() override;
@@ -262,6 +264,7 @@ class CoreController : public ExtendedControllerBase {
   PoolEntry<float> plVoltageEntry = PoolEntry<float>(0.0);
 
   core::HkSet hkSet;
+  const std::atomic_uint16_t& i2cErrors;
 
 #if OBSW_SD_CARD_MUST_BE_ON == 1
   bool remountAttemptFlag = true;

bsp_q7s/core/ObjectFactory.cpp

@@ -123,6 +123,7 @@ using gpio::Levels;
 
 ResetArgs RESET_ARGS_GNSS;
 std::atomic_bool LINK_STATE = CcsdsIpCoreHandler::LINK_DOWN;
+std::atomic_uint16_t I2C_FATAL_ERRORS = 0;
 
 void Factory::setStaticFrameworkObjectIds() {
   PusServiceBase::PUS_DISTRIBUTOR = objects::PUS_PACKET_DISTRIBUTOR;
@@ -953,7 +954,7 @@ void ObjectFactory::createImtqComponents(PowerSwitchIF* pwrSwitcher) {
   auto* imtqAssy = new ImtqAssembly(objects::IMTQ_ASSY);
   imtqAssy->connectModeTreeParent(satsystem::acs::ACS_SUBSYSTEM);
 
-  new ImtqPollingTask(objects::IMTQ_POLLING);
+  new ImtqPollingTask(objects::IMTQ_POLLING, I2C_FATAL_ERRORS);
   I2cCookie* imtqI2cCookie = new I2cCookie(addresses::IMTQ, imtq::MAX_REPLY_SIZE, q7s::I2C_PL_EIVE);
   auto imtqHandler = new ImtqHandler(objects::IMTQ_HANDLER, objects::IMTQ_POLLING, imtqI2cCookie,
                                      pcdu::Switches::PDU1_CH3_MGT_5V);

bsp_q7s/core/ObjectFactory.h

@@ -12,6 +12,7 @@
 #include <mission/tmtc/PusTmFunnel.h>
 
 #include <string>
+#include <atomic>
 
 class LinuxLibgpioIF;
 class SerialComIF;
@@ -22,6 +23,8 @@ class HealthTableIF;
 class AcsBoardAssembly;
 class GpioIF;
 
+extern std::atomic_uint16_t I2C_FATAL_ERRORS;
+
 namespace ObjectFactory {
 
 struct CcsdsComponentArgs {

bsp_q7s/fmObjectFactory.cpp

@@ -38,7 +38,7 @@ void ObjectFactory::produce(void* args) {
   q7s::gpioCallbacks::initSpiCsDecoder(gpioComIF);
   gpioCallbacks::disableAllDecoder(gpioComIF);
 
-  new CoreController(objects::CORE_CONTROLLER);
+  new CoreController(objects::CORE_CONTROLLER, I2C_FATAL_ERRORS);
   createPcduComponents(gpioComIF, &pwrSwitcher);
   auto* stackHandler = new Stack5VHandler(*pwrSwitcher);
 

linux/devices/ImtqPollingTask.cpp

@@ -10,7 +10,9 @@
 
 #include "fsfw/FSFW.h"
 
-ImtqPollingTask::ImtqPollingTask(object_id_t imtqPollingTask) : SystemObject(imtqPollingTask) {
+ImtqPollingTask::ImtqPollingTask(object_id_t imtqPollingTask,
+    std::atomic_uint16_t& i2cFatalErrors): SystemObject(imtqPollingTask),
+    i2cFatalErrors(i2cFatalErrors) {
   semaphore = SemaphoreFactory::instance()->createBinarySemaphore();
   semaphore->acquire();
   ipcLock = MutexFactory::instance()->createMutex();
@@ -427,12 +429,20 @@ ReturnValue_t ImtqPollingTask::performI2cFullRequest(uint8_t* reply, size_t repl
     if (ioctl(fd, I2C_SLAVE, i2cAddr) < 0) {
       sif::warning << "Opening IMTQ slave device failed with code " << errno << ": "
                    << strerror(errno) << std::endl;
+      if(errno == EBUSY) {
+        i2cFatalErrors++;
+      }
     }
 
     int written = write(fd, cmdBuf.data(), cmdLen);
     if (written < 0) {
       sif::error << "IMTQ: Failed to send with error code " << errno
                  << ". Error description: " << strerror(errno) << std::endl;
+      // This is a weird issue which sometimes occurs on debug builds. All I2C buses are busy
+      // for all writes,
+      if(errno == EBUSY) {
+        i2cFatalErrors++;
+      }
       return returnvalue::FAILED;
     } else if (static_cast<size_t>(written) != cmdLen) {
       sif::error << "IMTQ: Could not write all bytes" << std::endl;

linux/devices/ImtqPollingTask.h

@@ -8,12 +8,13 @@
 #include "fsfw/objectmanager/SystemObject.h"
 #include "fsfw/tasks/ExecutableObjectIF.h"
 #include "mission/devices/devicedefinitions/imtqHelpers.h"
+#include <atomic>
 
 class ImtqPollingTask : public SystemObject,
                         public ExecutableObjectIF,
                         public DeviceCommunicationIF {
  public:
-  ImtqPollingTask(object_id_t imtqPollingTask);
+  ImtqPollingTask(object_id_t imtqPollingTask, std::atomic_uint16_t& i2cFatalErrors);
 
   ReturnValue_t performOperation(uint8_t operationCode) override;
   ReturnValue_t initialize() override;
@@ -28,6 +29,7 @@ class ImtqPollingTask : public SystemObject,
   ReturnValue_t comStatus = returnvalue::OK;
   MutexIF* ipcLock;
   MutexIF* bufLock;
+  std::atomic_uint16_t& i2cFatalErrors;
   I2cCookie* i2cCookie = nullptr;
   const char* i2cDev = nullptr;
   address_t i2cAddr = 0;