bsp_q7s/core/CoreController.cpp

@@ -31,8 +31,9 @@
 xsc::Chip CoreController::CURRENT_CHIP = xsc::Chip::NO_CHIP;
 xsc::Copy CoreController::CURRENT_COPY = xsc::Copy::NO_COPY;
 
-CoreController::CoreController(object_id_t objectId)
-    : ExtendedControllerBase(objectId, 5), opDivider5(5), opDivider10(10), hkSet(this) {
+CoreController::CoreController(object_id_t objectId, const std::atomic_uint16_t& i2cErrors)
+    : ExtendedControllerBase(objectId, 5), opDivider5(5), opDivider10(10), hkSet(this),
+      i2cErrors(i2cErrors) {
   try {
     sdcMan = SdCardManager::instance();
     if (sdcMan == nullptr) {
@@ -100,6 +101,12 @@ void CoreController::performControlOperation() {
   sdStateMachine();
   performMountedSdCardOperations();
   readHkData();
+  if(i2cErrors >= 5) {
+    bool protOpPerformed = false;
+    triggerEvent(I2C_UNAVAILABLE_REBOOT);
+    gracefulShutdownTasks(CURRENT_CHIP, CURRENT_COPY, protOpPerformed);
+    std::system("xsc_boot_copy -r");
+  }
   opDivider5.checkAndIncrement();
   opDivider10.checkAndIncrement();
 }

bsp_q7s/core/CoreController.h

@@ -13,6 +13,7 @@
 #include "fsfw/controller/ExtendedControllerBase.h"
 #include "mission/devices/devicedefinitions/GPSDefinitions.h"
 #include "mission/trace.h"
+#include <atomic>
 
 class Timer;
 class SdCardManager;
@@ -127,8 +128,9 @@ class CoreController : public ExtendedControllerBase {
   //! P1: First 16 bits boot count of image 0 0, last 16 bits boot count of image 0 1.
   //! P2: First 16 bits boot count of image 1 0, last 16 bits boot count of image 1 1.
   static constexpr Event INDIVIDUAL_BOOT_COUNTS = event::makeEvent(SUBSYSTEM_ID, 8, severity::INFO);
+  static constexpr Event I2C_UNAVAILABLE_REBOOT = event::makeEvent(SUBSYSTEM_ID, 10, severity::MEDIUM);
 
-  CoreController(object_id_t objectId);
+  CoreController(object_id_t objectId, const std::atomic_uint16_t& i2cErrors);
   virtual ~CoreController();
 
   ReturnValue_t initialize() override;
@@ -251,6 +253,7 @@ class CoreController : public ExtendedControllerBase {
   PoolEntry<float> plVoltageEntry = PoolEntry<float>(0.0);
 
   core::HkSet hkSet;
+  const std::atomic_uint16_t& i2cErrors;
 
 #if OBSW_SD_CARD_MUST_BE_ON == 1
   bool remountAttemptFlag = true;

bsp_q7s/core/ObjectFactory.cpp

@@ -120,6 +120,7 @@
 
 ResetArgs RESET_ARGS_GNSS;
 std::atomic_bool LINK_STATE = CcsdsIpCoreHandler::LINK_DOWN;
+std::atomic_uint16_t I2C_FATAL_ERRORS = 0;
 
 void Factory::setStaticFrameworkObjectIds() {
   PusServiceBase::PUS_DISTRIBUTOR = objects::PUS_PACKET_DISTRIBUTOR;
@@ -946,7 +947,7 @@ void ObjectFactory::createImtqComponents(PowerSwitchIF* pwrSwitcher) {
   auto* imtqAssy = new ImtqAssembly(objects::IMTQ_ASSY);
   imtqAssy->connectModeTreeParent(satsystem::acs::ACS_SUBSYSTEM);
 
-  new ImtqPollingTask(objects::IMTQ_POLLING);
+  new ImtqPollingTask(objects::IMTQ_POLLING, I2C_FATAL_ERRORS);
   I2cCookie* imtqI2cCookie = new I2cCookie(addresses::IMTQ, imtq::MAX_REPLY_SIZE, q7s::I2C_PL_EIVE);
   auto imtqHandler = new ImtqHandler(objects::IMTQ_HANDLER, objects::IMTQ_POLLING, imtqI2cCookie,
                                      pcdu::Switches::PDU1_CH3_MGT_5V);

bsp_q7s/core/ObjectFactory.h

@@ -12,6 +12,7 @@
 #include <mission/tmtc/PusTmFunnel.h>
 
 #include <string>
+#include <atomic>
 
 class LinuxLibgpioIF;
 class SerialComIF;
@@ -22,6 +23,8 @@ class HealthTableIF;
 class AcsBoardAssembly;
 class GpioIF;
 
+extern std::atomic_uint16_t I2C_FATAL_ERRORS;
+
 namespace ObjectFactory {
 
 struct CcsdsComponentArgs {

bsp_q7s/fmObjectFactory.cpp

@@ -38,7 +38,7 @@ void ObjectFactory::produce(void* args) {
   q7s::gpioCallbacks::initSpiCsDecoder(gpioComIF);
   gpioCallbacks::disableAllDecoder(gpioComIF);
 
-  new CoreController(objects::CORE_CONTROLLER);
+  new CoreController(objects::CORE_CONTROLLER, I2C_FATAL_ERRORS);
   createPcduComponents(gpioComIF, &pwrSwitcher);
   auto* stackHandler = new Stack5VHandler(*pwrSwitcher);
 

linux/devices/ImtqPollingTask.cpp

@@ -10,7 +10,9 @@
 
 #include "fsfw/FSFW.h"
 
-ImtqPollingTask::ImtqPollingTask(object_id_t imtqPollingTask) : SystemObject(imtqPollingTask) {
+ImtqPollingTask::ImtqPollingTask(object_id_t imtqPollingTask,
+    std::atomic_uint16_t& i2cFatalErrors): SystemObject(imtqPollingTask),
+    i2cFatalErrors(i2cFatalErrors) {
   semaphore = SemaphoreFactory::instance()->createBinarySemaphore();
   semaphore->acquire();
   ipcLock = MutexFactory::instance()->createMutex();
@@ -427,12 +429,20 @@ ReturnValue_t ImtqPollingTask::performI2cFullRequest(uint8_t* reply, size_t repl
     if (ioctl(fd, I2C_SLAVE, i2cAddr) < 0) {
       sif::warning << "Opening IMTQ slave device failed with code " << errno << ": "
                    << strerror(errno) << std::endl;
+      if(errno == EBUSY) {
+        i2cFatalErrors++;
+      }
     }
 
     int written = write(fd, cmdBuf.data(), cmdLen);
     if (written < 0) {
       sif::error << "IMTQ: Failed to send with error code " << errno
                  << ". Error description: " << strerror(errno) << std::endl;
+      // This is a weird issue which sometimes occurs on debug builds. All I2C buses are busy
+      // for all writes,
+      if(errno == EBUSY) {
+        i2cFatalErrors++;
+      }
       return returnvalue::FAILED;
     } else if (static_cast<size_t>(written) != cmdLen) {
       sif::error << "IMTQ: Could not write all bytes" << std::endl;

linux/devices/ImtqPollingTask.h

@@ -8,12 +8,13 @@
 #include "fsfw/objectmanager/SystemObject.h"
 #include "fsfw/tasks/ExecutableObjectIF.h"
 #include "mission/devices/devicedefinitions/imtqHelpers.h"
+#include <atomic>
 
 class ImtqPollingTask : public SystemObject,
                         public ExecutableObjectIF,
                         public DeviceCommunicationIF {
  public:
-  ImtqPollingTask(object_id_t imtqPollingTask);
+  ImtqPollingTask(object_id_t imtqPollingTask, std::atomic_uint16_t& i2cFatalErrors);
 
   ReturnValue_t performOperation(uint8_t operationCode) override;
   ReturnValue_t initialize() override;
@@ -28,6 +29,7 @@ class ImtqPollingTask : public SystemObject,
   ReturnValue_t comStatus = returnvalue::OK;
   MutexIF* ipcLock;
   MutexIF* bufLock;
+  std::atomic_uint16_t& i2cFatalErrors;
   I2cCookie* i2cCookie = nullptr;
   const char* i2cDev = nullptr;
   address_t i2cAddr = 0;

tmtc

@@ -1 +1 @@
-Subproject commit a40c881b9fc292fe598204280db38720a784b71f
+Subproject commit c99a0701d29a5d5561bc08a95db23cdb696ade5b