Merge pull request 'Relax SUS FDIR' (#677) from relax-sus-fdir into main

Reviewed-on: #677
Reviewed-by: Marius Eggert <eggertm@irs.uni-stuttgart.de>

CHANGELOG.md

@@ -18,6 +18,9 @@ will consitute of a breaking change warranting a new major release:
 
 ## Changed
 
+- Relaxed SUS FDIR. The devices have shown to be glitchy in orbit, but still seem to deliver
+  sensible raw values most of the time. Some further testing is necessary, but some changes in the
+  code should cause the SUS devices to remain healthy for now.
 - The primary and the secondary temperature sensors for the PLOC mission boards are exchanged.
 - ACS parameters for the SUSMGM (FLP) safe mode have been adjusted. This safe mode is now the
   default one.

bsp_hosted/fsfwconfig/events/translateEvents.cpp

@@ -1,7 +1,7 @@
 /**
- * @brief    Auto-generated event translation file. Contains 304 translations.
+ * @brief    Auto-generated event translation file. Contains 306 translations.
  * @details
- * Generated on: 2023-08-21 15:54:29
+ * Generated on: 2023-09-12 10:05:17
  */
 #include "translateEvents.h"
 
@@ -309,6 +309,8 @@ const char *DUMP_NOK_CANCELLED_STRING = "DUMP_NOK_CANCELLED";
 const char *DUMP_MISC_CANCELLED_STRING = "DUMP_MISC_CANCELLED";
 const char *DUMP_HK_CANCELLED_STRING = "DUMP_HK_CANCELLED";
 const char *DUMP_CFDP_CANCELLED_STRING = "DUMP_CFDP_CANCELLED";
+const char *TEMPERATURE_ALL_ONES_START_STRING = "TEMPERATURE_ALL_ONES_START";
+const char *TEMPERATURE_ALL_ONES_RECOVERY_STRING = "TEMPERATURE_ALL_ONES_RECOVERY";
 
 const char *translateEvents(Event event) {
   switch ((event & 0xFFFF)) {
@@ -920,6 +922,10 @@ const char *translateEvents(Event event) {
       return DUMP_HK_CANCELLED_STRING;
     case (14314):
       return DUMP_CFDP_CANCELLED_STRING;
+    case (14500):
+      return TEMPERATURE_ALL_ONES_START_STRING;
+    case (14501):
+      return TEMPERATURE_ALL_ONES_RECOVERY_STRING;
     default:
       return "UNKNOWN_EVENT";
   }

bsp_hosted/fsfwconfig/objects/translateObjects.cpp

@@ -2,7 +2,7 @@
  * @brief  Auto-generated object translation file.
  * @details
  * Contains 171 translations.
- * Generated on: 2023-08-21 15:54:29
+ * Generated on: 2023-09-12 10:05:17
  */
 #include "translateObjects.h"
 

common/config/eive/eventSubsystemIds.h

@@ -40,6 +40,7 @@ enum : uint8_t {
   COM_SUBSYSTEM = 142,
   PERSISTENT_TM_STORE = 143,
   SYRLINKS_COM = 144,
+  SUS_HANDLER = 145,
   COMMON_SUBSYSTEM_ID_END
 
 };

generators/bsp_hosted_events.csv

@@ -303,3 +303,5 @@ Event ID (dec); Event ID (hex); Name; Severity; Description; File Path
 14312;0x37e8;DUMP_MISC_CANCELLED;LOW;P1: Number of dumped packets. P2: Total dumped bytes.;mission/persistentTmStoreDefs.h
 14313;0x37e9;DUMP_HK_CANCELLED;LOW;P1: Number of dumped packets. P2: Total dumped bytes.;mission/persistentTmStoreDefs.h
 14314;0x37ea;DUMP_CFDP_CANCELLED;LOW;P1: Number of dumped packets. P2: Total dumped bytes.;mission/persistentTmStoreDefs.h
+14500;0x38a4;TEMPERATURE_ALL_ONES_START;MEDIUM;Detected invalid values, starting invalid message counting;mission/acs/SusHandler.h
+14501;0x38a5;TEMPERATURE_ALL_ONES_RECOVERY;INFO;Detected valid values again, resetting invalid message counter. P1: Invalid message counter.;mission/acs/SusHandler.h

generators/bsp_hosted_returnvalues.csv

@@ -357,6 +357,7 @@ Full ID (hex); Name; Description; Unique ID; Subsytem Name; File Path
 0x3a02;MQI_Full;No space left for more messages;2;MESSAGE_QUEUE_IF;fsfw/src/fsfw/ipc/MessageQueueIF.h
 0x3a03;MQI_NoReplyPartner;Returned if a reply method was called without partner;3;MESSAGE_QUEUE_IF;fsfw/src/fsfw/ipc/MessageQueueIF.h
 0x3a04;MQI_DestinationInvalid;Returned if the target destination is invalid.;4;MESSAGE_QUEUE_IF;fsfw/src/fsfw/ipc/MessageQueueIF.h
+0x3b00;SPH_ConnBroken;No description;0;SEMAPHORE_IF;fsfw/src/fsfw/osal/common/TcpTmTcServer.h
 0x3b01;SPH_SemaphoreTimeout;No description;1;SEMAPHORE_IF;fsfw/src/fsfw/tasks/SemaphoreIF.h
 0x3b02;SPH_SemaphoreNotOwned;No description;2;SEMAPHORE_IF;fsfw/src/fsfw/tasks/SemaphoreIF.h
 0x3b03;SPH_SemaphoreInvalid;No description;3;SEMAPHORE_IF;fsfw/src/fsfw/tasks/SemaphoreIF.h
@@ -372,7 +373,6 @@ Full ID (hex); Name; Description; Unique ID; Subsytem Name; File Path
 0x3e05;HKM_DatasetNotFound;No description;5;HOUSEKEEPING_MANAGER;fsfw/src/fsfw/datapoollocal/LocalDataPoolManager.h
 0x3f01;DLEE_StreamTooShort;No description;1;DLE_ENCODER;fsfw/src/fsfw/globalfunctions/DleEncoder.h
 0x3f02;DLEE_DecodingError;No description;2;DLE_ENCODER;fsfw/src/fsfw/globalfunctions/DleEncoder.h
-0x4100;PUS9_ConnBroken;No description;0;PUS_SERVICE_9;fsfw/src/fsfw/osal/common/TcpTmTcServer.h
 0x4201;PUS11_InvalidTypeTimeWindow;No description;1;PUS_SERVICE_11;fsfw/src/fsfw/pus/Service11TelecommandScheduling.h
 0x4202;PUS11_InvalidTimeWindow;No description;2;PUS_SERVICE_11;fsfw/src/fsfw/pus/Service11TelecommandScheduling.h
 0x4203;PUS11_TimeshiftingNotPossible;No description;3;PUS_SERVICE_11;fsfw/src/fsfw/pus/Service11TelecommandScheduling.h
@@ -402,9 +402,9 @@ Full ID (hex); Name; Description; Unique ID; Subsytem Name; File Path
 0x4403;UXOS_CommandError;Command execution failed;3;LINUX_OSAL;fsfw/src/fsfw_hal/linux/CommandExecutor.h
 0x4404;UXOS_NoCommandLoadedOrPending;;4;LINUX_OSAL;fsfw/src/fsfw_hal/linux/CommandExecutor.h
 0x4406;UXOS_PcloseCallError;No description;6;LINUX_OSAL;fsfw/src/fsfw_hal/linux/CommandExecutor.h
-0x4500;HSPI_HalTimeoutRetval;No description;0;HAL_SPI;fsfw/src/fsfw_hal/stm32h7/spi/spiDefinitions.h
-0x4501;HSPI_HalBusyRetval;No description;1;HAL_SPI;fsfw/src/fsfw_hal/stm32h7/spi/spiDefinitions.h
-0x4502;HSPI_HalErrorRetval;No description;2;HAL_SPI;fsfw/src/fsfw_hal/stm32h7/spi/spiDefinitions.h
+0x4500;HSPI_OpeningFileFailed;No description;0;HAL_SPI;fsfw/src/fsfw_hal/common/spi/spiCommon.h
+0x4501;HSPI_FullDuplexTransferFailed;No description;1;HAL_SPI;fsfw/src/fsfw_hal/common/spi/spiCommon.h
+0x4502;HSPI_HalfDuplexTransferFailed;No description;2;HAL_SPI;fsfw/src/fsfw_hal/common/spi/spiCommon.h
 0x4503;HSPI_Timeout;No description;3;HAL_SPI;fsfw/src/fsfw_hal/common/spi/spiCommon.h
 0x4504;HSPI_Busy;No description;4;HAL_SPI;fsfw/src/fsfw_hal/common/spi/spiCommon.h
 0x4505;HSPI_GenericError;No description;5;HAL_SPI;fsfw/src/fsfw_hal/common/spi/spiCommon.h

generators/bsp_hosted_subsystems.csv

@@ -60,3 +60,4 @@
 142;COM_SUBSYSTEM
 143;PERSISTENT_TM_STORE
 144;SYRLINKS_COM
+145;SUS_HANDLER

generators/bsp_q7s_events.csv

@@ -303,3 +303,5 @@ Event ID (dec); Event ID (hex); Name; Severity; Description; File Path
 14312;0x37e8;DUMP_MISC_CANCELLED;LOW;P1: Number of dumped packets. P2: Total dumped bytes.;mission/persistentTmStoreDefs.h
 14313;0x37e9;DUMP_HK_CANCELLED;LOW;P1: Number of dumped packets. P2: Total dumped bytes.;mission/persistentTmStoreDefs.h
 14314;0x37ea;DUMP_CFDP_CANCELLED;LOW;P1: Number of dumped packets. P2: Total dumped bytes.;mission/persistentTmStoreDefs.h
+14500;0x38a4;TEMPERATURE_ALL_ONES_START;MEDIUM;Detected invalid values, starting invalid message counting;mission/acs/SusHandler.h
+14501;0x38a5;TEMPERATURE_ALL_ONES_RECOVERY;INFO;Detected valid values again, resetting invalid message counter. P1: Invalid message counter.;mission/acs/SusHandler.h

generators/bsp_q7s_returnvalues.csv

@@ -357,6 +357,7 @@ Full ID (hex); Name; Description; Unique ID; Subsytem Name; File Path
 0x3a02;MQI_Full;No space left for more messages;2;MESSAGE_QUEUE_IF;fsfw/src/fsfw/ipc/MessageQueueIF.h
 0x3a03;MQI_NoReplyPartner;Returned if a reply method was called without partner;3;MESSAGE_QUEUE_IF;fsfw/src/fsfw/ipc/MessageQueueIF.h
 0x3a04;MQI_DestinationInvalid;Returned if the target destination is invalid.;4;MESSAGE_QUEUE_IF;fsfw/src/fsfw/ipc/MessageQueueIF.h
+0x3b00;SPH_ConnBroken;No description;0;SEMAPHORE_IF;fsfw/src/fsfw/osal/common/TcpTmTcServer.h
 0x3b01;SPH_SemaphoreTimeout;No description;1;SEMAPHORE_IF;fsfw/src/fsfw/tasks/SemaphoreIF.h
 0x3b02;SPH_SemaphoreNotOwned;No description;2;SEMAPHORE_IF;fsfw/src/fsfw/tasks/SemaphoreIF.h
 0x3b03;SPH_SemaphoreInvalid;No description;3;SEMAPHORE_IF;fsfw/src/fsfw/tasks/SemaphoreIF.h
@@ -372,7 +373,6 @@ Full ID (hex); Name; Description; Unique ID; Subsytem Name; File Path
 0x3e05;HKM_DatasetNotFound;No description;5;HOUSEKEEPING_MANAGER;fsfw/src/fsfw/datapoollocal/LocalDataPoolManager.h
 0x3f01;DLEE_StreamTooShort;No description;1;DLE_ENCODER;fsfw/src/fsfw/globalfunctions/DleEncoder.h
 0x3f02;DLEE_DecodingError;No description;2;DLE_ENCODER;fsfw/src/fsfw/globalfunctions/DleEncoder.h
-0x4100;PUS9_ConnBroken;No description;0;PUS_SERVICE_9;fsfw/src/fsfw/osal/common/TcpTmTcServer.h
 0x4201;PUS11_InvalidTypeTimeWindow;No description;1;PUS_SERVICE_11;fsfw/src/fsfw/pus/Service11TelecommandScheduling.h
 0x4202;PUS11_InvalidTimeWindow;No description;2;PUS_SERVICE_11;fsfw/src/fsfw/pus/Service11TelecommandScheduling.h
 0x4203;PUS11_TimeshiftingNotPossible;No description;3;PUS_SERVICE_11;fsfw/src/fsfw/pus/Service11TelecommandScheduling.h
@@ -402,9 +402,9 @@ Full ID (hex); Name; Description; Unique ID; Subsytem Name; File Path
 0x4403;UXOS_CommandError;Command execution failed;3;LINUX_OSAL;fsfw/src/fsfw_hal/linux/CommandExecutor.h
 0x4404;UXOS_NoCommandLoadedOrPending;;4;LINUX_OSAL;fsfw/src/fsfw_hal/linux/CommandExecutor.h
 0x4406;UXOS_PcloseCallError;No description;6;LINUX_OSAL;fsfw/src/fsfw_hal/linux/CommandExecutor.h
-0x4500;HSPI_HalTimeoutRetval;No description;0;HAL_SPI;fsfw/src/fsfw_hal/stm32h7/spi/spiDefinitions.h
-0x4501;HSPI_HalBusyRetval;No description;1;HAL_SPI;fsfw/src/fsfw_hal/stm32h7/spi/spiDefinitions.h
-0x4502;HSPI_HalErrorRetval;No description;2;HAL_SPI;fsfw/src/fsfw_hal/stm32h7/spi/spiDefinitions.h
+0x4500;HSPI_OpeningFileFailed;No description;0;HAL_SPI;fsfw/src/fsfw_hal/common/spi/spiCommon.h
+0x4501;HSPI_FullDuplexTransferFailed;No description;1;HAL_SPI;fsfw/src/fsfw_hal/common/spi/spiCommon.h
+0x4502;HSPI_HalfDuplexTransferFailed;No description;2;HAL_SPI;fsfw/src/fsfw_hal/common/spi/spiCommon.h
 0x4503;HSPI_Timeout;No description;3;HAL_SPI;fsfw/src/fsfw_hal/common/spi/spiCommon.h
 0x4504;HSPI_Busy;No description;4;HAL_SPI;fsfw/src/fsfw_hal/common/spi/spiCommon.h
 0x4505;HSPI_GenericError;No description;5;HAL_SPI;fsfw/src/fsfw_hal/common/spi/spiCommon.h

generators/bsp_q7s_subsystems.csv

@@ -60,3 +60,4 @@
 142;COM_SUBSYSTEM
 143;PERSISTENT_TM_STORE
 144;SYRLINKS_COM
+145;SUS_HANDLER

generators/events/translateEvents.cpp

@@ -1,7 +1,7 @@
 /**
- * @brief    Auto-generated event translation file. Contains 304 translations.
+ * @brief    Auto-generated event translation file. Contains 306 translations.
  * @details
- * Generated on: 2023-08-21 15:54:29
+ * Generated on: 2023-09-12 10:05:17
  */
 #include "translateEvents.h"
 
@@ -309,6 +309,8 @@ const char *DUMP_NOK_CANCELLED_STRING = "DUMP_NOK_CANCELLED";
 const char *DUMP_MISC_CANCELLED_STRING = "DUMP_MISC_CANCELLED";
 const char *DUMP_HK_CANCELLED_STRING = "DUMP_HK_CANCELLED";
 const char *DUMP_CFDP_CANCELLED_STRING = "DUMP_CFDP_CANCELLED";
+const char *TEMPERATURE_ALL_ONES_START_STRING = "TEMPERATURE_ALL_ONES_START";
+const char *TEMPERATURE_ALL_ONES_RECOVERY_STRING = "TEMPERATURE_ALL_ONES_RECOVERY";
 
 const char *translateEvents(Event event) {
   switch ((event & 0xFFFF)) {
@@ -920,6 +922,10 @@ const char *translateEvents(Event event) {
       return DUMP_HK_CANCELLED_STRING;
     case (14314):
       return DUMP_CFDP_CANCELLED_STRING;
+    case (14500):
+      return TEMPERATURE_ALL_ONES_START_STRING;
+    case (14501):
+      return TEMPERATURE_ALL_ONES_RECOVERY_STRING;
     default:
       return "UNKNOWN_EVENT";
   }

generators/objects/translateObjects.cpp

@@ -2,7 +2,7 @@
  * @brief  Auto-generated object translation file.
  * @details
  * Contains 175 translations.
- * Generated on: 2023-08-21 15:54:29
+ * Generated on: 2023-09-12 10:05:17
  */
 #include "translateObjects.h"
 

linux/acs/SusPolling.cpp

@@ -170,11 +170,6 @@ ReturnValue_t SusPolling::handleSusPolling() {
       }
       MutexGuard mg(ipcLock);
       susDevs[idx].ownReply.tempRaw = ((rawReply[0] & 0x0f) << 8) | rawReply[1];
-      // Reply is all ones. Sensor is probably off or faulty when
-      // it should not be.
-      if (susDevs[idx].ownReply.tempRaw == 0x0fff) {
-        susDevs[idx].replyResult = returnvalue::FAILED;
-      } else {
       susDevs[idx].replyResult = returnvalue::OK;
       for (unsigned chIdx = 0; chIdx < 6; chIdx++) {
         susDevs[idx].ownReply.channelsRaw[chIdx] =
@@ -183,7 +178,6 @@ ReturnValue_t SusPolling::handleSusPolling() {
       susDevs[idx].ownReply.dataWasSet = true;
     }
   }
-  }
   return OK;
 }
 

linux/fsfwconfig/events/translateEvents.cpp

@@ -1,7 +1,7 @@
 /**
- * @brief    Auto-generated event translation file. Contains 304 translations.
+ * @brief    Auto-generated event translation file. Contains 306 translations.
  * @details
- * Generated on: 2023-08-21 15:54:29
+ * Generated on: 2023-09-12 10:05:17
  */
 #include "translateEvents.h"
 
@@ -309,6 +309,8 @@ const char *DUMP_NOK_CANCELLED_STRING = "DUMP_NOK_CANCELLED";
 const char *DUMP_MISC_CANCELLED_STRING = "DUMP_MISC_CANCELLED";
 const char *DUMP_HK_CANCELLED_STRING = "DUMP_HK_CANCELLED";
 const char *DUMP_CFDP_CANCELLED_STRING = "DUMP_CFDP_CANCELLED";
+const char *TEMPERATURE_ALL_ONES_START_STRING = "TEMPERATURE_ALL_ONES_START";
+const char *TEMPERATURE_ALL_ONES_RECOVERY_STRING = "TEMPERATURE_ALL_ONES_RECOVERY";
 
 const char *translateEvents(Event event) {
   switch ((event & 0xFFFF)) {
@@ -920,6 +922,10 @@ const char *translateEvents(Event event) {
       return DUMP_HK_CANCELLED_STRING;
     case (14314):
       return DUMP_CFDP_CANCELLED_STRING;
+    case (14500):
+      return TEMPERATURE_ALL_ONES_START_STRING;
+    case (14501):
+      return TEMPERATURE_ALL_ONES_RECOVERY_STRING;
     default:
       return "UNKNOWN_EVENT";
   }

linux/fsfwconfig/objects/translateObjects.cpp

@@ -2,7 +2,7 @@
  * @brief  Auto-generated object translation file.
  * @details
  * Contains 175 translations.
- * Generated on: 2023-08-21 15:54:29
+ * Generated on: 2023-09-12 10:05:17
  */
 #include "translateObjects.h"
 

mission/acs/SusHandler.cpp

@@ -20,6 +20,9 @@ void SusHandler::doStartUp() {
   }
   if (internalState == InternalState::STARTUP) {
     if (commandExecuted) {
+      if (waitingForRecovery) {
+        waitingForRecovery = false;
+      }
       setMode(MODE_ON);
       internalState = InternalState::NONE;
       commandExecuted = false;
@@ -83,15 +86,36 @@ ReturnValue_t SusHandler::scanForReply(const uint8_t *start, size_t len, DeviceC
 }
 ReturnValue_t SusHandler::interpretDeviceReply(DeviceCommandId_t id, const uint8_t *packet) {
   const auto *reply = reinterpret_cast<const acs::SusReply *>(packet);
-  if (reply->dataWasSet) {
+  if (!reply->dataWasSet) {
+    return returnvalue::OK;
+  }
   if (internalState == InternalState::STARTUP) {
     commandExecuted = true;
   }
   PoolReadGuard pg(&dataset);
+  // Simple FDIR variant to make the handler more robust to invalid messages which
+  // appear sometimes for the SUS device: Allow invalid message up to a certain threshold
+  // before triggering FDIR reactions.
+  if (reply->tempRaw == 0xfff and not waitingForRecovery) {
+    if (invalidMsgCounter == 0) {
+      triggerEvent(TEMPERATURE_ALL_ONES_START);
+    } else if (invalidMsgCounter == susMax1227::MAX_INVALID_MSG_COUNT) {
+      triggerEvent(DeviceHandlerIF::DEVICE_WANTS_HARD_REBOOT);
+      waitingForRecovery = true;
+    }
+    invalidMsgCounter++;
+    dataset.setValidity(false, true);
+    dataset.tempC = thermal::INVALID_TEMPERATURE;
+    std::memset(dataset.channels.value, 0, sizeof(dataset.channels.value));
+    return returnvalue::OK;
+  }
+  if (invalidMsgCounter > 0) {
+    triggerEvent(TEMPERATURE_ALL_ONES_RECOVERY, invalidMsgCounter);
+    invalidMsgCounter = 0;
+  }
   dataset.setValidity(true, true);
   dataset.tempC = max1227::getTemperature(reply->tempRaw);
   std::memcpy(dataset.channels.value, reply->channelsRaw, sizeof(reply->channelsRaw));
-  }
   return returnvalue::OK;
 }
 

mission/acs/SusHandler.h

@@ -15,7 +15,15 @@ class SusHandler : public DeviceHandlerBase {
   static constexpr DeviceCommandId_t REPLY = 0x77;
 
   static const uint8_t INTERFACE_ID = CLASS_ID::SUS_HANDLER;
-  static const uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::SUS_BOARD_ASS;
+  static const uint8_t SUBSYSTEM_ID = SUBSYSTEM_ID::SUS_HANDLER;
+
+  //! [EXPORT] : [COMMENT] Detected invalid values, starting invalid message counting
+  static constexpr Event TEMPERATURE_ALL_ONES_START =
+      event::makeEvent(SUBSYSTEM_ID, 0, severity::MEDIUM);
+  //! [EXPORT] : [COMMENT] Detected valid values again, resetting invalid message counter.
+  //! P1: Invalid message counter.
+  static constexpr Event TEMPERATURE_ALL_ONES_RECOVERY =
+      event::makeEvent(SUBSYSTEM_ID, 1, severity::INFO);
 
   SusHandler(uint32_t objectId, uint8_t susIdx, object_id_t deviceCommunication,
              CookieIF *comCookie);
@@ -46,6 +54,8 @@ class SusHandler : public DeviceHandlerBase {
   susMax1227::SusDataset dataset;
   acs::SusRequest request{};
   uint8_t susIdx;
+  bool waitingForRecovery = true;
+  uint32_t invalidMsgCounter = 0;
 
   uint32_t transitionDelay = 1000;
   bool goToNormalMode = false;

mission/acs/susMax1227Helpers.h

@@ -8,6 +8,14 @@
 
 namespace susMax1227 {
 
+// This is 16 seconds for a polling frequency of 0.4 seconds.
+static constexpr uint32_t MAX_INVALID_MSG_COUNT = 40;
+// Using a decrement time of 32 seconds should cause faulty device incrementation to best faster
+// the decrementation, so that FDIR reactions will eventuall be triggered.
+// NOTE: Not used currently, we perform the strange reply check logic in the handler and trigger
+// a reboot directly using the appropriate event.
+static constexpr uint32_t FAULTY_COM_DECREMENT_TIME_MS = 32000;
+
 static const DeviceCommandId_t NONE = 0x0;  // Set when no command is pending
 
 static const DeviceCommandId_t WRITE_SETUP = 1;

mission/system/acs/SusFdir.cpp

@@ -1,6 +1,7 @@
 #include "SusFdir.h"
 
 #include "eive/objects.h"
+#include "mission/acs/susMax1227Helpers.h"
 
 SusFdir::SusFdir(object_id_t sensorId)
     : DeviceHandlerFailureIsolation(sensorId, objects::SUS_BOARD_ASS) {}

tmtc

@@ -1 +1 @@
-Subproject commit b1fbad39e311634084ee3e6a44b6de6ba2ad6407
+Subproject commit e56f8732be52766bf73c1b7e222eff8cef114da5