Merge pull request 'Unit test for Internal Error Reporter' (#563) from gaisser/fsfw:gaisser_unit_test_internal_error into development

Reviewed-on: fsfw/fsfw#563

scripts/helper.py

@@ -143,10 +143,6 @@ def handle_tests_type(args, build_dir_list: list):
         if which("valgrind") is None:
             print("Please install valgrind first")
             sys.exit(1)
-        if os.path.split(os.getcwd())[1] != UNITTEST_FOLDER_NAME:
-            # If we are in a different directory we try to switch into it but
-            # this might fail
-            os.chdir(UNITTEST_FOLDER_NAME)
         cmd_runner("valgrind --leak-check=full ./fsfw-tests")
         os.chdir("..")
 

src/fsfw/datapoollocal/LocalDataPoolManager.cpp

@@ -84,8 +84,8 @@ ReturnValue_t LocalDataPoolManager::initializeHousekeepingPoolEntriesOnce() {
     return result;
   }
 
-  printWarningOrError(sif::OutputTypes::OUT_WARNING, "initialize", HasReturnvaluesIF::RETURN_FAILED,
-                      "The map should only be initialized once");
+  printWarningOrError(sif::OutputTypes::OUT_WARNING, "initializeHousekeepingPoolEntriesOnce",
+                      HasReturnvaluesIF::RETURN_FAILED, "The map should only be initialized once");
   return HasReturnvaluesIF::RETURN_OK;
 }
 

src/fsfw/internalerror/InternalErrorReporter.cpp

@@ -57,6 +57,9 @@ ReturnValue_t InternalErrorReporter::performOperation(uint8_t opCode) {
       internalErrorDataset.storeHits.value += newStoreHits;
       internalErrorDataset.tmHits.value += newTmHits;
       internalErrorDataset.setValidity(true, true);
+      if ((newQueueHits != 0) or (newStoreHits != 0) or (newTmHits != 0)) {
+        internalErrorDataset.setChanged(true);
+      }
     }
   }
 
@@ -77,14 +80,6 @@ uint32_t InternalErrorReporter::getAndResetQueueHits() {
   return value;
 }
 
-uint32_t InternalErrorReporter::getQueueHits() {
-  uint32_t value;
-  mutex->lockMutex(timeoutType, timeoutMs);
-  value = queueHits;
-  mutex->unlockMutex();
-  return value;
-}
-
 void InternalErrorReporter::incrementQueueHits() {
   mutex->lockMutex(timeoutType, timeoutMs);
   queueHits++;
@@ -100,14 +95,6 @@ uint32_t InternalErrorReporter::getAndResetTmHits() {
   return value;
 }
 
-uint32_t InternalErrorReporter::getTmHits() {
-  uint32_t value;
-  mutex->lockMutex(timeoutType, timeoutMs);
-  value = tmHits;
-  mutex->unlockMutex();
-  return value;
-}
-
 void InternalErrorReporter::incrementTmHits() {
   mutex->lockMutex(timeoutType, timeoutMs);
   tmHits++;
@@ -125,14 +112,6 @@ uint32_t InternalErrorReporter::getAndResetStoreHits() {
   return value;
 }
 
-uint32_t InternalErrorReporter::getStoreHits() {
-  uint32_t value;
-  mutex->lockMutex(timeoutType, timeoutMs);
-  value = storeHits;
-  mutex->unlockMutex();
-  return value;
-}
-
 void InternalErrorReporter::incrementStoreHits() {
   mutex->lockMutex(timeoutType, timeoutMs);
   storeHits++;

src/fsfw/internalerror/InternalErrorReporter.h

@@ -46,11 +46,11 @@ class InternalErrorReporter : public SystemObject,
   virtual ReturnValue_t initializeAfterTaskCreation() override;
   virtual ReturnValue_t performOperation(uint8_t opCode) override;
 
-  virtual void queueMessageNotSent();
+  virtual void queueMessageNotSent() override;
 
-  virtual void lostTm();
+  virtual void lostTm() override;
 
-  virtual void storeFull();
+  virtual void storeFull() override;
 
   virtual void setTaskIF(PeriodicTaskIF* task) override;
 
@@ -74,15 +74,12 @@ class InternalErrorReporter : public SystemObject,
   uint32_t storeHits = 0;
 
   uint32_t getAndResetQueueHits();
-  uint32_t getQueueHits();
   void incrementQueueHits();
 
   uint32_t getAndResetTmHits();
-  uint32_t getTmHits();
   void incrementTmHits();
 
   uint32_t getAndResetStoreHits();
-  uint32_t getStoreHits();
   void incrementStoreHits();
 };
 

src/fsfw/internalerror/InternalErrorReporterIF.h

@@ -1,22 +1,34 @@
 #ifndef INTERNALERRORREPORTERIF_H_
 #define INTERNALERRORREPORTERIF_H_
 
+/**
+ * @brief   Interface which is used to report internal errors like full message queues or stores.
+ * @details
+ * This interface smust be used for the InteralErrorReporter object.
+ * It should be used to indicate that there was a Problem with Queues or Stores.
+ *
+ * It can be used to report missing Telemetry which could not be sent due to a internal problem.
+ *
+ */
 class InternalErrorReporterIF {
  public:
   virtual ~InternalErrorReporterIF() {}
-
   /**
-   * Thread safe
+   * @brief Function to be called if a message queue could not be sent.
+   * @details OSAL Implementations should call this function to indicate that
+   * a message was lost.
+   *
+   *  Implementations are required to be Thread safe
    */
   virtual void queueMessageNotSent() = 0;
-
   /**
-   * Thread safe
+   * @brief Function to be called if Telemetry could not be sent
+   * @details Implementations must be Thread safe
    */
   virtual void lostTm() = 0;
-
   /**
-   * Thread safe
+   * @brief Function to be called if a onboard storage is full
+   * @details Implementations must be Thread safe
    */
   virtual void storeFull() = 0;
 };

src/fsfw/osal/host/MessageQueue.cpp

@@ -125,6 +125,13 @@ ReturnValue_t MessageQueue::sendMessageFromMessageQueue(MessageQueueId_t sendTo,
     memcpy(targetQueue->messageQueue.back().data(), message->getBuffer(),
            message->getMaximumMessageSize());
   } else {
+    if (not ignoreFault) {
+      InternalErrorReporterIF* internalErrorReporter =
+          ObjectManager::instance()->get<InternalErrorReporterIF>(objects::INTERNAL_ERROR_REPORTER);
+      if (internalErrorReporter != nullptr) {
+        internalErrorReporter->queueMessageNotSent();
+      }
+    }
     return MessageQueueIF::FULL;
   }
   return HasReturnvaluesIF::RETURN_OK;

tests/src/fsfw_tests/unit/CMakeLists.txt

@@ -23,3 +23,4 @@ add_subdirectory(timemanager)
 add_subdirectory(tmtcpacket)
 add_subdirectory(cfdp)
 add_subdirectory(hal)
+add_subdirectory(internalerror)

tests/src/fsfw_tests/unit/internalerror/CMakeLists.txt

@@ -0,0 +1,3 @@
+target_sources(${FSFW_TEST_TGT} PRIVATE
+	TestInternalErrorReporter.cpp
+)

tests/src/fsfw_tests/unit/internalerror/TestInternalErrorReporter.cpp

@@ -0,0 +1,118 @@
+#include <fsfw/housekeeping/HousekeepingSnapshot.h>
+#include <fsfw/internalerror/InternalErrorReporter.h>
+#include <fsfw/ipc/MessageQueueIF.h>
+#include <fsfw/ipc/QueueFactory.h>
+#include <fsfw/objectmanager/ObjectManager.h>
+#include <fsfw/timemanager/CCSDSTime.h>
+
+#include <array>
+#include <catch2/catch_test_macros.hpp>
+
+#include "fsfw/action/ActionMessage.h"
+#include "fsfw/ipc/CommandMessage.h"
+#include "fsfw/ipc/MessageQueueMessage.h"
+#include "fsfw/objectmanager/frameworkObjects.h"
+#include "fsfw_tests/unit/CatchDefinitions.h"
+#include "fsfw_tests/unit/mocks/PeriodicTaskIFMock.h"
+
+TEST_CASE("Internal Error Reporter", "[TestInternalError]") {
+  PeriodicTaskMock task(10);
+  ObjectManagerIF* manager = ObjectManager::instance();
+  if (manager == nullptr) {
+    FAIL();
+  }
+  InternalErrorReporter* internalErrorReporter = dynamic_cast<InternalErrorReporter*>(
+      ObjectManager::instance()->get<InternalErrorReporterIF>(objects::INTERNAL_ERROR_REPORTER));
+  if (internalErrorReporter == nullptr) {
+    FAIL();
+  }
+  task.addComponent(objects::INTERNAL_ERROR_REPORTER);
+  MessageQueueIF* testQueue = QueueFactory::instance()->createMessageQueue(1);
+  MessageQueueIF* hkQueue = QueueFactory::instance()->createMessageQueue(1);
+  internalErrorReporter->getSubscriptionInterface()->subscribeForSetUpdateMessage(
+      InternalErrorDataset::ERROR_SET_ID, objects::NO_OBJECT, hkQueue->getId(), true);
+  StorageManagerIF* ipcStore = ObjectManager::instance()->get<StorageManagerIF>(objects::IPC_STORE);
+  SECTION("MessageQueueFull") {
+    CommandMessage message;
+    ActionMessage::setCompletionReply(&message, 10, true);
+    auto result = hkQueue->sendMessage(testQueue->getId(), &message);
+    REQUIRE(result == retval::CATCH_OK);
+    uint32_t queueHits = 0;
+    uint32_t lostTm = 0;
+    uint32_t storeHits = 0;
+    /* We don't know if another test caused a queue Hit so we will enforce one,
+     then remeber the queueHit count and force another hit */
+    internalErrorReporter->queueMessageNotSent();
+    internalErrorReporter->performOperation(0);
+    {
+      CommandMessage hkMessage;
+      result = hkQueue->receiveMessage(&hkMessage);
+      REQUIRE(result == HasReturnvaluesIF::RETURN_OK);
+      REQUIRE(hkMessage.getCommand() == HousekeepingMessage::UPDATE_SNAPSHOT_SET);
+      store_address_t storeAddress;
+      gp_id_t gpid =
+          HousekeepingMessage::getUpdateSnapshotVariableCommand(&hkMessage, &storeAddress);
+      REQUIRE(gpid.objectId == objects::INTERNAL_ERROR_REPORTER);
+      // We need the object ID of the reporter here (NO_OBJECT)
+      InternalErrorDataset dataset(objects::INTERNAL_ERROR_REPORTER);
+      CCSDSTime::CDS_short time;
+      ConstAccessorPair data = ipcStore->getData(storeAddress);
+      REQUIRE(data.first == HasReturnvaluesIF::RETURN_OK);
+      HousekeepingSnapshot hkSnapshot(&time, &dataset);
+      const uint8_t* buffer = data.second.data();
+      size_t size = data.second.size();
+      result = hkSnapshot.deSerialize(&buffer, &size, SerializeIF::Endianness::MACHINE);
+      REQUIRE(result == HasReturnvaluesIF::RETURN_OK);
+      // Remember the amount of queueHits before to see the increase
+      queueHits = dataset.queueHits.value;
+      lostTm = dataset.tmHits.value;
+      storeHits = dataset.storeHits.value;
+    }
+    result = hkQueue->sendMessage(testQueue->getId(), &message);
+    REQUIRE(result == MessageQueueIF::FULL);
+    internalErrorReporter->lostTm();
+    internalErrorReporter->storeFull();
+    {
+      internalErrorReporter->performOperation(0);
+      CommandMessage hkMessage;
+      result = hkQueue->receiveMessage(&hkMessage);
+      REQUIRE(result == HasReturnvaluesIF::RETURN_OK);
+      REQUIRE(hkMessage.getCommand() == HousekeepingMessage::UPDATE_SNAPSHOT_SET);
+      store_address_t storeAddress;
+      gp_id_t gpid =
+          HousekeepingMessage::getUpdateSnapshotVariableCommand(&hkMessage, &storeAddress);
+      REQUIRE(gpid.objectId == objects::INTERNAL_ERROR_REPORTER);
+
+      ConstAccessorPair data = ipcStore->getData(storeAddress);
+      REQUIRE(data.first == HasReturnvaluesIF::RETURN_OK);
+      CCSDSTime::CDS_short time;
+      // We need the object ID of the reporter here (NO_OBJECT)
+      InternalErrorDataset dataset(objects::INTERNAL_ERROR_REPORTER);
+      HousekeepingSnapshot hkSnapshot(&time, &dataset);
+      const uint8_t* buffer = data.second.data();
+      size_t size = data.second.size();
+      result = hkSnapshot.deSerialize(&buffer, &size, SerializeIF::Endianness::MACHINE);
+      REQUIRE(result == HasReturnvaluesIF::RETURN_OK);
+      // Test that we had one more queueHit
+      REQUIRE(dataset.queueHits.value == (queueHits + 1));
+      REQUIRE(dataset.tmHits.value == (lostTm + 1));
+      REQUIRE(dataset.storeHits.value == (storeHits + 1));
+    }
+    // Complete Coverage
+    internalErrorReporter->setDiagnosticPrintout(true);
+    internalErrorReporter->setMutexTimeout(MutexIF::TimeoutType::BLOCKING, 0);
+    {
+      // Message Queue Id
+      MessageQueueId_t id = internalErrorReporter->getCommandQueue();
+      REQUIRE(id != MessageQueueIF::NO_QUEUE);
+      CommandMessage message;
+      sid_t sid(objects::INTERNAL_ERROR_REPORTER, InternalErrorDataset::ERROR_SET_ID);
+      HousekeepingMessage::setToggleReportingCommand(&message, sid, true, false);
+      result = hkQueue->sendMessage(id, &message);
+      REQUIRE(result == HasReturnvaluesIF::RETURN_OK);
+      internalErrorReporter->performOperation(0);
+    }
+  }
+  QueueFactory::instance()->deleteMessageQueue(testQueue);
+  QueueFactory::instance()->deleteMessageQueue(hkQueue);
+}

tests/src/fsfw_tests/unit/mocks/PeriodicTaskIFMock.h

@@ -0,0 +1,37 @@
+#ifndef FSFW_UNITTEST_TESTS_MOCKS_PERIODICTASKMOCK_H_
+#define FSFW_UNITTEST_TESTS_MOCKS_PERIODICTASKMOCK_H_
+
+#include <fsfw/tasks/ExecutableObjectIF.h>
+#include <fsfw/tasks/PeriodicTaskIF.h>
+
+class PeriodicTaskMock : public PeriodicTaskIF {
+ public:
+  PeriodicTaskMock(uint32_t period = 5) : period(period) {}
+  /**
+   * @brief	A virtual destructor as it is mandatory for interfaces.
+   */
+  virtual ~PeriodicTaskMock() {}
+  /**
+   * @brief	With the startTask method, a created task can be started
+   *          for the first time.
+   */
+  virtual ReturnValue_t startTask() override { return HasReturnvaluesIF::RETURN_OK; };
+
+  virtual ReturnValue_t addComponent(object_id_t object) override {
+    ExecutableObjectIF* executableObject =
+        ObjectManager::instance()->get<ExecutableObjectIF>(objects::INTERNAL_ERROR_REPORTER);
+    if (executableObject == nullptr) {
+      return HasReturnvaluesIF::RETURN_FAILED;
+    }
+    executableObject->setTaskIF(this);
+    executableObject->initializeAfterTaskCreation();
+    return HasReturnvaluesIF::RETURN_OK;
+  };
+
+  virtual ReturnValue_t sleepFor(uint32_t ms) override { return HasReturnvaluesIF::RETURN_OK; };
+
+  virtual uint32_t getPeriodMs() const override { return period; };
+  uint32_t period;
+};
+
+#endif  // FSFW_UNITTEST_TESTS_MOCKS_PERIODICTASKMOCK_H_

tests/src/fsfw_tests/unit/osal/TestMessageQueue.cpp

@@ -35,4 +35,25 @@ TEST_CASE("MessageQueue Basic Test", "[TestMq]") {
     senderId = testReceiverMq->getLastPartner();
     CHECK(senderId == testSenderMqId);
   }
+  SECTION("Test Full") {
+    auto result = testSenderMq->sendMessage(testReceiverMqId, &testMessage);
+    REQUIRE(result == retval::CATCH_OK);
+    result = testSenderMq->sendMessage(testReceiverMqId, &testMessage);
+    REQUIRE(result == MessageQueueIF::FULL);
+    // We try another message
+    result = testSenderMq->sendMessage(testReceiverMqId, &testMessage);
+    REQUIRE(result == MessageQueueIF::FULL);
+    MessageQueueMessage recvMessage;
+    result = testReceiverMq->receiveMessage(&recvMessage);
+    REQUIRE(result == retval::CATCH_OK);
+    CHECK(recvMessage.getData()[0] == 42);
+    result = testSenderMq->sendMessage(testReceiverMqId, &testMessage);
+    REQUIRE(result == retval::CATCH_OK);
+    result = testReceiverMq->receiveMessage(&recvMessage);
+    REQUIRE(result == retval::CATCH_OK);
+    CHECK(recvMessage.getData()[0] == 42);
+  }
+  // We have to clear MQs ourself ATM
+  QueueFactory::instance()->deleteMessageQueue(testSenderMq);
+  QueueFactory::instance()->deleteMessageQueue(testReceiverMq);
 }