New total reboot counter handling #708

Merged
muellerr merged 16 commits from total-boot-counter-file into main 2023-06-26 15:33:22 +02:00
7 changed files with 440 additions and 255 deletions

View File

@ -16,6 +16,15 @@ will consitute of a breaking change warranting a new major release:
# [unreleased]
# [v4.1.0]
## Changed
- Added `sync` syscall in graceful shutdown handler
- Graceful shutdown is now performed by the reboot watchdog
- There is now a separate file for the total reboot counter. The reboot watchdog has its own local
counters to determine whether a reboot is necessary.
# [v4.0.1] 2023-06-24
## Fixed

View File

@ -321,23 +321,23 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_
if (size < 1) {
return HasActionsIF::INVALID_PARAMETERS;
}
std::string path = sdcMan->getCurrentMountPrefix() + REBOOT_FILE;
parseRebootFile(path, rebootFile);
std::string path = sdcMan->getCurrentMountPrefix() + REBOOT_WATCHDOG_FILE;
parseRebootWatchdogFile(path, rebootWatchdogFile);
if (data[0] == 0) {
rebootFile.enabled = false;
rewriteRebootFile(rebootFile);
rebootWatchdogFile.enabled = false;
rewriteRebootWatchdogFile(rebootWatchdogFile);
} else if (data[0] == 1) {
rebootFile.enabled = true;
rewriteRebootFile(rebootFile);
rebootWatchdogFile.enabled = true;
rewriteRebootWatchdogFile(rebootWatchdogFile);
} else {
return HasActionsIF::INVALID_PARAMETERS;
}
return HasActionsIF::EXECUTION_FINISHED;
}
case (READ_REBOOT_MECHANISM_INFO): {
std::string path = sdcMan->getCurrentMountPrefix() + REBOOT_FILE;
parseRebootFile(path, rebootFile);
RebootMechanismPacket packet(rebootFile);
std::string path = sdcMan->getCurrentMountPrefix() + REBOOT_WATCHDOG_FILE;
parseRebootWatchdogFile(path, rebootWatchdogFile);
RebootWatchdogPacket packet(rebootWatchdogFile);
ReturnValue_t result = actionHelper.reportData(commandedBy, actionId, &packet);
if (result != returnvalue::OK) {
return result;
@ -346,12 +346,13 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_
}
case (RESET_REBOOT_COUNTERS): {
if (size == 0) {
resetRebootCount(xsc::ALL_CHIP, xsc::ALL_COPY);
resetRebootWatchdogCounters(xsc::ALL_CHIP, xsc::ALL_COPY);
} else if (size == 2) {
if (data[0] > 1 or data[1] > 1) {
return HasActionsIF::INVALID_PARAMETERS;
}
resetRebootCount(static_cast<xsc::Chip>(data[0]), static_cast<xsc::Copy>(data[1]));
resetRebootWatchdogCounters(static_cast<xsc::Chip>(data[0]),
static_cast<xsc::Copy>(data[1]));
}
return HasActionsIF::EXECUTION_FINISHED;
}
@ -446,11 +447,11 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_
if (size < 1) {
return HasActionsIF::INVALID_PARAMETERS;
}
std::string path = sdcMan->getCurrentMountPrefix() + REBOOT_FILE;
std::string path = sdcMan->getCurrentMountPrefix() + REBOOT_WATCHDOG_FILE;
// Disable the reboot file mechanism
parseRebootFile(path, rebootFile);
rebootFile.maxCount = data[0];
rewriteRebootFile(rebootFile);
parseRebootWatchdogFile(path, rebootWatchdogFile);
rebootWatchdogFile.maxCount = data[0];
rewriteRebootWatchdogFile(rebootWatchdogFile);
return HasActionsIF::EXECUTION_FINISHED;
}
case (XSC_REBOOT_OBC): {
@ -1208,45 +1209,7 @@ ReturnValue_t CoreController::actionXscReboot(const uint8_t *data, size_t size)
auto tgtChip = static_cast<xsc::Chip>(data[1]);
auto tgtCopy = static_cast<xsc::Copy>(data[2]);
// This function can not really fail
gracefulShutdownTasks(tgtChip, tgtCopy, protOpPerformed);
switch (tgtChip) {
case (xsc::Chip::CHIP_0): {
switch (tgtCopy) {
case (xsc::Copy::COPY_0): {
xsc_boot_copy(XSC_LIBNOR_CHIP_0, XSC_LIBNOR_COPY_NOMINAL);
break;
}
case (xsc::Copy::COPY_1): {
xsc_boot_copy(XSC_LIBNOR_CHIP_0, XSC_LIBNOR_COPY_GOLD);
break;
}
default: {
break;
}
}
break;
}
case (xsc::Chip::CHIP_1): {
switch (tgtCopy) {
case (xsc::Copy::COPY_0): {
xsc_boot_copy(XSC_LIBNOR_CHIP_1, XSC_LIBNOR_COPY_NOMINAL);
break;
}
case (xsc::Copy::COPY_1): {
xsc_boot_copy(XSC_LIBNOR_CHIP_1, XSC_LIBNOR_COPY_GOLD);
break;
}
default: {
break;
}
}
break;
}
default:
break;
}
performGracefulShutdown(tgtChip, tgtCopy);
return returnvalue::FAILED;
}
@ -1263,10 +1226,15 @@ ReturnValue_t CoreController::gracefulShutdownTasks(xsc::Chip chip, xsc::Copy co
sdcMan->markUnusable();
// Wait two seconds to ensure no one uses the SD cards
TaskFactory::delayTask(2000);
// Ensure that all writes/reads do finish.
sync();
// Attempt graceful shutdown by unmounting and switching off SD cards
sdcMan->switchOffSdCard(sd::SdCard::SLOT_0);
sdcMan->switchOffSdCard(sd::SdCard::SLOT_1);
// If any boot copies are unprotected
// If any boot copies are unprotected.
// Actually this function only ensures that reboots to the own image are protected..
ReturnValue_t result = setBootCopyProtection(xsc::Chip::SELF_CHIP, xsc::Copy::SELF_COPY, true,
protOpPerformed, false);
if (result == returnvalue::OK and protOpPerformed) {
@ -1555,7 +1523,8 @@ void CoreController::performMountedSdCardOperations() {
if (not timeFileInitDone) {
initClockFromTimeFile();
}
performRebootFileHandling(false);
performRebootWatchdogHandling(false);
performRebootCountersHandling(false);
}
backupTimeFileHandler();
};
@ -1627,117 +1596,126 @@ ReturnValue_t CoreController::performSdCardCheck() {
return returnvalue::OK;
}
void CoreController::performRebootFileHandling(bool recreateFile) {
void CoreController::performRebootWatchdogHandling(bool recreateFile) {
using namespace std;
std::string path = currMntPrefix + REBOOT_FILE;
std::string path = currMntPrefix + REBOOT_WATCHDOG_FILE;
std::string legacyPath = currMntPrefix + LEGACY_REBOOT_WATCHDOG_FILE;
std::error_code e;
// TODO: Remove at some point in the future.
if (std::filesystem::exists(legacyPath, e)) {
meggert marked this conversation as resolved Outdated

use variant with error_code instead

use variant with `error_code` instead
// Old file might still exist, so copy it to new path
std::filesystem::copy(legacyPath, path, std::filesystem::copy_options::overwrite_existing, e);
if (e) {
sif::error << "File copy has failed: " << e.message() << std::endl;
}
}
if (not std::filesystem::exists(path, e) or recreateFile) {
#if OBSW_VERBOSE_LEVEL >= 1
sif::info << "CoreController::performRebootFileHandling: Recreating reboot file" << std::endl;
sif::info << "CoreController::performRebootFileHandling: Recreating reboot watchdog file"
<< std::endl;
#endif
rebootFile.enabled = false;
rebootFile.img00Cnt = 0;
rebootFile.img01Cnt = 0;
rebootFile.img10Cnt = 0;
rebootFile.img11Cnt = 0;
rebootFile.lastChip = xsc::Chip::CHIP_0;
rebootFile.lastCopy = xsc::Copy::COPY_0;
rebootFile.img00Lock = false;
rebootFile.img01Lock = false;
rebootFile.img10Lock = false;
rebootFile.img11Lock = false;
rebootFile.mechanismNextChip = xsc::Chip::NO_CHIP;
rebootFile.mechanismNextCopy = xsc::Copy::NO_COPY;
rebootFile.bootFlag = false;
rewriteRebootFile(rebootFile);
rebootWatchdogFile.enabled = false;
rebootWatchdogFile.img00Cnt = 0;
rebootWatchdogFile.img01Cnt = 0;
rebootWatchdogFile.img10Cnt = 0;
rebootWatchdogFile.img11Cnt = 0;
rebootWatchdogFile.lastChip = xsc::Chip::CHIP_0;
rebootWatchdogFile.lastCopy = xsc::Copy::COPY_0;
rebootWatchdogFile.img00Lock = false;
rebootWatchdogFile.img01Lock = false;
rebootWatchdogFile.img10Lock = false;
rebootWatchdogFile.img11Lock = false;
rebootWatchdogFile.mechanismNextChip = xsc::Chip::NO_CHIP;
rebootWatchdogFile.mechanismNextCopy = xsc::Copy::NO_COPY;
rebootWatchdogFile.bootFlag = false;
rewriteRebootWatchdogFile(rebootWatchdogFile);
} else {
if (not parseRebootFile(path, rebootFile)) {
performRebootFileHandling(true);
if (not parseRebootWatchdogFile(path, rebootWatchdogFile)) {
performRebootWatchdogHandling(true);
return;
}
}
if (CURRENT_CHIP == xsc::CHIP_0) {
if (CURRENT_COPY == xsc::COPY_0) {
rebootFile.img00Cnt++;
rebootWatchdogFile.img00Cnt++;
} else {
rebootFile.img01Cnt++;
rebootWatchdogFile.img01Cnt++;
}
} else {
if (CURRENT_COPY == xsc::COPY_0) {
rebootFile.img10Cnt++;
rebootWatchdogFile.img10Cnt++;
} else {
rebootFile.img11Cnt++;
rebootWatchdogFile.img11Cnt++;
}
}
if (rebootFile.bootFlag) {
if (rebootWatchdogFile.bootFlag) {
// Trigger event to inform ground that a reboot was triggered
uint32_t p1 = rebootFile.lastChip << 16 | rebootFile.lastCopy;
uint32_t p1 = rebootWatchdogFile.lastChip << 16 | rebootWatchdogFile.lastCopy;
triggerEvent(core::REBOOT_MECHANISM_TRIGGERED, p1, 0);
// Clear the boot flag
rebootFile.bootFlag = false;
rebootWatchdogFile.bootFlag = false;
}
announceBootCounts();
if (rebootFile.mechanismNextChip != xsc::NO_CHIP and
rebootFile.mechanismNextCopy != xsc::NO_COPY) {
if (CURRENT_CHIP != rebootFile.mechanismNextChip or
CURRENT_COPY != rebootFile.mechanismNextCopy) {
std::string infoString = std::to_string(rebootFile.mechanismNextChip) + " " +
std::to_string(rebootFile.mechanismNextCopy);
if (rebootWatchdogFile.mechanismNextChip != xsc::NO_CHIP and
rebootWatchdogFile.mechanismNextCopy != xsc::NO_COPY) {
if (CURRENT_CHIP != rebootWatchdogFile.mechanismNextChip or
CURRENT_COPY != rebootWatchdogFile.mechanismNextCopy) {
std::string infoString = std::to_string(rebootWatchdogFile.mechanismNextChip) + " " +
std::to_string(rebootWatchdogFile.mechanismNextCopy);
sif::warning << "CoreController::performRebootFileHandling: Expected to be on image "
<< infoString << " but currently on other image. Locking " << infoString
<< std::endl;
// Firmware or other component might be corrupt and we are on another image then the target
// image specified by the mechanism. We can't really trust the target image anymore.
// Lock it for now
if (rebootFile.mechanismNextChip == xsc::CHIP_0) {
if (rebootFile.mechanismNextCopy == xsc::COPY_0) {
rebootFile.img00Lock = true;
if (rebootWatchdogFile.mechanismNextChip == xsc::CHIP_0) {
if (rebootWatchdogFile.mechanismNextCopy == xsc::COPY_0) {
rebootWatchdogFile.img00Lock = true;
} else {
rebootFile.img01Lock = true;
rebootWatchdogFile.img01Lock = true;
}
} else {
if (rebootFile.mechanismNextCopy == xsc::COPY_0) {
rebootFile.img10Lock = true;
if (rebootWatchdogFile.mechanismNextCopy == xsc::COPY_0) {
rebootWatchdogFile.img10Lock = true;
} else {
rebootFile.img11Lock = true;
rebootWatchdogFile.img11Lock = true;
}
}
}
}
rebootFile.lastChip = CURRENT_CHIP;
rebootFile.lastCopy = CURRENT_COPY;
rebootWatchdogFile.lastChip = CURRENT_CHIP;
rebootWatchdogFile.lastCopy = CURRENT_COPY;
// Only reboot if the reboot functionality is enabled.
// The handler will still increment the boot counts
if (rebootFile.enabled and (*rebootFile.relevantBootCnt >= rebootFile.maxCount)) {
if (rebootWatchdogFile.enabled and
(*rebootWatchdogFile.relevantBootCnt >= rebootWatchdogFile.maxCount)) {
// Reboot to other image
bool doReboot = false;
xsc::Chip tgtChip = xsc::NO_CHIP;
xsc::Copy tgtCopy = xsc::NO_COPY;
determineAndExecuteReboot(rebootFile, doReboot, tgtChip, tgtCopy);
rebootWatchdogAlgorithm(rebootWatchdogFile, doReboot, tgtChip, tgtCopy);
if (doReboot) {
rebootFile.bootFlag = true;
rebootWatchdogFile.bootFlag = true;
#if OBSW_VERBOSE_LEVEL >= 1
sif::info << "Boot counter for image " << CURRENT_CHIP << " " << CURRENT_COPY
<< " too high. Rebooting to " << tgtChip << " " << tgtCopy << std::endl;
#endif
rebootFile.mechanismNextChip = tgtChip;
rebootFile.mechanismNextCopy = tgtCopy;
rewriteRebootFile(rebootFile);
xsc_boot_copy(static_cast<xsc_libnor_chip_t>(tgtChip),
static_cast<xsc_libnor_copy_t>(tgtCopy));
rebootWatchdogFile.mechanismNextChip = tgtChip;
rebootWatchdogFile.mechanismNextCopy = tgtCopy;
rewriteRebootWatchdogFile(rebootWatchdogFile);
performGracefulShutdown(tgtChip, tgtCopy);
}
} else {
rebootFile.mechanismNextChip = xsc::NO_CHIP;
rebootFile.mechanismNextCopy = xsc::NO_COPY;
rebootWatchdogFile.mechanismNextChip = xsc::NO_CHIP;
rebootWatchdogFile.mechanismNextCopy = xsc::NO_COPY;
}
rewriteRebootFile(rebootFile);
rewriteRebootWatchdogFile(rebootWatchdogFile);
}
void CoreController::determineAndExecuteReboot(RebootFile &rf, bool &needsReboot,
void CoreController::rebootWatchdogAlgorithm(RebootWatchdogFile &rf, bool &needsReboot,
xsc::Chip &tgtChip, xsc::Copy &tgtCopy) {
tgtChip = xsc::CHIP_0;
tgtCopy = xsc::COPY_0;
@ -1826,7 +1804,7 @@ void CoreController::determineAndExecuteReboot(RebootFile &rf, bool &needsReboot
}
}
bool CoreController::parseRebootFile(std::string path, RebootFile &rf) {
bool CoreController::parseRebootWatchdogFile(std::string path, RebootWatchdogFile &rf) {
using namespace std;
std::string selfMatch;
if (CURRENT_CHIP == xsc::CHIP_0) {
@ -2008,34 +1986,123 @@ bool CoreController::parseRebootFile(std::string path, RebootFile &rf) {
return true;
}
void CoreController::resetRebootCount(xsc::Chip tgtChip, xsc::Copy tgtCopy) {
std::string path = currMntPrefix + REBOOT_FILE;
parseRebootFile(path, rebootFile);
bool CoreController::parseRebootCountersFile(std::string path, RebootCountersFile &rf) {
using namespace std;
ifstream file(path);
string word;
string line;
uint8_t lineIdx = 0;
while (std::getline(file, line)) {
istringstream iss(line);
switch (lineIdx) {
case 0: {
iss >> word;
if (word.find("img00:") == string::npos) {
return false;
}
iss >> rf.img00Cnt;
break;
}
case 1: {
iss >> word;
if (word.find("img01:") == string::npos) {
return false;
}
iss >> rf.img01Cnt;
break;
}
case 2: {
iss >> word;
if (word.find("img10:") == string::npos) {
return false;
}
iss >> rf.img10Cnt;
break;
}
case 3: {
iss >> word;
if (word.find("img11:") == string::npos) {
return false;
}
iss >> rf.img11Cnt;
break;
}
}
lineIdx++;
}
return true;
}
void CoreController::resetRebootWatchdogCounters(xsc::Chip tgtChip, xsc::Copy tgtCopy) {
std::string path = currMntPrefix + REBOOT_WATCHDOG_FILE;
parseRebootWatchdogFile(path, rebootWatchdogFile);
if (tgtChip == xsc::ALL_CHIP and tgtCopy == xsc::ALL_COPY) {
rebootFile.img00Cnt = 0;
rebootFile.img01Cnt = 0;
rebootFile.img10Cnt = 0;
rebootFile.img11Cnt = 0;
rebootWatchdogFile.img00Cnt = 0;
rebootWatchdogFile.img01Cnt = 0;
rebootWatchdogFile.img10Cnt = 0;
rebootWatchdogFile.img11Cnt = 0;
} else {
if (tgtChip == xsc::CHIP_0) {
if (tgtCopy == xsc::COPY_0) {
rebootFile.img00Cnt = 0;
rebootWatchdogFile.img00Cnt = 0;
} else {
rebootFile.img01Cnt = 0;
rebootWatchdogFile.img01Cnt = 0;
}
} else {
if (tgtCopy == xsc::COPY_0) {
rebootFile.img10Cnt = 0;
rebootWatchdogFile.img10Cnt = 0;
} else {
rebootFile.img11Cnt = 0;
rebootWatchdogFile.img11Cnt = 0;
}
}
}
rewriteRebootFile(rebootFile);
rewriteRebootWatchdogFile(rebootWatchdogFile);
}
void CoreController::rewriteRebootFile(RebootFile file) {
std::string path = currMntPrefix + REBOOT_FILE;
void CoreController::performRebootCountersHandling(bool recreateFile) {
std::string path = currMntPrefix + REBOOT_COUNTERS_FILE;
std::error_code e;
if (not std::filesystem::exists(path, e) or recreateFile) {
#if OBSW_VERBOSE_LEVEL >= 1
sif::info << "CoreController::performRebootFileHandling: Recreating reboot counters file"
<< std::endl;
#endif
rebootCountersFile.img00Cnt = 0;
rebootCountersFile.img01Cnt = 0;
rebootCountersFile.img10Cnt = 0;
rebootCountersFile.img11Cnt = 0;
rewriteRebootCountersFile(rebootCountersFile);
} else {
if (not parseRebootCountersFile(path, rebootCountersFile)) {
performRebootCountersHandling(true);
return;
}
}
if (CURRENT_CHIP == xsc::CHIP_0) {
if (CURRENT_COPY == xsc::COPY_0) {
rebootCountersFile.img00Cnt++;
} else {
rebootCountersFile.img01Cnt++;
}
} else {
if (CURRENT_COPY == xsc::COPY_0) {
rebootCountersFile.img10Cnt++;
} else {
rebootCountersFile.img11Cnt++;
}
}
announceBootCounts();
rewriteRebootCountersFile(rebootCountersFile);
}
void CoreController::rewriteRebootWatchdogFile(RebootWatchdogFile file) {
using namespace std::filesystem;
std::string path = currMntPrefix + REBOOT_WATCHDOG_FILE;
std::string legacyPath = currMntPrefix + LEGACY_REBOOT_WATCHDOG_FILE;
{
std::ofstream rebootFile(path);
if (rebootFile.is_open()) {
// Initiate reboot file first. Reboot handling will be on on initialization
@ -2050,25 +2117,43 @@ void CoreController::rewriteRebootFile(RebootFile file) {
<< static_cast<int>(file.mechanismNextCopy) << "\n";
}
}
std::error_code e;
// TODO: Remove at some point in the future when all images have been updated.
if (std::filesystem::exists(legacyPath)) {
// Keep those two files in sync
std::filesystem::copy(path, legacyPath, std::filesystem::copy_options::overwrite_existing, e);
if (e) {
sif::error << "File copy has failed: " << e.message() << std::endl;
}
}
}
void CoreController::rewriteRebootCountersFile(RebootCountersFile file) {
std::string path = currMntPrefix + REBOOT_COUNTERS_FILE;
std::ofstream rebootFile(path);
if (rebootFile.is_open()) {
rebootFile << "img00: " << file.img00Cnt << "\nimg01: " << file.img01Cnt
<< "\nimg10: " << file.img10Cnt << "\nimg11: " << file.img11Cnt << "\n";
}
}
void CoreController::setRebootMechanismLock(bool lock, xsc::Chip tgtChip, xsc::Copy tgtCopy) {
std::string path = currMntPrefix + REBOOT_FILE;
// Disable the reboot file mechanism
parseRebootFile(path, rebootFile);
std::string path = currMntPrefix + REBOOT_WATCHDOG_FILE;
parseRebootWatchdogFile(path, rebootWatchdogFile);
if (tgtChip == xsc::CHIP_0) {
if (tgtCopy == xsc::COPY_0) {
rebootFile.img00Lock = lock;
rebootWatchdogFile.img00Lock = lock;
} else {
rebootFile.img01Lock = lock;
rebootWatchdogFile.img01Lock = lock;
}
} else {
if (tgtCopy == xsc::COPY_0) {
rebootFile.img10Lock = lock;
rebootWatchdogFile.img10Lock = lock;
} else {
rebootFile.img11Lock = lock;
rebootWatchdogFile.img11Lock = lock;
}
}
rewriteRebootFile(rebootFile);
rewriteRebootWatchdogFile(rebootWatchdogFile);
}
ReturnValue_t CoreController::backupTimeFileHandler() {
@ -2355,10 +2440,12 @@ bool CoreController::startSdStateMachine(sd::SdCard targetActiveSd, SdCfgMode mo
}
void CoreController::announceBootCounts() {
uint64_t totalBootCount =
rebootFile.img00Cnt + rebootFile.img01Cnt + rebootFile.img10Cnt + rebootFile.img11Cnt;
uint32_t individualBootCountsP1 = (rebootFile.img00Cnt << 16) | rebootFile.img01Cnt;
uint32_t individualBootCountsP2 = (rebootFile.img10Cnt << 16) | rebootFile.img11Cnt;
uint64_t totalBootCount = rebootCountersFile.img00Cnt + rebootCountersFile.img01Cnt +
rebootCountersFile.img10Cnt + rebootCountersFile.img11Cnt;
uint32_t individualBootCountsP1 =
(rebootCountersFile.img00Cnt << 16) | rebootCountersFile.img01Cnt;
uint32_t individualBootCountsP2 =
(rebootCountersFile.img10Cnt << 16) | rebootCountersFile.img11Cnt;
triggerEvent(core::INDIVIDUAL_BOOT_COUNTS, individualBootCountsP1, individualBootCountsP2);
triggerEvent(core::REBOOT_COUNTER, (totalBootCount >> 32) & 0xffffffff,
totalBootCount & 0xffffffff);
@ -2443,6 +2530,50 @@ void CoreController::announceCurrentImageInfo() {
triggerEvent(CURRENT_IMAGE_INFO, CURRENT_CHIP, CURRENT_COPY);
}
ReturnValue_t CoreController::performGracefulShutdown(xsc::Chip tgtChip, xsc::Copy tgtCopy) {
bool protOpPerformed = false;
// This function can not really fail
gracefulShutdownTasks(tgtChip, tgtCopy, protOpPerformed);
switch (tgtChip) {
case (xsc::Chip::CHIP_0): {
switch (tgtCopy) {
case (xsc::Copy::COPY_0): {
xsc_boot_copy(XSC_LIBNOR_CHIP_0, XSC_LIBNOR_COPY_NOMINAL);
break;
}
case (xsc::Copy::COPY_1): {
xsc_boot_copy(XSC_LIBNOR_CHIP_0, XSC_LIBNOR_COPY_GOLD);
break;
}
default: {
break;
}
}
break;
}
case (xsc::Chip::CHIP_1): {
switch (tgtCopy) {
case (xsc::Copy::COPY_0): {
xsc_boot_copy(XSC_LIBNOR_CHIP_1, XSC_LIBNOR_COPY_NOMINAL);
break;
}
case (xsc::Copy::COPY_1): {
xsc_boot_copy(XSC_LIBNOR_CHIP_1, XSC_LIBNOR_COPY_GOLD);
break;
}
default: {
break;
}
}
break;
}
default:
break;
}
return returnvalue::OK;
}
bool CoreController::isNumber(const std::string &s) {
return !s.empty() && std::find_if(s.begin(), s.end(),
[](unsigned char c) { return !std::isdigit(c); }) == s.end();

View File

@ -24,7 +24,7 @@
class Timer;
class SdCardManager;
struct RebootFile {
struct RebootWatchdogFile {
static constexpr uint8_t DEFAULT_MAX_BOOT_CNT = 10;
bool enabled = true;
@ -45,9 +45,9 @@ struct RebootFile {
xsc::Copy mechanismNextCopy = xsc::Copy::NO_COPY;
};
class RebootMechanismPacket : public SerialLinkedListAdapter<SerializeIF> {
class RebootWatchdogPacket : public SerialLinkedListAdapter<SerializeIF> {
public:
RebootMechanismPacket(RebootFile& rf) {
RebootWatchdogPacket(RebootWatchdogFile& rf) {
enabled = rf.enabled;
maxCount = rf.maxCount;
img00Count = rf.img00Cnt;
@ -100,6 +100,38 @@ class RebootMechanismPacket : public SerialLinkedListAdapter<SerializeIF> {
SerializeElement<uint8_t> nextCopy = 0;
};
struct RebootCountersFile {
// 16 bit values so all boot counters fit into one event.
uint16_t img00Cnt = 0;
uint16_t img01Cnt = 0;
uint16_t img10Cnt = 0;
uint16_t img11Cnt = 0;
};
class RebootCountersPacket : public SerialLinkedListAdapter<SerializeIF> {
RebootCountersPacket(RebootCountersFile& rf) {
img00Count = rf.img00Cnt;
img01Count = rf.img01Cnt;
img10Count = rf.img10Cnt;
img11Count = rf.img11Cnt;
setLinks();
}
private:
void setLinks() {
setStart(&img00Count);
img00Count.setNext(&img01Count);
img01Count.setNext(&img10Count);
img10Count.setNext(&img11Count);
setLast(&img11Count);
}
SerializeElement<uint16_t> img00Count = 0;
SerializeElement<uint16_t> img01Count = 0;
SerializeElement<uint16_t> img10Count = 0;
SerializeElement<uint16_t> img11Count = 0;
};
class CoreController : public ExtendedControllerBase, public ReceivesParameterMessagesIF {
public:
enum ParamId : uint8_t { PREF_SD = 0, NUM_IDS };
@ -113,10 +145,15 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe
const std::string VERSION_FILE =
"/" + std::string(core::CONF_FOLDER) + "/" + std::string(core::VERSION_FILE_NAME);
const std::string REBOOT_FILE =
"/" + std::string(core::CONF_FOLDER) + "/" + std::string(core::REBOOT_FILE_NAME);
const std::string LEGACY_REBOOT_WATCHDOG_FILE =
"/" + std::string(core::CONF_FOLDER) + "/" +
std::string(core::LEGACY_REBOOT_WATCHDOG_FILE_NAME);
const std::string REBOOT_WATCHDOG_FILE =
"/" + std::string(core::CONF_FOLDER) + "/" + std::string(core::REBOOT_WATCHDOG_FILE_NAME);
const std::string BACKUP_TIME_FILE =
"/" + std::string(core::CONF_FOLDER) + "/" + std::string(core::TIME_FILE_NAME);
const std::string REBOOT_COUNTERS_FILE =
"/" + std::string(core::CONF_FOLDER) + "/" + std::string(core::REBOOT_COUNTER_FILE_NAME);
static constexpr char CHIP_0_COPY_0_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi0-nom-rootfs";
static constexpr char CHIP_0_COPY_1_MOUNT_DIR[] = "/tmp/mntupdate-xdi-qspi0-gold-rootfs";
@ -248,7 +285,8 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe
std::array<uint8_t, 1024> dirListingBuf{};
DirListingDumpContext dumpContext{};
RebootFile rebootFile = {};
RebootWatchdogFile rebootWatchdogFile = {};
RebootCountersFile rebootCountersFile = {};
CommandExecutor cmdExecutor;
SimpleRingBuffer cmdReplyBuf;
@ -318,12 +356,14 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe
void currentStateSetter(sd::SdCard sdCard, sd::SdState newState);
void executeNextExternalSdCommand();
void checkExternalSdCommandStatus();
void performRebootFileHandling(bool recreateFile);
void performRebootWatchdogHandling(bool recreateFile);
void performRebootCountersHandling(bool recreateFile);
ReturnValue_t actionListDirectoryIntoFile(ActionId_t actionId, MessageQueueId_t commandedBy,
const uint8_t* data, size_t size);
ReturnValue_t actionListDirectoryDumpDirectly(ActionId_t actionId, MessageQueueId_t commandedBy,
const uint8_t* data, size_t size);
ReturnValue_t performGracefulShutdown(xsc::Chip targetChip, xsc::Copy targetCopy);
ReturnValue_t actionListDirectoryCommonCommandCreator(const uint8_t* data, size_t size,
std::ostringstream& oss);
@ -337,12 +377,14 @@ class CoreController : public ExtendedControllerBase, public ReceivesParameterMe
int handleBootCopyProtAtIndex(xsc::Chip targetChip, xsc::Copy targetCopy, bool protect,
bool& protOperationPerformed, bool selfChip, bool selfCopy,
bool allChips, bool allCopies, uint8_t arrIdx);
void determineAndExecuteReboot(RebootFile& rf, bool& needsReboot, xsc::Chip& tgtChip,
void rebootWatchdogAlgorithm(RebootWatchdogFile& rf, bool& needsReboot, xsc::Chip& tgtChip,
xsc::Copy& tgtCopy);
void resetRebootCount(xsc::Chip tgtChip, xsc::Copy tgtCopy);
void resetRebootWatchdogCounters(xsc::Chip tgtChip, xsc::Copy tgtCopy);
void setRebootMechanismLock(bool lock, xsc::Chip tgtChip, xsc::Copy tgtCopy);
bool parseRebootFile(std::string path, RebootFile& file);
void rewriteRebootFile(RebootFile file);
bool parseRebootWatchdogFile(std::string path, RebootWatchdogFile& file);
bool parseRebootCountersFile(std::string path, RebootCountersFile& file);
void rewriteRebootWatchdogFile(RebootWatchdogFile file);
void rewriteRebootCountersFile(RebootCountersFile file);
void announceBootCounts();
void announceVersionInfo();
void announceCurrentImageInfo();

View File

@ -41,7 +41,9 @@ enum SystemctlCmd : uint8_t { START = 0, STOP = 1, RESTART = 2, NUM_CMDS = 3 };
static constexpr char CONF_FOLDER[] = "conf";
static constexpr char VERSION_FILE_NAME[] = "version.txt";
static constexpr char REBOOT_FILE_NAME[] = "reboot.txt";
static constexpr char LEGACY_REBOOT_WATCHDOG_FILE_NAME[] = "reboot.txt";
static constexpr char REBOOT_WATCHDOG_FILE_NAME[] = "reboot_watchdog.txt";
static constexpr char REBOOT_COUNTER_FILE_NAME[] = "reboot_counters.txt";
static constexpr char TIME_FILE_NAME[] = "time_backup.txt";
static constexpr uint32_t SYS_ROM_BASE_ADDR = 0x80000000;

2
tmtc

@ -1 +1 @@
Subproject commit c9f4a8070d20bc659809d5b822ac5a17548f57a4
Subproject commit deb0275bb5603394122e26f74760d2051685f324

View File

@ -19,116 +19,117 @@ CoreController::CoreController() {
setCurrentBootCopy(xsc::CHIP_0, xsc::COPY_0);
}
void CoreController::performRebootFileHandling(bool recreateFile) {
void CoreController::performRebootWatchdogHandling(bool recreateFile) {
using namespace std;
std::string path = sdcMan->getCurrentMountPrefix(sdInfo.active) + REBOOT_FILE;
std::string path = sdcMan->getCurrentMountPrefix(sdInfo.active) + REBOOT_WATCHDOG_FILE;
if (not std::filesystem::exists(path) or recreateFile) {
#if OBSW_VERBOSE_LEVEL >= 1
sif::info << "CoreController::performRebootFileHandling: Recreating reboot file" << std::endl;
#endif
rebootFile.enabled = true;
rebootFile.img00Cnt = 0;
rebootFile.img01Cnt = 0;
rebootFile.img10Cnt = 0;
rebootFile.img11Cnt = 0;
rebootFile.lastChip = xsc::Chip::CHIP_0;
rebootFile.lastCopy = xsc::Copy::COPY_0;
rebootFile.img00Lock = false;
rebootFile.img01Lock = false;
rebootFile.img10Lock = false;
rebootFile.img11Lock = false;
rebootFile.mechanismNextChip = xsc::Chip::NO_CHIP;
rebootFile.mechanismNextCopy = xsc::Copy::NO_COPY;
rebootFile.bootFlag = false;
rewriteRebootFile(rebootFile);
rebootWatchdogFile.enabled = true;
rebootWatchdogFile.img00Cnt = 0;
rebootWatchdogFile.img01Cnt = 0;
rebootWatchdogFile.img10Cnt = 0;
rebootWatchdogFile.img11Cnt = 0;
rebootWatchdogFile.lastChip = xsc::Chip::CHIP_0;
rebootWatchdogFile.lastCopy = xsc::Copy::COPY_0;
rebootWatchdogFile.img00Lock = false;
rebootWatchdogFile.img01Lock = false;
rebootWatchdogFile.img10Lock = false;
rebootWatchdogFile.img11Lock = false;
rebootWatchdogFile.mechanismNextChip = xsc::Chip::NO_CHIP;
rebootWatchdogFile.mechanismNextCopy = xsc::Copy::NO_COPY;
rebootWatchdogFile.bootFlag = false;
rewriteRebootWatchdogFile(rebootWatchdogFile);
} else {
if (not parseRebootFile(path, rebootFile)) {
performRebootFileHandling(true);
if (not parseRebootWatchdogFile(path, rebootWatchdogFile)) {
performRebootWatchdogHandling(true);
}
}
if (CURRENT_CHIP == xsc::CHIP_0) {
if (CURRENT_COPY == xsc::COPY_0) {
rebootFile.img00Cnt++;
rebootWatchdogFile.img00Cnt++;
} else {
rebootFile.img01Cnt++;
rebootWatchdogFile.img01Cnt++;
}
} else {
if (CURRENT_COPY == xsc::COPY_0) {
rebootFile.img10Cnt++;
rebootWatchdogFile.img10Cnt++;
} else {
rebootFile.img11Cnt++;
rebootWatchdogFile.img11Cnt++;
}
}
if (rebootFile.bootFlag) {
if (rebootWatchdogFile.bootFlag) {
// Trigger event to inform ground that a reboot was triggered
uint32_t p1 = rebootFile.lastChip << 16 | rebootFile.lastCopy;
uint32_t p2 = rebootFile.img00Cnt << 24 | rebootFile.img01Cnt << 16 | rebootFile.img10Cnt << 8 |
rebootFile.img11Cnt;
uint32_t p1 = rebootWatchdogFile.lastChip << 16 | rebootWatchdogFile.lastCopy;
uint32_t p2 = rebootWatchdogFile.img00Cnt << 24 | rebootWatchdogFile.img01Cnt << 16 |
rebootWatchdogFile.img10Cnt << 8 | rebootWatchdogFile.img11Cnt;
triggerEvent(REBOOT_MECHANISM_TRIGGERED, p1, p2);
// Clear the boot flag
rebootFile.bootFlag = false;
rebootWatchdogFile.bootFlag = false;
}
if (rebootFile.mechanismNextChip != xsc::NO_CHIP and
rebootFile.mechanismNextCopy != xsc::NO_COPY) {
if (CURRENT_CHIP != rebootFile.mechanismNextChip or
CURRENT_COPY != rebootFile.mechanismNextCopy) {
std::string infoString = std::to_string(rebootFile.mechanismNextChip) + " " +
std::to_string(rebootFile.mechanismNextCopy);
if (rebootWatchdogFile.mechanismNextChip != xsc::NO_CHIP and
rebootWatchdogFile.mechanismNextCopy != xsc::NO_COPY) {
if (CURRENT_CHIP != rebootWatchdogFile.mechanismNextChip or
CURRENT_COPY != rebootWatchdogFile.mechanismNextCopy) {
std::string infoString = std::to_string(rebootWatchdogFile.mechanismNextChip) + " " +
std::to_string(rebootWatchdogFile.mechanismNextCopy);
sif::warning << "CoreController::performRebootFileHandling: Expected to be on image "
<< infoString << " but currently on other image. Locking " << infoString
<< std::endl;
// Firmware or other component might be corrupt and we are on another image then the target
// image specified by the mechanism. We can't really trust the target image anymore.
// Lock it for now
if (rebootFile.mechanismNextChip == xsc::CHIP_0) {
if (rebootFile.mechanismNextCopy == xsc::COPY_0) {
rebootFile.img00Lock = true;
if (rebootWatchdogFile.mechanismNextChip == xsc::CHIP_0) {
if (rebootWatchdogFile.mechanismNextCopy == xsc::COPY_0) {
rebootWatchdogFile.img00Lock = true;
} else {
rebootFile.img01Lock = true;
rebootWatchdogFile.img01Lock = true;
}
} else {
if (rebootFile.mechanismNextCopy == xsc::COPY_0) {
rebootFile.img10Lock = true;
if (rebootWatchdogFile.mechanismNextCopy == xsc::COPY_0) {
rebootWatchdogFile.img10Lock = true;
} else {
rebootFile.img11Lock = true;
rebootWatchdogFile.img11Lock = true;
}
}
}
}
rebootFile.lastChip = CURRENT_CHIP;
rebootFile.lastCopy = CURRENT_COPY;
rebootWatchdogFile.lastChip = CURRENT_CHIP;
rebootWatchdogFile.lastCopy = CURRENT_COPY;
// Only reboot if the reboot functionality is enabled.
// The handler will still increment the boot counts
if (rebootFile.enabled and (*rebootFile.relevantBootCnt >= rebootFile.maxCount)) {
if (rebootWatchdogFile.enabled and
(*rebootWatchdogFile.relevantBootCnt >= rebootWatchdogFile.maxCount)) {
// Reboot to other image
bool doReboot = false;
xsc::Chip tgtChip = xsc::NO_CHIP;
xsc::Copy tgtCopy = xsc::NO_COPY;
determineAndExecuteReboot(rebootFile, doReboot, tgtChip, tgtCopy);
determineAndExecuteReboot(rebootWatchdogFile, doReboot, tgtChip, tgtCopy);
if (doReboot) {
rebootFile.bootFlag = true;
rebootWatchdogFile.bootFlag = true;
#if OBSW_VERBOSE_LEVEL >= 1
sif::info << "Boot counter for image " << CURRENT_CHIP << " " << CURRENT_COPY
<< " too high. Rebooting to " << tgtChip << " " << tgtCopy << std::endl;
#endif
rebootFile.mechanismNextChip = tgtChip;
rebootFile.mechanismNextCopy = tgtCopy;
rewriteRebootFile(rebootFile);
rebootWatchdogFile.mechanismNextChip = tgtChip;
rebootWatchdogFile.mechanismNextCopy = tgtCopy;
rewriteRebootWatchdogFile(rebootWatchdogFile);
xsc_boot_copy(static_cast<xsc_libnor_chip_t>(tgtChip),
static_cast<xsc_libnor_copy_t>(tgtCopy));
}
} else {
rebootFile.mechanismNextChip = xsc::NO_CHIP;
rebootFile.mechanismNextCopy = xsc::NO_COPY;
rebootWatchdogFile.mechanismNextChip = xsc::NO_CHIP;
rebootWatchdogFile.mechanismNextCopy = xsc::NO_COPY;
}
rewriteRebootFile(rebootFile);
rewriteRebootWatchdogFile(rebootWatchdogFile);
}
void CoreController::determineAndExecuteReboot(RebootFile &rf, bool &needsReboot,
void CoreController::determineAndExecuteReboot(RebootWatchdogFile &rf, bool &needsReboot,
xsc::Chip &tgtChip, xsc::Copy &tgtCopy) {
tgtChip = xsc::CHIP_0;
tgtCopy = xsc::COPY_0;
@ -217,7 +218,7 @@ void CoreController::determineAndExecuteReboot(RebootFile &rf, bool &needsReboot
}
}
bool CoreController::parseRebootFile(std::string path, RebootFile &rf) {
bool CoreController::parseRebootWatchdogFile(std::string path, RebootWatchdogFile &rf) {
using namespace std;
std::string selfMatch;
if (CURRENT_CHIP == xsc::CHIP_0) {
@ -400,34 +401,34 @@ bool CoreController::parseRebootFile(std::string path, RebootFile &rf) {
}
void CoreController::resetRebootCount(xsc::Chip tgtChip, xsc::Copy tgtCopy) {
std::string path = sdcMan->getCurrentMountPrefix(sdInfo.active) + REBOOT_FILE;
std::string path = sdcMan->getCurrentMountPrefix(sdInfo.active) + REBOOT_WATCHDOG_FILE;
// Disable the reboot file mechanism
parseRebootFile(path, rebootFile);
parseRebootWatchdogFile(path, rebootWatchdogFile);
if (tgtChip == xsc::ALL_CHIP and tgtCopy == xsc::ALL_COPY) {
rebootFile.img00Cnt = 0;
rebootFile.img01Cnt = 0;
rebootFile.img10Cnt = 0;
rebootFile.img11Cnt = 0;
rebootWatchdogFile.img00Cnt = 0;
rebootWatchdogFile.img01Cnt = 0;
rebootWatchdogFile.img10Cnt = 0;
rebootWatchdogFile.img11Cnt = 0;
} else {
if (tgtChip == xsc::CHIP_0) {
if (tgtCopy == xsc::COPY_0) {
rebootFile.img00Cnt = 0;
rebootWatchdogFile.img00Cnt = 0;
} else {
rebootFile.img01Cnt = 0;
rebootWatchdogFile.img01Cnt = 0;
}
} else {
if (tgtCopy == xsc::COPY_0) {
rebootFile.img10Cnt = 0;
rebootWatchdogFile.img10Cnt = 0;
} else {
rebootFile.img11Cnt = 0;
rebootWatchdogFile.img11Cnt = 0;
}
}
}
rewriteRebootFile(rebootFile);
rewriteRebootWatchdogFile(rebootWatchdogFile);
}
void CoreController::rewriteRebootFile(RebootFile file) {
std::string path = sdcMan->getCurrentMountPrefix(sdInfo.active) + REBOOT_FILE;
void CoreController::rewriteRebootWatchdogFile(RebootWatchdogFile file) {
std::string path = sdcMan->getCurrentMountPrefix(sdInfo.active) + REBOOT_WATCHDOG_FILE;
std::ofstream rebootFile(path);
if (rebootFile.is_open()) {
// Initiate reboot file first. Reboot handling will be on on initialization
@ -450,15 +451,15 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_
if (size < 1) {
return HasActionsIF::INVALID_PARAMETERS;
}
std::string path = sdcMan->getCurrentMountPrefix(sdInfo.active) + REBOOT_FILE;
std::string path = sdcMan->getCurrentMountPrefix(sdInfo.active) + REBOOT_WATCHDOG_FILE;
// Disable the reboot file mechanism
parseRebootFile(path, rebootFile);
parseRebootWatchdogFile(path, rebootWatchdogFile);
if (data[0] == 0) {
rebootFile.enabled = false;
rewriteRebootFile(rebootFile);
rebootWatchdogFile.enabled = false;
rewriteRebootWatchdogFile(rebootWatchdogFile);
} else if (data[0] == 1) {
rebootFile.enabled = true;
rewriteRebootFile(rebootFile);
rebootWatchdogFile.enabled = true;
rewriteRebootWatchdogFile(rebootWatchdogFile);
} else {
return HasActionsIF::INVALID_PARAMETERS;
}
@ -490,11 +491,11 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_
if (size < 1) {
return HasActionsIF::INVALID_PARAMETERS;
}
std::string path = sdcMan->getCurrentMountPrefix(sdInfo.active) + REBOOT_FILE;
std::string path = sdcMan->getCurrentMountPrefix(sdInfo.active) + REBOOT_WATCHDOG_FILE;
// Disable the reboot file mechanism
parseRebootFile(path, rebootFile);
rebootFile.maxCount = data[0];
rewriteRebootFile(rebootFile);
parseRebootWatchdogFile(path, rebootWatchdogFile);
rebootWatchdogFile.maxCount = data[0];
rewriteRebootWatchdogFile(rebootWatchdogFile);
return HasActionsIF::EXECUTION_FINISHED;
}
default: {
@ -504,23 +505,23 @@ ReturnValue_t CoreController::executeAction(ActionId_t actionId, MessageQueueId_
}
void CoreController::setRebootMechanismLock(bool lock, xsc::Chip tgtChip, xsc::Copy tgtCopy) {
std::string path = sdcMan->getCurrentMountPrefix(sdInfo.active) + REBOOT_FILE;
std::string path = sdcMan->getCurrentMountPrefix(sdInfo.active) + REBOOT_WATCHDOG_FILE;
// Disable the reboot file mechanism
parseRebootFile(path, rebootFile);
parseRebootWatchdogFile(path, rebootWatchdogFile);
if (tgtChip == xsc::CHIP_0) {
if (tgtCopy == xsc::COPY_0) {
rebootFile.img00Lock = lock;
rebootWatchdogFile.img00Lock = lock;
} else {
rebootFile.img01Lock = lock;
rebootWatchdogFile.img01Lock = lock;
}
} else {
if (tgtCopy == xsc::COPY_0) {
rebootFile.img10Lock = lock;
rebootWatchdogFile.img10Lock = lock;
} else {
rebootFile.img11Lock = lock;
rebootWatchdogFile.img11Lock = lock;
}
}
rewriteRebootFile(rebootFile);
rewriteRebootWatchdogFile(rebootWatchdogFile);
}
void CoreController::setCurrentBootCopy(xsc::Chip chip, xsc::Copy copy) {

View File

@ -14,7 +14,7 @@ enum Copy : int { COPY_0, COPY_1, NO_COPY, SELF_COPY, ALL_COPY };
} // namespace xsc
struct RebootFile {
struct RebootWatchdogFile {
static constexpr uint8_t DEFAULT_MAX_BOOT_CNT = 10;
bool enabled = true;
@ -39,7 +39,7 @@ class SdCardManager;
class CoreController {
public:
static constexpr char REBOOT_FILE[] = "/conf/reboot.txt";
static constexpr char REBOOT_WATCHDOG_FILE[] = "/conf/reboot.txt";
//! [EXPORT] : [COMMENT] The reboot mechanism was triggered.
//! P1: First 16 bits: Last Chip, Last 16 bits: Last Copy,
//! P2: Each byte is the respective reboot count for the slots
@ -57,13 +57,13 @@ class CoreController {
static void setCurrentBootCopy(xsc::Chip chip, xsc::Copy copy);
ReturnValue_t executeAction(ActionId_t actionId, MessageQueueId_t commandedBy,
const uint8_t* data, size_t size);
void performRebootFileHandling(bool recreateFile);
void determineAndExecuteReboot(RebootFile& rf, bool& needsReboot, xsc::Chip& tgtChip,
void performRebootWatchdogHandling(bool recreateFile);
void determineAndExecuteReboot(RebootWatchdogFile& rf, bool& needsReboot, xsc::Chip& tgtChip,
xsc::Copy& tgtCopy);
void setRebootMechanismLock(bool lock, xsc::Chip tgtChip, xsc::Copy tgtCopy);
void resetRebootCount(xsc::Chip tgtChip, xsc::Copy tgtCopy);
bool parseRebootFile(std::string path, RebootFile& file);
void rewriteRebootFile(RebootFile file);
bool parseRebootWatchdogFile(std::string path, RebootWatchdogFile& file);
void rewriteRebootWatchdogFile(RebootWatchdogFile file);
private:
struct SdFsmParams {
@ -74,6 +74,6 @@ class CoreController {
} sdInfo;
SdCardManager* sdcMan = nullptr;
RebootFile rebootFile = {};
RebootWatchdogFile rebootWatchdogFile = {};
bool doPerformRebootFileHandling = true;
};