FDIR extensions and improvements for MGM device handler

satrs-example/src/acs/mgm.rs

@@ -1,5 +1,5 @@
-use satrs::fdir::FaultCounterStd;
-use satrs::health::{HealthState, HealthTableMapSync, HealthTableProvider};
+use satrs::fdir::{FaultCounterStd, FaultResponse, RecoveryEvent, RecoveryFdir};
+use satrs::health::HealthTableMapSync;
 use satrs::spacepackets::CcsdsPacketIdAndPsc;
 use satrs_example::{HkHelperSingleSet, TimestampHelper, TmtcQueues};
 use satrs_minisim::acs::MgmRequestLis3Mdl;
@@ -36,6 +36,13 @@ pub const Z_LOWBYTE_IDX: usize = 13;
 pub const SPI_FAULT_THRESHOLD: u32 = 2;
 pub const SPI_FAULT_DECREMENT_AFTER: Duration = Duration::from_secs(30);
 
+// FDIR configuration for power cycle recoveries. A second recovery before the counter was
+// decremented again marks the component faulty.
+pub const RECOVERY_THRESHOLD: u32 = 1;
+pub const RECOVERY_DECREMENT_AFTER: Duration = Duration::from_secs(30 * 60);
+/// Time the device stays unpowered during a power cycle, so it can fully discharge.
+pub const RECOVERY_OFF_DURATION: Duration = Duration::from_secs(1);
+
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum MgmId {
     _0,
@@ -158,6 +165,14 @@ pub struct ModeLeafHelper {
 }
 
 /// Example MGM device handler strongly based on the LIS3MDL MEMS device.
+///
+/// This device handler includes several components beyond the scope of reading sensor values:
+///
+/// - FDIR handling on communication issues.
+/// - FDIR helper for power cycling the device on communication issues.
+/// - Event generation for certain events like communication issues.
+/// - HK helper for periodic data generation.
+/// - Mode leaf helper to allow integration into a full ACS mode tree
 pub struct MgmHandlerLis3Mdl {
     id: MgmId,
     tmtc_queues: TmtcQueues,
@@ -169,7 +184,8 @@ pub struct MgmHandlerLis3Mdl {
     switch_and_mode_helper: SwitchAndModeHelper<DeviceMode>,
     mode_leaf_helper: ModeLeafHelper,
     spi_fault_counter: FaultCounterStd,
-    health_table: HealthTableMapSync,
+    fdir: RecoveryFdir<HealthTableMapSync>,
+    recovery_off_duration: Duration,
     event_tx: mpsc::SyncSender<(ComponentId, mgm::Event)>,
 }
 
@@ -202,7 +218,13 @@ impl MgmHandlerLis3Mdl {
             hk_helper: HkHelperSingleSet::new(false, Duration::from_millis(200)),
             mode_leaf_helper,
             spi_fault_counter: FaultCounterStd::new(SPI_FAULT_THRESHOLD, SPI_FAULT_DECREMENT_AFTER),
-            health_table,
+            fdir: RecoveryFdir::new(
+                id.component_id().into(),
+                health_table,
+                RECOVERY_THRESHOLD,
+                RECOVERY_DECREMENT_AFTER,
+            ),
+            recovery_off_duration: RECOVERY_OFF_DURATION,
             event_tx,
         }
     }
@@ -222,6 +244,9 @@ impl MgmHandlerLis3Mdl {
         // Handle assembly related messages.
         self.handle_mode_leaf_handling();
 
+        self.fdir.periodic_operation();
+        self.check_needs_recovery();
+
         // Handle mode transitions first.
         if let Some(event) = self.switch_and_mode_helper.handle_mode_transition() {
             match event {
@@ -231,11 +256,19 @@ impl MgmHandlerLis3Mdl {
                 ModeTransitionEvent::Failed(tc_commander) => {
                     self.handle_mode_transition_failure(tc_commander)
                 }
+                // The mode did not change for other components, so there is nothing to report.
+                ModeTransitionEvent::PowerCycleDone => self.handle_recovery_done(),
+                ModeTransitionEvent::PowerCycleFailed => {
+                    self.handle_recovery_failure();
+                    self.announce_mode();
+                    self.report_mode_to_parent();
+                }
             }
         }
 
-        // Poll sensor before checking and generating HK.
-        if self.mode() == DeviceMode::Normal {
+        // Poll sensor before checking and generating HK. Faults are expected while the device
+        // is power cycled, so it is not polled during a recovery.
+        if self.mode() == DeviceMode::Normal && !self.switch_and_mode_helper.power_cycle_active() {
             log::trace!("polling LIS3MDL sensor {}", self.id.str());
             self.poll_sensor();
         }
@@ -267,12 +300,12 @@ impl MgmHandlerLis3Mdl {
                                 }
                                 mgm::request::Request::Mode(device_mode) => match device_mode {
                                     ModeRequest::SetMode(device_mode) => {
-                                        self.start_transition(device_mode, Some(tc_id));
+                                        self.handle_mode_command(device_mode, Some(tc_id));
                                     }
                                     ModeRequest::ReadMode => self.send_telemetry(
                                         Some(tc_id),
                                         mgm::response::Response::Mode(ModeResponse::Mode(
-                                            self.mode(),
+                                            self.switch_and_mode_helper.reported_mode(),
                                         )),
                                     ),
                                 },
@@ -284,10 +317,7 @@ impl MgmHandlerLis3Mdl {
                                                 self.id.str(),
                                                 health_state
                                             );
-                                            self.health_table.set_health(
-                                                self.id.component_id().into(),
-                                                health_state,
-                                            );
+                                            self.fdir.set_health(health_state);
                                             self.send_telemetry(
                                                 Some(tc_id),
                                                 mgm::response::Response::Ok,
@@ -316,7 +346,9 @@ impl MgmHandlerLis3Mdl {
         loop {
             match self.mode_leaf_helper.request_rx.try_recv() {
                 Ok(request) => match request {
-                    ModeRequest::SetMode(device_mode) => self.start_transition(device_mode, None),
+                    ModeRequest::SetMode(device_mode) => {
+                        self.handle_mode_command(device_mode, None)
+                    }
                     ModeRequest::ReadMode => self.report_mode_to_parent(),
                 },
                 Err(e) => match e {
@@ -419,38 +451,38 @@ impl MgmHandlerLis3Mdl {
     }
 
     /// Registers one stuck-bus SPI fault with the FDIR fault counter, invalidating the current
-    /// sensor set. If the failure threshold is exceeded, the component is marked faulty in the
-    /// global health table.
+    /// sensor set. If the failure threshold is exceeded, the device is power cycled. If it was
+    /// power cycled too often, the component is marked faulty and commanded off instead.
     fn register_spi_fault(&mut self) {
         log::warn!("{}: stuck-bus SPI fault", self.id.str());
         self.shared_mgm_set.lock().unwrap().valid = false;
         if !self.spi_fault_counter.increment_and_check() {
             return;
         }
-        // Ground may have taken manual control, or already given up on this component.
-        // Autonomous FDIR should not override that decision.
-        let component_id = self.id.component_id().into();
-        match self.health_table.health(component_id) {
-            Some(HealthState::ExternalControl) | Some(HealthState::PermanentFaulty) => {
+        match self.fdir.handle_fault() {
+            FaultResponse::Ignored => {
                 log::info!(
                     "{}: SPI fault threshold exceeded, but health is externally controlled, \
                      not overriding",
                     self.id.str()
                 );
             }
-            _ => {
-                log::error!(
-                    "{}: SPI fault threshold exceeded, marking component faulty",
+            FaultResponse::Recover => {
+                log::warn!(
+                    "{}: SPI fault threshold exceeded, power cycling device",
                     self.id.str()
                 );
-                self.health_table
-                    .set_health(component_id, HealthState::Faulty);
-                if let Err(e) = self.event_tx.send((
-                    self.id.component_id(),
-                    mgm::Event::SpiFaultThresholdExceeded,
-                )) {
-                    log::warn!("{}: failed to send fault event: {}", self.id.str(), e);
-                }
+                self.send_event(mgm::Event::SpiFaultThresholdExceeded);
+                self.check_needs_recovery();
+            }
+            FaultResponse::SetFaulty => {
+                log::error!(
+                    "{}: SPI fault threshold exceeded after too many recoveries, marking \
+                     component faulty",
+                    self.id.str()
+                );
+                self.send_event(mgm::Event::SpiFaultThresholdExceeded);
+                self.send_event(mgm::Event::Recovery(RecoveryEvent::ThresholdExceeded));
                 // Do not restart an already pending Off transition: poll_sensor still calls
                 // this every cycle the fault persists, and current stays Normal until the
                 // transition completes, so re-triggering here would keep resetting the
@@ -463,6 +495,67 @@ impl MgmHandlerLis3Mdl {
         }
     }
 
+    /// Starts a power cycle if the health is [satrs::health::HealthState::NeedsRecovery]. The health is set
+    /// either by the FDIR or by ground.
+    fn check_needs_recovery(&mut self) {
+        if self.switch_and_mode_helper.power_cycle_active()
+            || self.switch_and_mode_helper.target().is_some()
+            || !self.fdir.needs_recovery()
+        {
+            return;
+        }
+        if self.mode() == DeviceMode::Off {
+            // Nothing to power cycle, the next switch-on is a fresh start anyway.
+            log::info!("{}: device is off, no recovery required", self.id.str());
+            self.fdir.recovery_done();
+            return;
+        }
+        log::warn!("{}: starting power cycle recovery", self.id.str());
+        self.spi_fault_counter.clear();
+        self.shared_mgm_set.lock().unwrap().valid = false;
+        self.switch_and_mode_helper
+            .start_power_cycle(self.recovery_off_duration);
+        self.send_event(mgm::Event::Recovery(RecoveryEvent::Started));
+    }
+
+    fn handle_recovery_done(&mut self) {
+        log::info!("{}: power cycle recovery done", self.id.str());
+        self.fdir.recovery_done();
+        self.send_event(mgm::Event::Recovery(RecoveryEvent::Done));
+    }
+
+    fn handle_recovery_failure(&mut self) {
+        log::error!(
+            "{}: power cycle recovery failed, marking component faulty",
+            self.id.str()
+        );
+        self.fdir.recovery_failed();
+        self.send_event(mgm::Event::Recovery(RecoveryEvent::Failed));
+    }
+
+    /// Mode commands from ground or the parent abort a running recovery.
+    fn handle_mode_command(
+        &mut self,
+        target_mode: DeviceMode,
+        tc_commander: Option<CcsdsPacketIdAndPsc>,
+    ) {
+        if self.switch_and_mode_helper.power_cycle_active() {
+            log::warn!(
+                "{}: mode command aborts power cycle recovery",
+                self.id.str()
+            );
+            // Otherwise, the recovery would restart right away.
+            self.fdir.recovery_done();
+        }
+        self.start_transition(target_mode, tc_commander);
+    }
+
+    fn send_event(&self, event: mgm::Event) {
+        if let Err(e) = self.event_tx.send((self.id.component_id(), event)) {
+            log::warn!("{}: failed to send event {:?}: {}", self.id.str(), event, e);
+        }
+    }
+
     fn start_transition(
         &mut self,
         target_mode: DeviceMode,
@@ -502,22 +595,15 @@ impl MgmHandlerLis3Mdl {
 
     fn announce_mode(&self) {
         log::info!("{} announcing mode: {:?}", self.id.str(), self.mode());
-        if let Err(e) = self
-            .event_tx
-            .send((self.id.component_id(), mgm::Event::ModeChanged(self.mode())))
-        {
-            log::warn!(
-                "{}: failed to send mode changed event: {}",
-                self.id.str(),
-                e
-            );
-        }
+        self.send_event(mgm::Event::ModeChanged(self.mode()));
     }
 
     fn report_mode_to_parent(&self) {
         self.mode_leaf_helper
             .report_tx
-            .send(ModeResponse::Mode(self.mode()))
+            .send(ModeResponse::Mode(
+                self.switch_and_mode_helper.reported_mode(),
+            ))
             .unwrap();
     }
 
@@ -534,6 +620,7 @@ mod tests {
     };
 
     use arbitrary_int::u11;
+    use satrs::health::{HealthState, HealthTableProvider};
     use satrs::spacepackets::SpacePacketHeader;
     use satrs_minisim::acs::lis3mdl::MgmLis3RawValues;
     use types::{
@@ -589,7 +676,7 @@ mod tests {
     impl MgmTestbench {
         pub fn new() -> Self {
             let (assembly_mode_request_tx, assembly_mode_request_rx) = mpsc::sync_channel(5);
-            let (mode_report_tx, mode_report_rx) = mpsc::sync_channel(5);
+            let (mode_report_tx, mode_report_rx) = mpsc::sync_channel(10);
             let mode_leaf_helper = ModeLeafHelper {
                 request_rx: assembly_mode_request_rx,
                 report_tx: mode_report_tx,
@@ -603,8 +690,8 @@ mod tests {
             let switch_map = SwitchSet::new(switch_map);
             let shared_switch_set = SharedSwitchSet::new(Mutex::new(switch_map));
             let health_table = HealthTableMapSync::default();
-            let (event_tx, event_rx) = mpsc::sync_channel(5);
-            let handler = MgmHandlerLis3Mdl::new(
+            let (event_tx, event_rx) = mpsc::sync_channel(20);
+            let mut handler = MgmHandlerLis3Mdl::new(
                 MgmId::_0,
                 TmtcQueues { tc_rx, tm_tx },
                 PowerSwitchHelper::new(switcher_tx, shared_switch_set.clone()),
@@ -615,6 +702,7 @@ mod tests {
                 health_table.clone(),
                 event_tx,
             );
+            handler.recovery_off_duration = Duration::ZERO;
             Self {
                 assembly_mode_request_tx,
                 mode_report_rx,
@@ -645,6 +733,61 @@ mod tests {
             assert_eq!(self.handler.mode(), DeviceMode::Normal);
         }
 
+        pub fn set_switch_state(&self, state: SwitchState) {
+            self.shared_switch_set
+                .lock()
+                .unwrap()
+                .set_switch_state(SwitchId::Mgm0, state);
+        }
+
+        pub fn inject_stuck_bus(&mut self) {
+            self.test_spi_interface().next_mgm_data = MgmLis3RawValues {
+                x: -1,
+                y: -1,
+                z: -1,
+            };
+        }
+
+        /// Drives SPI faults until the SPI fault threshold is exceeded once.
+        pub fn exceed_spi_fault_threshold(&mut self) {
+            self.inject_stuck_bus();
+            for _ in 0..SPI_FAULT_THRESHOLD + 1 {
+                self.handler.periodic_operation();
+            }
+        }
+
+        /// Drives a started power cycle recovery to completion, completing both power-switch
+        /// handshakes.
+        pub fn complete_power_cycle(&mut self) {
+            self.handler.periodic_operation();
+            self.set_switch_state(SwitchState::Off);
+            self.handler.periodic_operation();
+            assert_eq!(self.handler.mode(), DeviceMode::Off);
+            self.handler.periodic_operation();
+            assert_eq!(
+                self.handler.switch_and_mode_helper.target(),
+                Some(DeviceMode::Normal)
+            );
+            self.set_switch_state(SwitchState::On);
+            self.handler.periodic_operation();
+            assert_eq!(self.handler.mode(), DeviceMode::Normal);
+        }
+
+        pub fn health(&self) -> Option<HealthState> {
+            self.health_table.health(ComponentId::AcsMgm0.into())
+        }
+
+        pub fn drain_events(&self) -> Vec<mgm::Event> {
+            self.event_rx.try_iter().map(|(_, event)| event).collect()
+        }
+
+        pub fn drain_switch_requests(&self) -> Vec<SwitchStateBinary> {
+            self.switch_rx
+                .try_iter()
+                .map(|req| req.target_state)
+                .collect()
+        }
+
         pub fn test_spi_interface(&mut self) -> &mut TestSpiInterface {
             match &mut self.handler.spi_com {
                 SpiCommunication::Dummy(_) | SpiCommunication::Sim(_) => {
@@ -894,73 +1037,211 @@ mod tests {
     }
 
     #[test]
-    fn test_spi_fault_above_threshold_marks_component_faulty() {
+    fn test_spi_fault_above_threshold_starts_recovery() {
         let mut testbench = MgmTestbench::new();
         testbench.switch_to_normal();
-        // Drain the mode changed event emitted by switch_to_normal().
-        testbench
-            .event_rx
-            .try_recv()
-            .expect("expected mode changed event");
-        testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues {
-            x: -1,
-            y: -1,
-            z: -1,
-        };
-        // SPI_FAULT_THRESHOLD is exceeded on the (threshold + 1)-th stuck-bus reading.
-        for _ in 0..SPI_FAULT_THRESHOLD + 1 {
-            testbench.handler.periodic_operation();
-        }
-        assert_eq!(
-            testbench.health_table.health(ComponentId::AcsMgm0.into()),
-            Some(HealthState::Faulty)
-        );
+        testbench.drain_events();
+        testbench.exceed_spi_fault_threshold();
+        assert_eq!(testbench.health(), Some(HealthState::NeedsRecovery));
         assert!(!testbench.handler.shared_mgm_set.lock().unwrap().valid);
-        let (sender_id, event) = testbench.event_rx.try_recv().expect("expected fault event");
-        assert_eq!(sender_id, ComponentId::AcsMgm0);
-        assert!(matches!(event, mgm::Event::SpiFaultThresholdExceeded));
+        let events = testbench.drain_events();
+        assert!(matches!(
+            events[..],
+            [
+                mgm::Event::SpiFaultThresholdExceeded,
+                mgm::Event::Recovery(RecoveryEvent::Started)
+            ]
+        ));
     }
 
     #[test]
-    fn test_spi_fault_above_threshold_commands_device_off() {
+    fn test_recovery_power_cycles_device() {
         let mut testbench = MgmTestbench::new();
         testbench.switch_to_normal();
-        // Drain the switch-on request left over from switch_to_normal().
-        testbench
-            .switch_rx
-            .try_recv()
-            .expect("no switch-on request sent");
-        testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues {
-            x: -1,
-            y: -1,
-            z: -1,
-        };
-        for _ in 0..SPI_FAULT_THRESHOLD + 1 {
+        testbench.drain_events();
+        testbench.drain_switch_requests();
+        testbench.mode_report_rx.try_iter().for_each(drop);
+        testbench.exceed_spi_fault_threshold();
+        testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues::default();
+        let call_count = testbench.test_spi_interface().call_count;
+
+        testbench.complete_power_cycle();
+
+        // The power cycle is hidden from the parent.
+        assert!(testbench.mode_report_rx.try_recv().is_err());
+
+        assert_eq!(testbench.health(), Some(HealthState::Healthy));
+        assert_eq!(
+            testbench.drain_switch_requests(),
+            [SwitchStateBinary::Off, SwitchStateBinary::On]
+        );
+        let events = testbench.drain_events();
+        assert!(matches!(
+            events[..],
+            [
+                mgm::Event::SpiFaultThresholdExceeded,
+                mgm::Event::Recovery(RecoveryEvent::Started),
+                mgm::Event::Recovery(RecoveryEvent::Done),
+            ]
+        ));
+        // The device is not polled during the power cycle, only in the last cycle.
+        assert_eq!(testbench.test_spi_interface().call_count, call_count + 1);
+        assert!(testbench.handler.shared_mgm_set.lock().unwrap().valid);
+    }
+
+    #[test]
+    fn test_repeated_recovery_marks_component_faulty() {
+        let mut testbench = MgmTestbench::new();
+        testbench.switch_to_normal();
+        testbench.exceed_spi_fault_threshold();
+        // Bus is still stuck after the power cycle.
+        testbench.complete_power_cycle();
+        testbench.drain_events();
+        testbench.drain_switch_requests();
+
+        // The last cycle of the power cycle already polled once.
+        for _ in 0..SPI_FAULT_THRESHOLD {
             testbench.handler.periodic_operation();
         }
-        assert_eq!(
-            testbench.health_table.health(ComponentId::AcsMgm0.into()),
-            Some(HealthState::Faulty)
-        );
-        // The Off transition was only started on the last iteration above, so it has not sent
-        // its switch-off request yet: drive one more cycle to let it do so.
-        testbench.handler.periodic_operation();
-        let switch_req = testbench
-            .switch_rx
-            .try_recv()
-            .expect("no switch-off request sent after fault");
-        assert_eq!(switch_req.switch_id, SwitchId::Mgm0);
-        assert_eq!(switch_req.target_state, SwitchStateBinary::Off);
+        assert_eq!(testbench.health(), Some(HealthState::Faulty));
+        let events = testbench.drain_events();
+        assert!(matches!(
+            events[..],
+            [
+                mgm::Event::SpiFaultThresholdExceeded,
+                mgm::Event::Recovery(RecoveryEvent::ThresholdExceeded)
+            ]
+        ));
 
-        // Simulate the PCDU acting on the switch-off request.
-        testbench
-            .shared_switch_set
-            .lock()
-            .unwrap()
-            .set_switch_state(SwitchId::Mgm0, SwitchState::Off);
         testbench.handler.periodic_operation();
-
+        assert_eq!(testbench.drain_switch_requests(), [SwitchStateBinary::Off]);
+        testbench.set_switch_state(SwitchState::Off);
+        testbench.handler.periodic_operation();
         assert_eq!(testbench.handler.mode(), DeviceMode::Off);
+        assert_eq!(testbench.health(), Some(HealthState::Faulty));
+    }
+
+    #[test]
+    fn test_ground_needs_recovery_power_cycles_device() {
+        let mut testbench = MgmTestbench::new();
+        testbench.switch_to_normal();
+        testbench.drain_events();
+        testbench
+            .tc_tx
+            .send(create_request_tc(
+                MgmSelect::_0,
+                mgm::request::Request::Health(mgm::request::HealthRequest::SetHealth(
+                    HealthState::NeedsRecovery,
+                )),
+            ))
+            .unwrap();
+        testbench.handler.periodic_operation();
+        assert_eq!(
+            testbench.handler.switch_and_mode_helper.target(),
+            Some(DeviceMode::Off)
+        );
+        testbench.set_switch_state(SwitchState::Off);
+        testbench.handler.periodic_operation();
+        assert_eq!(testbench.handler.mode(), DeviceMode::Off);
+        testbench.handler.periodic_operation();
+        testbench.set_switch_state(SwitchState::On);
+        testbench.handler.periodic_operation();
+        assert_eq!(testbench.handler.mode(), DeviceMode::Normal);
+        assert_eq!(testbench.health(), Some(HealthState::Healthy));
+        let events = testbench.drain_events();
+        assert!(matches!(
+            events[0],
+            mgm::Event::Recovery(RecoveryEvent::Started)
+        ));
+        assert!(matches!(
+            events.last(),
+            Some(mgm::Event::Recovery(RecoveryEvent::Done))
+        ));
+    }
+
+    #[test]
+    fn test_needs_recovery_while_off_sets_healthy() {
+        let mut testbench = MgmTestbench::new();
+        testbench
+            .health_table
+            .set_health(ComponentId::AcsMgm0.into(), HealthState::NeedsRecovery);
+        testbench.handler.periodic_operation();
+        assert_eq!(testbench.health(), Some(HealthState::Healthy));
+        assert!(testbench.drain_switch_requests().is_empty());
+    }
+
+    #[test]
+    fn test_recovery_switch_on_timeout_marks_component_faulty() {
+        let mut testbench = MgmTestbench::new();
+        testbench.switch_to_normal();
+        testbench.exceed_spi_fault_threshold();
+        testbench.handler.periodic_operation();
+        testbench.set_switch_state(SwitchState::Off);
+        testbench.handler.periodic_operation();
+        testbench.drain_events();
+        testbench.mode_report_rx.try_iter().for_each(drop);
+
+        // The switch never turns on again.
+        testbench.handler.periodic_operation();
+        std::thread::sleep(Duration::from_millis(110));
+        testbench.handler.periodic_operation();
+        assert_eq!(testbench.handler.mode(), DeviceMode::Off);
+        assert_eq!(testbench.health(), Some(HealthState::Faulty));
+        let events = testbench.drain_events();
+        assert!(matches!(
+            events[..],
+            [
+                mgm::Event::Recovery(RecoveryEvent::Failed),
+                mgm::Event::ModeChanged(DeviceMode::Off)
+            ]
+        ));
+        // A failed power cycle is not hidden from the parent.
+        assert!(matches!(
+            testbench.mode_report_rx.try_recv(),
+            Ok(ModeResponse::Mode(DeviceMode::Off))
+        ));
+    }
+
+    #[test]
+    fn test_read_mode_during_power_cycle_returns_restored_mode() {
+        let mut testbench = MgmTestbench::new();
+        testbench.switch_to_normal();
+        testbench.exceed_spi_fault_threshold();
+        testbench.handler.periodic_operation();
+        testbench.set_switch_state(SwitchState::Off);
+        // Keep the device off until the parent asked for its mode.
+        testbench.handler.recovery_off_duration = Duration::from_secs(60);
+        testbench.handler.periodic_operation();
+        assert_eq!(testbench.handler.mode(), DeviceMode::Off);
+        testbench.mode_report_rx.try_iter().for_each(drop);
+
+        testbench
+            .assembly_mode_request_tx
+            .send(ModeRequest::ReadMode)
+            .unwrap();
+        testbench.handler.periodic_operation();
+        assert!(matches!(
+            testbench.mode_report_rx.try_recv(),
+            Ok(ModeResponse::Mode(DeviceMode::Normal))
+        ));
+    }
+
+    #[test]
+    fn test_mode_command_aborts_recovery() {
+        let mut testbench = MgmTestbench::new();
+        testbench.switch_to_normal();
+        testbench.exceed_spi_fault_threshold();
+        testbench.handler.periodic_operation();
+        testbench.set_switch_state(SwitchState::Off);
+        testbench
+            .assembly_mode_request_tx
+            .send(ModeRequest::SetMode(DeviceMode::Off))
+            .unwrap();
+        testbench.handler.periodic_operation();
+        testbench.handler.periodic_operation();
+        assert_eq!(testbench.handler.mode(), DeviceMode::Off);
+        assert_eq!(testbench.handler.switch_and_mode_helper.target(), None);
+        assert_eq!(testbench.health(), Some(HealthState::Healthy));
     }
 
     #[test]

satrs-example/src/device_mode.rs

@@ -1,16 +1,21 @@
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use types::pcdu::SwitchId;
 
 use crate::eps::PowerSwitchHelper;
 
-/// Modes that distinguish a powered-off state from one or more powered-on states, so
+/// This is a helper trait required to make [SwitchAndModeHelper] generic.
+///
+/// It allows distinguish a powered-off state from one or more powered-on states, so
 /// [`SwitchAndModeHelper`] knows which way to drive the switch for a given target mode.
 pub trait PowerSwitchedMode: Copy + PartialEq {
+    const OFF: Self;
     fn requires_power(&self) -> bool;
 }
 
 impl PowerSwitchedMode for types::DeviceMode {
+    const OFF: Self = types::DeviceMode::Off;
+
     fn requires_power(&self) -> bool {
         *self != types::DeviceMode::Off
     }
@@ -24,6 +29,14 @@ enum SwitchTransitionState {
     Done,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum PowerCycleState<Mode> {
+    Idle,
+    SwitchingOff { restore_mode: Mode },
+    WaitingOff { restore_mode: Mode, since: Instant },
+    SwitchingOn,
+}
+
 /// Outcome of a pending mode transition, once [`SwitchAndModeHelper::handle_mode_transition`]
 /// has driven it to completion. Carries back whichever TC commanded the transition, if any, so
 /// the caller can reply to it -- what that reply looks like is handler-specific, so this stays
@@ -31,6 +44,11 @@ enum SwitchTransitionState {
 pub enum ModeTransitionEvent {
     Reached(Option<satrs::spacepackets::CcsdsPacketIdAndPsc>),
     Failed(Option<satrs::spacepackets::CcsdsPacketIdAndPsc>),
+    /// The power cycle completed and the mode before the power cycle was restored.
+    PowerCycleDone,
+    /// Power switching failed during the power cycle. The power cycle is not hidden anymore,
+    /// so [SwitchAndModeHelper::reported_mode] returns the actual mode again.
+    PowerCycleFailed,
 }
 
 /// Drives the on/off power-switch commanding state machine (Idle -> PowerSwitching -> Done)
@@ -43,6 +61,8 @@ pub struct SwitchAndModeHelper<Mode: PowerSwitchedMode> {
     mode_helper: satrs_example::ModeHelper<Mode, SwitchTransitionState>,
     switch_helper: PowerSwitchHelper,
     switch_id: SwitchId,
+    power_cycle: PowerCycleState<Mode>,
+    power_cycle_off_duration: Duration,
 }
 
 impl<Mode: PowerSwitchedMode> SwitchAndModeHelper<Mode> {
@@ -56,6 +76,8 @@ impl<Mode: PowerSwitchedMode> SwitchAndModeHelper<Mode> {
             mode_helper: satrs_example::ModeHelper::new(init_mode, timeout),
             switch_helper,
             switch_id,
+            power_cycle: PowerCycleState::Idle,
+            power_cycle_off_duration: Duration::ZERO,
         }
     }
 
@@ -69,16 +91,87 @@ impl<Mode: PowerSwitchedMode> SwitchAndModeHelper<Mode> {
         self.mode_helper.target
     }
 
+    /// Mode which should be reported to other components. A power cycle is hidden from them,
+    /// so this is the mode which is restored after the power cycle while one is active.
+    pub fn reported_mode(&self) -> Mode {
+        match self.power_cycle {
+            PowerCycleState::SwitchingOff { restore_mode }
+            | PowerCycleState::WaitingOff { restore_mode, .. } => restore_mode,
+            PowerCycleState::Idle | PowerCycleState::SwitchingOn => self.mode(),
+        }
+    }
+
+    #[inline]
+    pub fn power_cycle_active(&self) -> bool {
+        self.power_cycle != PowerCycleState::Idle
+    }
+
+    /// Starts a new transition, aborting a running power cycle.
     pub fn start_transition(
         &mut self,
         target_mode: Mode,
         tc_commander: Option<satrs::spacepackets::CcsdsPacketIdAndPsc>,
+    ) {
+        self.power_cycle = PowerCycleState::Idle;
+        self.start_transition_internal(target_mode, tc_commander);
+    }
+
+    /// Switches the device off, keeps it off for `off_duration` and then restores the current
+    /// mode. Reaching the intermediate off mode does not generate an event.
+    pub fn start_power_cycle(&mut self, off_duration: Duration) {
+        self.power_cycle = PowerCycleState::SwitchingOff {
+            restore_mode: self.mode(),
+        };
+        self.power_cycle_off_duration = off_duration;
+        self.start_transition_internal(Mode::OFF, None);
+    }
+
+    fn start_transition_internal(
+        &mut self,
+        target_mode: Mode,
+        tc_commander: Option<satrs::spacepackets::CcsdsPacketIdAndPsc>,
     ) {
         self.mode_helper.tc_commander = tc_commander;
         self.mode_helper.start(target_mode);
     }
 
     pub fn handle_mode_transition(&mut self) -> Option<ModeTransitionEvent> {
+        if let PowerCycleState::WaitingOff {
+            restore_mode,
+            since,
+        } = self.power_cycle
+            && since.elapsed() >= self.power_cycle_off_duration
+        {
+            self.power_cycle = PowerCycleState::SwitchingOn;
+            self.start_transition_internal(restore_mode, None);
+        }
+        let event = self.handle_switch_transition()?;
+        if self.power_cycle == PowerCycleState::Idle {
+            return Some(event);
+        }
+        match event {
+            ModeTransitionEvent::Reached(_) => match self.power_cycle {
+                PowerCycleState::SwitchingOff { restore_mode } => {
+                    self.power_cycle = PowerCycleState::WaitingOff {
+                        restore_mode,
+                        since: Instant::now(),
+                    };
+                    None
+                }
+                _ => {
+                    self.power_cycle = PowerCycleState::Idle;
+                    Some(ModeTransitionEvent::PowerCycleDone)
+                }
+            },
+            ModeTransitionEvent::Failed(_) => {
+                self.power_cycle = PowerCycleState::Idle;
+                Some(ModeTransitionEvent::PowerCycleFailed)
+            }
+            event => Some(event),
+        }
+    }
+
+    fn handle_switch_transition(&mut self) -> Option<ModeTransitionEvent> {
         let target_mode = self.mode_helper.target?;
         let switch_target_on = target_mode.requires_power();
         if self.mode_helper.transition_state == SwitchTransitionState::Idle {
@@ -110,3 +203,253 @@ impl<Mode: PowerSwitchedMode> SwitchAndModeHelper<Mode> {
         None
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::sync::{Arc, Mutex, mpsc};
+
+    use arbitrary_int::u11;
+    use satrs::spacepackets::{CcsdsPacketIdAndPsc, SpacePacketHeader};
+    use types::{
+        DeviceMode,
+        pcdu::{SwitchRequest, SwitchState, SwitchStateBinary},
+    };
+
+    use crate::eps::pcdu::{SharedSwitchSet, SwitchMap, SwitchSet};
+
+    use super::*;
+
+    const TIMEOUT: Duration = Duration::from_millis(50);
+
+    struct Testbench {
+        helper: SwitchAndModeHelper<DeviceMode>,
+        switch_rx: mpsc::Receiver<SwitchRequest>,
+        shared_switch_set: SharedSwitchSet,
+    }
+
+    impl Testbench {
+        fn new() -> Self {
+            let (switch_tx, switch_rx) = mpsc::sync_channel(10);
+            let mut switch_map = SwitchMap::new();
+            switch_map.insert(SwitchId::Mgm0, SwitchState::Off);
+            let shared_switch_set: SharedSwitchSet =
+                Arc::new(Mutex::new(SwitchSet::new(switch_map)));
+            Self {
+                helper: SwitchAndModeHelper::new(
+                    DeviceMode::Off,
+                    TIMEOUT,
+                    PowerSwitchHelper::new(switch_tx, shared_switch_set.clone()),
+                    SwitchId::Mgm0,
+                ),
+                switch_rx,
+                shared_switch_set,
+            }
+        }
+
+        fn set_switch_state(&self, state: SwitchState) {
+            self.shared_switch_set
+                .lock()
+                .unwrap()
+                .set_switch_state(SwitchId::Mgm0, state);
+        }
+
+        fn switch_requests(&self) -> Vec<SwitchStateBinary> {
+            self.switch_rx
+                .try_iter()
+                .map(|req| req.target_state)
+                .collect()
+        }
+
+        /// Drives a transition to `Normal` to completion.
+        fn switch_to_normal(&mut self) {
+            self.helper.start_transition(DeviceMode::Normal, None);
+            self.set_switch_state(SwitchState::On);
+            assert!(matches!(
+                self.helper.handle_mode_transition(),
+                Some(ModeTransitionEvent::Reached(None))
+            ));
+            self.switch_requests();
+        }
+
+        /// Starts a power cycle from `Normal` and drives it until the device is off.
+        fn power_cycle_until_off(&mut self, off_duration: Duration) {
+            self.switch_to_normal();
+            self.helper.start_power_cycle(off_duration);
+            assert!(self.helper.handle_mode_transition().is_none());
+            assert_eq!(self.switch_requests(), [SwitchStateBinary::Off]);
+            self.set_switch_state(SwitchState::Off);
+            assert!(self.helper.handle_mode_transition().is_none());
+            assert_eq!(self.helper.mode(), DeviceMode::Off);
+        }
+    }
+
+    fn tc_id() -> CcsdsPacketIdAndPsc {
+        CcsdsPacketIdAndPsc::new_from_ccsds_packet(&SpacePacketHeader::new_from_apid(u11::new(1)))
+    }
+
+    #[test]
+    fn test_no_transition() {
+        let mut tb = Testbench::new();
+        assert_eq!(tb.helper.mode(), DeviceMode::Off);
+        assert_eq!(tb.helper.target(), None);
+        assert!(tb.helper.handle_mode_transition().is_none());
+        assert!(tb.switch_requests().is_empty());
+        assert!(!tb.helper.power_cycle_active());
+        assert_eq!(tb.helper.reported_mode(), DeviceMode::Off);
+    }
+
+    #[test]
+    fn test_switch_on() {
+        let mut tb = Testbench::new();
+        tb.helper
+            .start_transition(DeviceMode::Normal, Some(tc_id()));
+        assert!(tb.helper.handle_mode_transition().is_none());
+        assert_eq!(tb.switch_requests(), [SwitchStateBinary::On]);
+        assert_eq!(tb.helper.mode(), DeviceMode::Off);
+        assert_eq!(tb.helper.target(), Some(DeviceMode::Normal));
+
+        tb.set_switch_state(SwitchState::On);
+        match tb.helper.handle_mode_transition() {
+            Some(ModeTransitionEvent::Reached(Some(id))) => assert_eq!(id, tc_id()),
+            _ => panic!("expected mode reached event with TC commander"),
+        }
+        assert_eq!(tb.helper.mode(), DeviceMode::Normal);
+        assert_eq!(tb.helper.target(), None);
+        assert!(tb.switch_requests().is_empty());
+    }
+
+    #[test]
+    fn test_switch_off() {
+        let mut tb = Testbench::new();
+        tb.switch_to_normal();
+        tb.helper.start_transition(DeviceMode::Off, None);
+        assert!(tb.helper.handle_mode_transition().is_none());
+        assert_eq!(tb.switch_requests(), [SwitchStateBinary::Off]);
+        tb.set_switch_state(SwitchState::Off);
+        assert!(matches!(
+            tb.helper.handle_mode_transition(),
+            Some(ModeTransitionEvent::Reached(None))
+        ));
+        assert_eq!(tb.helper.mode(), DeviceMode::Off);
+    }
+
+    #[test]
+    fn test_switch_already_in_target_state() {
+        let mut tb = Testbench::new();
+        tb.set_switch_state(SwitchState::On);
+        tb.helper.start_transition(DeviceMode::On, None);
+        assert!(matches!(
+            tb.helper.handle_mode_transition(),
+            Some(ModeTransitionEvent::Reached(None))
+        ));
+        // The switch command is still sent.
+        assert_eq!(tb.switch_requests(), [SwitchStateBinary::On]);
+        assert_eq!(tb.helper.mode(), DeviceMode::On);
+    }
+
+    #[test]
+    fn test_switch_timeout() {
+        let mut tb = Testbench::new();
+        tb.helper
+            .start_transition(DeviceMode::Normal, Some(tc_id()));
+        assert!(tb.helper.handle_mode_transition().is_none());
+        std::thread::sleep(TIMEOUT);
+        match tb.helper.handle_mode_transition() {
+            Some(ModeTransitionEvent::Failed(Some(id))) => assert_eq!(id, tc_id()),
+            _ => panic!("expected mode failed event with TC commander"),
+        }
+        assert_eq!(tb.helper.mode(), DeviceMode::Off);
+        assert_eq!(tb.helper.target(), None);
+    }
+
+    #[test]
+    fn test_power_cycle() {
+        let mut tb = Testbench::new();
+        tb.power_cycle_until_off(Duration::ZERO);
+        assert!(tb.helper.power_cycle_active());
+
+        // The off duration elapsed, so switching on starts right away.
+        assert!(tb.helper.handle_mode_transition().is_none());
+        assert_eq!(tb.switch_requests(), [SwitchStateBinary::On]);
+        assert_eq!(tb.helper.target(), Some(DeviceMode::Normal));
+        tb.set_switch_state(SwitchState::On);
+        assert!(matches!(
+            tb.helper.handle_mode_transition(),
+            Some(ModeTransitionEvent::PowerCycleDone)
+        ));
+        assert_eq!(tb.helper.mode(), DeviceMode::Normal);
+        assert!(!tb.helper.power_cycle_active());
+    }
+
+    #[test]
+    fn test_power_cycle_reports_restored_mode() {
+        let mut tb = Testbench::new();
+        tb.switch_to_normal();
+        tb.helper.start_power_cycle(Duration::from_secs(60));
+        assert_eq!(tb.helper.reported_mode(), DeviceMode::Normal);
+        tb.helper.handle_mode_transition();
+        tb.set_switch_state(SwitchState::Off);
+        tb.helper.handle_mode_transition();
+        assert_eq!(tb.helper.mode(), DeviceMode::Off);
+        assert_eq!(tb.helper.reported_mode(), DeviceMode::Normal);
+    }
+
+    #[test]
+    fn test_power_cycle_waits_off_duration() {
+        let mut tb = Testbench::new();
+        tb.power_cycle_until_off(Duration::from_secs(60));
+        for _ in 0..3 {
+            assert!(tb.helper.handle_mode_transition().is_none());
+        }
+        assert!(tb.switch_requests().is_empty());
+        assert_eq!(tb.helper.target(), None);
+        assert!(tb.helper.power_cycle_active());
+    }
+
+    #[test]
+    fn test_power_cycle_switch_off_timeout() {
+        let mut tb = Testbench::new();
+        tb.switch_to_normal();
+        tb.helper.start_power_cycle(Duration::ZERO);
+        assert!(tb.helper.handle_mode_transition().is_none());
+        std::thread::sleep(TIMEOUT);
+        assert!(matches!(
+            tb.helper.handle_mode_transition(),
+            Some(ModeTransitionEvent::PowerCycleFailed)
+        ));
+        assert_eq!(tb.helper.mode(), DeviceMode::Normal);
+        assert!(!tb.helper.power_cycle_active());
+    }
+
+    #[test]
+    fn test_power_cycle_switch_on_timeout() {
+        let mut tb = Testbench::new();
+        tb.power_cycle_until_off(Duration::ZERO);
+        assert!(tb.helper.handle_mode_transition().is_none());
+        std::thread::sleep(TIMEOUT);
+        assert!(matches!(
+            tb.helper.handle_mode_transition(),
+            Some(ModeTransitionEvent::PowerCycleFailed)
+        ));
+        assert_eq!(tb.helper.mode(), DeviceMode::Off);
+        assert!(!tb.helper.power_cycle_active());
+        // The failed power cycle is not hidden anymore.
+        assert_eq!(tb.helper.reported_mode(), DeviceMode::Off);
+    }
+
+    #[test]
+    fn test_transition_aborts_power_cycle() {
+        let mut tb = Testbench::new();
+        tb.power_cycle_until_off(Duration::from_secs(60));
+        tb.helper.start_transition(DeviceMode::On, Some(tc_id()));
+        assert!(!tb.helper.power_cycle_active());
+        assert_eq!(tb.helper.reported_mode(), DeviceMode::Off);
+        tb.set_switch_state(SwitchState::On);
+        // A regular transition event instead of a power cycle event.
+        assert!(matches!(
+            tb.helper.handle_mode_transition(),
+            Some(ModeTransitionEvent::Reached(Some(_)))
+        ));
+        assert_eq!(tb.helper.mode(), DeviceMode::On);
+    }
+}

satrs-example/types/src/acs/mgm.rs

@@ -62,11 +62,11 @@ pub struct SensorData {
 #[strum_discriminants(derive(num_enum::IntoPrimitive))]
 #[repr(u16)]
 pub enum Event {
-    /// The SPI fault counter exceeded its threshold, the component was marked faulty and
-    /// commanded off.
+    /// The SPI fault counter exceeded its threshold. Followed by a recovery event.
     SpiFaultThresholdExceeded,
     /// A commanded or autonomous mode transition completed.
     ModeChanged(crate::DeviceMode),
+    Recovery(satrs::fdir::RecoveryEvent),
 }
 
 impl crate::Message for Event {
@@ -77,7 +77,10 @@ impl crate::Message for Event {
 
 impl crate::EventId for Event {
     fn event_id(&self) -> u16 {
-        EventDiscriminants::from(self).into()
+        match self {
+            Event::Recovery(event) => crate::recovery_event_id(*event),
+            _ => EventDiscriminants::from(self).into(),
+        }
     }
 }
 

satrs-example/types/src/lib.rs

@@ -175,6 +175,14 @@ pub trait EventId {
     fn event_id(&self) -> u16;
 }
 
+/// Start of the event ID range for generic FDIR events, which are embedded into the event types
+/// of the components. These events have the same ID for all components.
+pub const FDIR_EVENT_ID_BASE: u16 = 0x100;
+
+pub const fn recovery_event_id(event: satrs::fdir::RecoveryEvent) -> u16 {
+    FDIR_EVENT_ID_BASE + event as u16
+}
+
 /// Generic device mode which covers the requirements of most devices.
 ///
 /// The states are related both to the physical and the logical state of the device. Some

satrs/CHANGELOG.md

@@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Added `hk` module helpers to track whether a single HK set needs regeneration:
   `SingleSetHkHelperStd` (`std`), `SingleSetHkHelperEmbassy` (new `embassy-time` feature),
   and `SingleSetHkHelperCountdown`, generic over the existing `Countdown` trait.
+- Added `fdir::RecoveryFdir` (`std`), which escalates component faults to a power cycle
+  recovery first and to a faulty component if it has to be recovered too often.
+- Added `fdir::RecoveryEvent` for recovery related events.
 
 # [v0.3.0-alpha.3] 2025-11-06
 

satrs/src/fdir.rs

@@ -15,8 +15,44 @@
     feature = "embassy-time",
     doc = "- [FaultCounterEmbassy]: `embassy_time::Instant`, behind the `embassy-time` feature."
 )]
+//!
+//! [RecoveryFdir] builds on top of that. It decides whether a component is power cycled or
+//! marked faulty when one of its fault counters exceeds its threshold, and keeps the health
+//! table up to date during the recovery. It follows the FSFW `DeviceHandlerFailureIsolation`.
 #![deny(missing_docs)]
 
+#[cfg(feature = "std")]
+use crate::health::{HealthState, HealthTableProvider};
+
+/// Events related to the recovery of a component. Components are expected to embed this into
+/// their own event type.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(feature = "defmt", derive(defmt::Format))]
+pub enum RecoveryEvent {
+    /// The component health is [crate::health::HealthState::NeedsRecovery] and it is being power cycled.
+    Started,
+    /// The power cycle completed and the component is healthy again.
+    Done,
+    /// The power cycle failed, the component was marked faulty.
+    Failed,
+    /// The component was recovered too often, it was marked faulty.
+    ThresholdExceeded,
+}
+
+/// Outcome of [RecoveryFdir::handle_fault].
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum FaultResponse {
+    /// The health is externally controlled or permanently faulty, so nothing was changed.
+    Ignored,
+    /// The health was set to [crate::health::HealthState::NeedsRecovery]. The component should be power
+    /// cycled.
+    Recover,
+    /// The component was recovered too often and its health was set to [crate::health::HealthState::Faulty].
+    /// The component should be switched off.
+    SetFaulty,
+}
+
 /// Fault counter backed by [std::time::Instant].
 #[cfg(feature = "std")]
 #[derive(Debug, Clone)]
@@ -230,6 +266,93 @@ impl FaultCounterEmbassy {
     }
 }
 
+/// Escalates faults of a component to a power cycle recovery first, and to a faulty
+/// component if it has to be recovered too often.
+///
+/// The component itself runs the power cycle while [Self::needs_recovery] returns `true` and
+/// reports the outcome with [Self::recovery_done] or [Self::recovery_failed]. Setting
+/// [HealthState::NeedsRecovery] from outside, for example by ground, triggers a recovery as well.
+#[cfg(feature = "std")]
+#[derive(Debug, Clone)]
+pub struct RecoveryFdir<HealthTable: HealthTableProvider> {
+    id: crate::ComponentId,
+    health_table: HealthTable,
+    recovery_counter: FaultCounterStd,
+}
+
+#[cfg(feature = "std")]
+impl<HealthTable: HealthTableProvider> RecoveryFdir<HealthTable> {
+    /// Create a new [RecoveryFdir] for component `id`.
+    ///
+    /// The component is marked faulty when it would be recovered more than `recovery_threshold`
+    /// times, with the recovery count being decremented every `recovery_decrement_after`.
+    pub fn new(
+        id: crate::ComponentId,
+        health_table: HealthTable,
+        recovery_threshold: u32,
+        recovery_decrement_after: core::time::Duration,
+    ) -> Self {
+        Self {
+            id,
+            health_table,
+            recovery_counter: FaultCounterStd::new(recovery_threshold, recovery_decrement_after),
+        }
+    }
+
+    /// Health of the component. Absent entries are returned as `None`.
+    pub fn health(&self) -> Option<HealthState> {
+        self.health_table.health(self.id)
+    }
+
+    /// Set the health of the component.
+    pub fn set_health(&mut self, health: HealthState) {
+        self.health_table.set_health(self.id, health);
+    }
+
+    /// Should be called periodically to decrement the recovery counter.
+    pub fn periodic_operation(&mut self) {
+        self.recovery_counter.try_decrement();
+    }
+
+    /// Should be called when a fault counter of the component exceeded its threshold.
+    pub fn handle_fault(&mut self) -> FaultResponse {
+        // Ground may have taken manual control, or already given up on this component.
+        // Autonomous FDIR should not override that decision.
+        if matches!(
+            self.health(),
+            Some(HealthState::ExternalControl) | Some(HealthState::PermanentFaulty)
+        ) {
+            return FaultResponse::Ignored;
+        }
+        if self.recovery_counter.increment_and_check() {
+            self.set_health(HealthState::Faulty);
+            return FaultResponse::SetFaulty;
+        }
+        self.set_health(HealthState::NeedsRecovery);
+        FaultResponse::Recover
+    }
+
+    /// The component should be power cycled.
+    pub fn needs_recovery(&self) -> bool {
+        self.health() == Some(HealthState::NeedsRecovery)
+    }
+
+    /// The power cycle completed, or was not required. Sets the health back to healthy.
+    pub fn recovery_done(&mut self) {
+        // The health might have changed during the recovery.
+        if self.needs_recovery() {
+            self.set_health(HealthState::Healthy);
+        }
+    }
+
+    /// The power cycle failed. Marks the component faulty.
+    pub fn recovery_failed(&mut self) {
+        if self.needs_recovery() {
+            self.set_health(HealthState::Faulty);
+        }
+    }
+}
+
 #[cfg(all(test, feature = "std"))]
 mod tests {
     use super::*;
@@ -281,6 +404,62 @@ mod tests {
         assert!(!fc.try_decrement());
     }
 
+    fn recovery_fdir() -> RecoveryFdir<crate::health::HealthTableMapSync> {
+        RecoveryFdir::new(
+            1,
+            crate::health::HealthTableMapSync::default(),
+            1,
+            Duration::from_secs(60),
+        )
+    }
+
+    #[test]
+    fn first_fault_triggers_recovery() {
+        let mut fdir = recovery_fdir();
+        assert_eq!(fdir.handle_fault(), FaultResponse::Recover);
+        assert!(fdir.needs_recovery());
+        fdir.recovery_done();
+        assert_eq!(fdir.health(), Some(HealthState::Healthy));
+    }
+
+    #[test]
+    fn repeated_recovery_sets_faulty() {
+        let mut fdir = recovery_fdir();
+        assert_eq!(fdir.handle_fault(), FaultResponse::Recover);
+        fdir.recovery_done();
+        assert_eq!(fdir.handle_fault(), FaultResponse::SetFaulty);
+        assert_eq!(fdir.health(), Some(HealthState::Faulty));
+    }
+
+    #[test]
+    fn failed_recovery_sets_faulty() {
+        let mut fdir = recovery_fdir();
+        fdir.handle_fault();
+        fdir.recovery_failed();
+        assert_eq!(fdir.health(), Some(HealthState::Faulty));
+    }
+
+    #[test]
+    fn external_health_is_not_overridden() {
+        let mut fdir = recovery_fdir();
+        for health in [HealthState::ExternalControl, HealthState::PermanentFaulty] {
+            fdir.set_health(health);
+            assert_eq!(fdir.handle_fault(), FaultResponse::Ignored);
+            assert_eq!(fdir.health(), Some(health));
+        }
+    }
+
+    #[test]
+    fn health_changed_during_recovery_is_kept() {
+        let mut fdir = recovery_fdir();
+        fdir.handle_fault();
+        fdir.set_health(HealthState::ExternalControl);
+        fdir.recovery_done();
+        assert_eq!(fdir.health(), Some(HealthState::ExternalControl));
+        fdir.recovery_failed();
+        assert_eq!(fdir.health(), Some(HealthState::ExternalControl));
+    }
+
     #[test]
     fn clear_resets_state() {
         let mut fc = FaultCounterStd::new(1, Duration::from_secs(60));