rust/sat-rs
12
1
Fork 0
Code Issues 6 Pull Requests 2 Projects Releases 29 Wiki Activity

FDIR extensions and improvements for MGM device handler

Browse Source
This commit is contained in:
Robin Mueller
2026-09-23 17:26:28 +02:00
parent 1c6e777d24
commit ea31e89433
9 changed files with 1116 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
satrs-example/client/src/main.rs
+22 -6
View File
@@ -4,7 +4,8 @@ use clap::Parser as _;
use satrs_example::config::{OBSW_SERVER_ADDR, SERVER_PORT};
use satrs_minisim::{
SerializableSimMsgPayload, SimComponent, SimCtrlReply, SimCtrlRequest, SimMessageProvider,
SimReply, SimRequest, acs::MgmRequestLis3Mdl, acs::SpiFaultMode, udp::SIM_CTRL_PORT,
SimReply, SimRequest, acs::MgmRequestLis3Mdl, acs::SpiFault, acs::SpiFaultMode,
udp::SIM_CTRL_PORT,
};
use spacepackets::{CcsdsPacketIdAndPsc, SpacePacketHeader};
use std::{
@@ -105,6 +106,15 @@ impl From<SpiFaultModeSelect> for SpiFaultMode {
}
}
#[derive(Debug, Default, PartialEq, Eq, Clone, Copy, clap::ValueEnum)]
enum SpiFaultKindSelect {
/// Cleared when the device is switched off, so a power cycle recovers from it.
Transient,
/// Survives power cycles.
#[default]
Permanent,
}
#[derive(Debug, PartialEq, Eq, Clone, Copy, clap::ValueEnum)]
enum HealthStateSelect {
Healthy,
@@ -140,6 +150,9 @@ struct MgmArgs {
/// regardless of which MGM the request names (a pre-existing minisim limitation).
#[arg(long, value_enum)]
spi_fault: Option<SpiFaultModeSelect>,
/// Whether a power cycle clears the injected SPI fault.
#[arg(long, value_enum, default_value_t)]
spi_fault_kind: SpiFaultKindSelect,
/// Override the device's FDIR health state, for example to clear a `Faulty` state set by
/// the handler after the underlying issue has been fixed or worked around.
#[arg(long, value_enum)]
@@ -191,7 +204,10 @@ fn handle_mgm_command(
if target_id != types::ComponentId::AcsMgm0 {
bail!("SPI fault injection is only supported for MGM0 right now (minisim limitation)");
}
inject_mgm_failure(mode.into())?;
inject_mgm_failure(SpiFault {
mode: mode.into(),
cleared_by_power_cycle: args.spi_fault_kind == SpiFaultKindSelect::Transient,
})?;
}
if args.ping {
let request = types::ccsds::CcsdsTcPacketOwned::new_with_request(
@@ -477,12 +493,12 @@ fn main() -> anyhow::Result<()> {
Ok(())
}
/// Injects the given SPI fault mode directly into minisim's MGM0 model, bypassing the OBSW.
/// Injects the given SPI fault directly into minisim's MGM0 model, bypassing the OBSW.
///
/// Confirms the simulator is actually reachable first (same ping/pong check the OBSW's own
/// internal sim client does, see `SimClientUdp::attempt_connection`), since a fire-and-forget
/// UDP send would otherwise silently do nothing if minisim is not running.
fn inject_mgm_failure(mode: SpiFaultMode) -> anyhow::Result<()> {
fn inject_mgm_failure(fault: SpiFault) -> anyhow::Result<()> {
let sim_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), SIM_CTRL_PORT);
let sim_socket = UdpSocket::bind("127.0.0.1:0")?;
sim_socket.set_read_timeout(Some(Duration::from_millis(200)))?;
@@ -514,9 +530,9 @@ fn inject_mgm_failure(mode: SpiFaultMode) -> anyhow::Result<()> {
Err(e) => return Err(e.into()),
}
let request = SimRequest::new_with_epoch_time(MgmRequestLis3Mdl::SetSpiFault(mode));
let request = SimRequest::new_with_epoch_time(MgmRequestLis3Mdl::SetSpiFault(fault));
sim_socket.send_to(&serde_json::to_vec(&request)?, sim_addr)?;
log::info!("injected SPI fault mode {mode:?} into minisim MGM0");
log::info!("injected SPI fault {fault:?} into minisim MGM0");
Ok(())
}
satrs-example/minisim/src/acs.rs
+63 -19
View File
@@ -7,7 +7,7 @@ use nexosim::{
use satrs_minisim::{
acs::{
lis3mdl::MgmLis3MdlReply, MgmReplyCommon, MgmReplyProvider, MgmSensorValuesMicroTesla,
MgtDipole, MgtHkSet, MgtReply, SpiFaultMode, MGT_GEN_MAGNETIC_FIELD,
MgtDipole, MgtHkSet, MgtReply, SpiFault, MGT_GEN_MAGNETIC_FIELD,
},
SimReply,
};
@@ -34,7 +34,7 @@ pub struct MagnetometerModel<ReplyProvider: MgmReplyProvider> {
#[allow(dead_code)]
pub periodicity: Duration,
pub external_mag_field: Option<MgmSensorValuesMicroTesla>,
pub spi_fault: SpiFaultMode,
pub spi_fault: SpiFault,
pub reply_sender: mpsc::Sender<SimReply>,
pub phatom: std::marker::PhantomData<ReplyProvider>,
}
@@ -45,7 +45,7 @@ impl MagnetometerModel<MgmLis3MdlReply> {
switch_state: SwitchStateBinary::Off,
periodicity,
external_mag_field: None,
spi_fault: SpiFaultMode::None,
spi_fault: SpiFault::default(),
reply_sender,
phatom: std::marker::PhantomData,
}
@@ -55,11 +55,14 @@ impl MagnetometerModel<MgmLis3MdlReply> {
impl<ReplyProvider: MgmReplyProvider> MagnetometerModel<ReplyProvider> {
pub async fn switch_device(&mut self, switch_state: SwitchStateBinary) {
self.switch_state = switch_state;
if switch_state == SwitchStateBinary::Off && self.spi_fault.cleared_by_power_cycle {
self.spi_fault = SpiFault::default();
}
}
/// Force (or clear) a stuck-bus SPI fault, for FDIR testing purposes.
pub async fn set_spi_fault(&mut self, fault_mode: SpiFaultMode) {
self.spi_fault = fault_mode;
pub async fn set_spi_fault(&mut self, fault: SpiFault) {
self.spi_fault = fault;
}
pub async fn send_sensor_values(&mut self, _: (), scheduler: &mut Context<Self>) {
@@ -70,7 +73,7 @@ impl<ReplyProvider: MgmReplyProvider> MagnetometerModel<ReplyProvider> {
sensor_values: self
.calculate_current_mgm_tuple(current_millis(scheduler.time())),
},
self.spi_fault,
self.spi_fault.mode,
))
.expect("sending MGM sensor values failed");
}
@@ -193,13 +196,16 @@ pub mod tests {
use satrs_minisim::{
acs::{
lis3mdl::{self, MgmLis3MdlReply},
MgmRequestLis3Mdl, MgtDipole, MgtHkSet, MgtReply, MgtRequest, SpiFaultMode,
MgmRequestLis3Mdl, MgtDipole, MgtHkSet, MgtReply, MgtRequest, SpiFault, SpiFaultMode,
},
SerializableSimMsgPayload, SimComponent, SimMessageProvider, SimRequest,
};
use types::pcdu::{SwitchId, SwitchStateBinary};
use crate::{eps::tests::switch_device_on, test_helpers::SimTestbench};
use crate::{
eps::tests::{switch_device_off, switch_device_on},
test_helpers::SimTestbench,
};
#[test]
fn test_basic_mgm_request() {
@@ -222,19 +228,20 @@ pub mod tests {
assert_eq!(reply.common.sensor_values.z, 0.0);
}
#[test]
fn test_mgm_spi_fault_injection_all_ones() {
let mut sim_testbench = SimTestbench::new();
switch_device_on(&mut sim_testbench, SwitchId::Mgm0);
fn inject_spi_fault(sim_testbench: &mut SimTestbench, cleared_by_power_cycle: bool) {
let fault_request =
SimRequest::new_with_epoch_time(MgmRequestLis3Mdl::SetSpiFault(SpiFaultMode::AllOnes));
SimRequest::new_with_epoch_time(MgmRequestLis3Mdl::SetSpiFault(SpiFault {
mode: SpiFaultMode::AllOnes,
cleared_by_power_cycle,
}));
sim_testbench
.send_request(fault_request)
.expect("sending MGM fault injection request failed");
sim_testbench.handle_sim_requests_time_agnostic();
sim_testbench.step().unwrap();
}
fn request_mgm_reply(sim_testbench: &mut SimTestbench) -> MgmLis3MdlReply {
let data_request = SimRequest::new_with_epoch_time(MgmRequestLis3Mdl::RequestSensorData);
sim_testbench
.send_request(data_request)
@@ -244,13 +251,50 @@ pub mod tests {
let sim_reply = sim_testbench
.try_receive_next_reply()
.expect("no MGM reply received");
let reply = MgmLis3MdlReply::from_sim_message(&sim_reply)
.expect("failed to deserialize MGM sensor values");
MgmLis3MdlReply::from_sim_message(&sim_reply)
.expect("failed to deserialize MGM sensor values")
}
fn is_stuck_bus_reply(reply: &MgmLis3MdlReply) -> bool {
reply.raw.x == -1 && reply.raw.y == -1 && reply.raw.z == -1
}
#[test]
fn test_mgm_spi_fault_injection_all_ones() {
let mut sim_testbench = SimTestbench::new();
switch_device_on(&mut sim_testbench, SwitchId::Mgm0);
inject_spi_fault(&mut sim_testbench, false);
let reply = request_mgm_reply(&mut sim_testbench);
// Even though the device is switched on, the injected fault forces a stuck-bus reply.
assert_eq!(reply.common.switch_state, SwitchStateBinary::On);
assert_eq!(reply.raw.x, -1);
assert_eq!(reply.raw.y, -1);
assert_eq!(reply.raw.z, -1);
assert!(is_stuck_bus_reply(&reply));
}
#[test]
fn test_mgm_spi_fault_cleared_by_power_cycle() {
let mut sim_testbench = SimTestbench::new();
switch_device_on(&mut sim_testbench, SwitchId::Mgm0);
inject_spi_fault(&mut sim_testbench, true);
assert!(is_stuck_bus_reply(&request_mgm_reply(&mut sim_testbench)));
switch_device_off(&mut sim_testbench, SwitchId::Mgm0);
switch_device_on(&mut sim_testbench, SwitchId::Mgm0);
sim_testbench.step_until(Duration::from_millis(50)).unwrap();
assert!(!is_stuck_bus_reply(&request_mgm_reply(&mut sim_testbench)));
}
#[test]
fn test_mgm_spi_fault_persists_after_power_cycle() {
let mut sim_testbench = SimTestbench::new();
switch_device_on(&mut sim_testbench, SwitchId::Mgm0);
inject_spi_fault(&mut sim_testbench, false);
switch_device_off(&mut sim_testbench, SwitchId::Mgm0);
switch_device_on(&mut sim_testbench, SwitchId::Mgm0);
let reply = request_mgm_reply(&mut sim_testbench);
assert_eq!(reply.common.switch_state, SwitchStateBinary::On);
assert!(is_stuck_bus_reply(&reply));
}
#[test]
satrs-example/minisim/src/lib.rs
+9 -1
View File
@@ -217,12 +217,20 @@ pub mod acs {
AllOnes,
}
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SpiFault {
pub mode: SpiFaultMode,
/// The fault is cleared when the device is switched off, so a power cycle recovers
/// from it.
pub cleared_by_power_cycle: bool,
}
#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
pub enum MgmRequestLis3Mdl {
RequestSensorData,
/// Force the raw register reply into a stuck-bus pattern, regardless of switch state.
/// Used to test FDIR handling of SPI bus faults.
SetSpiFault(SpiFaultMode),
SetSpiFault(SpiFault),
}
impl SerializableSimMsgPayload<SimRequest> for MgmRequestLis3Mdl {
satrs-example/src/acs/mgm.rs
+448 -100
View File
@@ -1,5 +1,5 @@
use satrs::fdir::FaultCounterStd;
use satrs::health::{HealthState, HealthTableMapSync, HealthTableProvider};
use satrs::fdir::{FaultCounterStd, FaultResponse, RecoveryEvent, RecoveryFdir};
use satrs::health::{HealthState, HealthTableMapSync};
use satrs::spacepackets::CcsdsPacketIdAndPsc;
use satrs_example::{HkHelperSingleSet, TimestampHelper, TmtcQueues};
use satrs_minisim::acs::MgmRequestLis3Mdl;
@@ -36,6 +36,13 @@ pub const Z_LOWBYTE_IDX: usize = 13;
pub const SPI_FAULT_THRESHOLD: u32 = 2;
pub const SPI_FAULT_DECREMENT_AFTER: Duration = Duration::from_secs(30);
// FDIR configuration for power cycle recoveries. A second recovery before the counter was
// decremented again marks the component faulty.
pub const RECOVERY_THRESHOLD: u32 = 1;
pub const RECOVERY_DECREMENT_AFTER: Duration = Duration::from_secs(30 * 60);
/// Time the device stays unpowered during a power cycle, so it can fully discharge.
pub const RECOVERY_OFF_DURATION: Duration = Duration::from_secs(1);
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum MgmId {
_0,
@@ -158,6 +165,14 @@ pub struct ModeLeafHelper {
}
/// Example MGM device handler strongly based on the LIS3MDL MEMS device.
///
/// This device handler includes several components beyond the scope of reading sensor values:
///
/// - FDIR handling on communication issues.
/// - FDIR helper for power cycling the device on communication issues.
/// - Event generation for certain events like communication issues.
/// - HK helper for periodic data generation.
/// - Mode leaf helper to allow integration into a full ACS mode tree
pub struct MgmHandlerLis3Mdl {
id: MgmId,
tmtc_queues: TmtcQueues,
@@ -169,7 +184,8 @@ pub struct MgmHandlerLis3Mdl {
switch_and_mode_helper: SwitchAndModeHelper<DeviceMode>,
mode_leaf_helper: ModeLeafHelper,
spi_fault_counter: FaultCounterStd,
health_table: HealthTableMapSync,
fdir: RecoveryFdir<HealthTableMapSync>,
recovery_off_duration: Duration,
event_tx: mpsc::SyncSender<(ComponentId, mgm::Event)>,
}
@@ -202,7 +218,13 @@ impl MgmHandlerLis3Mdl {
hk_helper: HkHelperSingleSet::new(false, Duration::from_millis(200)),
mode_leaf_helper,
spi_fault_counter: FaultCounterStd::new(SPI_FAULT_THRESHOLD, SPI_FAULT_DECREMENT_AFTER),
health_table,
fdir: RecoveryFdir::new(
id.component_id().into(),
health_table,
RECOVERY_THRESHOLD,
RECOVERY_DECREMENT_AFTER,
),
recovery_off_duration: RECOVERY_OFF_DURATION,
event_tx,
}
}
@@ -222,6 +244,9 @@ impl MgmHandlerLis3Mdl {
// Handle assembly related messages.
self.handle_mode_leaf_handling();
self.fdir.periodic_operation();
self.check_needs_recovery();
// Handle mode transitions first.
if let Some(event) = self.switch_and_mode_helper.handle_mode_transition() {
match event {
@@ -231,11 +256,15 @@ impl MgmHandlerLis3Mdl {
ModeTransitionEvent::Failed(tc_commander) => {
self.handle_mode_transition_failure(tc_commander)
}
// The mode did not change for other components, so there is nothing to report.
ModeTransitionEvent::PowerCycleDone => self.handle_recovery_done(),
ModeTransitionEvent::PowerCycleFailed => self.handle_recovery_failure(),
}
}
// Poll sensor before checking and generating HK.
if self.mode() == DeviceMode::Normal {
// Poll sensor before checking and generating HK. Faults are expected while the device
// is power cycled, so it is not polled during a recovery.
if self.mode() == DeviceMode::Normal && !self.switch_and_mode_helper.power_cycle_active() {
log::trace!("polling LIS3MDL sensor {}", self.id.str());
self.poll_sensor();
}
@@ -267,12 +296,12 @@ impl MgmHandlerLis3Mdl {
}
mgm::request::Request::Mode(device_mode) => match device_mode {
ModeRequest::SetMode(device_mode) => {
self.start_transition(device_mode, Some(tc_id));
self.handle_mode_command(device_mode, Some(tc_id));
}
ModeRequest::ReadMode => self.send_telemetry(
Some(tc_id),
mgm::response::Response::Mode(ModeResponse::Mode(
self.mode(),
self.switch_and_mode_helper.reported_mode(),
)),
),
},
@@ -284,10 +313,7 @@ impl MgmHandlerLis3Mdl {
self.id.str(),
health_state
);
self.health_table.set_health(
self.id.component_id().into(),
health_state,
);
self.fdir.set_health(health_state);
self.send_telemetry(
Some(tc_id),
mgm::response::Response::Ok,
@@ -316,7 +342,9 @@ impl MgmHandlerLis3Mdl {
loop {
match self.mode_leaf_helper.request_rx.try_recv() {
Ok(request) => match request {
ModeRequest::SetMode(device_mode) => self.start_transition(device_mode, None),
ModeRequest::SetMode(device_mode) => {
self.handle_mode_command(device_mode, None)
}
ModeRequest::ReadMode => self.report_mode_to_parent(),
},
Err(e) => match e {
@@ -419,38 +447,38 @@ impl MgmHandlerLis3Mdl {
}
/// Registers one stuck-bus SPI fault with the FDIR fault counter, invalidating the current
/// sensor set. If the failure threshold is exceeded, the component is marked faulty in the
/// global health table.
/// sensor set. If the failure threshold is exceeded, the device is power cycled. If it was
/// power cycled too often, the component is marked faulty and commanded off instead.
fn register_spi_fault(&mut self) {
log::warn!("{}: stuck-bus SPI fault", self.id.str());
self.shared_mgm_set.lock().unwrap().valid = false;
if !self.spi_fault_counter.increment_and_check() {
return;
}
// Ground may have taken manual control, or already given up on this component.
// Autonomous FDIR should not override that decision.
let component_id = self.id.component_id().into();
match self.health_table.health(component_id) {
Some(HealthState::ExternalControl) | Some(HealthState::PermanentFaulty) => {
match self.fdir.handle_fault() {
FaultResponse::Ignored => {
log::info!(
"{}: SPI fault threshold exceeded, but health is externally controlled, \
not overriding",
"{}: SPI fault threshold exceeded, but component is already faulty, \
recovering or externally controlled",
self.id.str()
);
}
_ => {
log::error!(
"{}: SPI fault threshold exceeded, marking component faulty",
FaultResponse::Recover => {
log::warn!(
"{}: SPI fault threshold exceeded, power cycling device",
self.id.str()
);
self.health_table
.set_health(component_id, HealthState::Faulty);
if let Err(e) = self.event_tx.send((
self.id.component_id(),
mgm::Event::SpiFaultThresholdExceeded,
)) {
log::warn!("{}: failed to send fault event: {}", self.id.str(), e);
}
self.send_event(mgm::Event::SpiFaultThresholdExceeded);
self.check_needs_recovery();
}
FaultResponse::SetFaulty => {
log::error!(
"{}: SPI fault threshold exceeded after too many recoveries, marking \
component faulty",
self.id.str()
);
self.send_event(mgm::Event::SpiFaultThresholdExceeded);
self.send_event(mgm::Event::Recovery(RecoveryEvent::ThresholdExceeded));
// Do not restart an already pending Off transition: poll_sensor still calls
// this every cycle the fault persists, and current stays Normal until the
// transition completes, so re-triggering here would keep resetting the
@@ -463,6 +491,77 @@ impl MgmHandlerLis3Mdl {
}
}
/// Starts a power cycle if the health is [satrs::health::HealthState::NeedsRecovery]. The health is set
/// either by the FDIR or by ground.
fn check_needs_recovery(&mut self) {
if self.switch_and_mode_helper.power_cycle_active()
|| self.switch_and_mode_helper.target().is_some()
|| !self.fdir.needs_recovery()
{
return;
}
if self.mode() == DeviceMode::Off {
// Nothing to power cycle, the next switch-on is a fresh start anyway.
log::info!("{}: device is off, no recovery required", self.id.str());
self.fdir.recovery_done();
return;
}
log::warn!("{}: starting power cycle recovery", self.id.str());
self.spi_fault_counter.clear();
self.shared_mgm_set.lock().unwrap().valid = false;
self.switch_and_mode_helper
.start_power_cycle(self.recovery_off_duration);
self.send_event(mgm::Event::Recovery(RecoveryEvent::Started));
}
fn handle_recovery_done(&mut self) {
log::info!("{}: power cycle recovery done", self.id.str());
self.fdir.recovery_done();
self.send_event(mgm::Event::Recovery(RecoveryEvent::Done));
}
fn handle_recovery_failure(&mut self) {
log::error!(
"{}: power cycle recovery failed, marking component faulty",
self.id.str()
);
self.fdir.recovery_failed();
self.send_event(mgm::Event::Recovery(RecoveryEvent::Failed));
if self.mode() == DeviceMode::Off {
// Switching back on failed. Other components still assume the mode before the
// power cycle.
self.announce_mode();
self.report_mode_to_parent();
} else if self.fdir.health() == Some(HealthState::Faulty) {
// Switching off failed. Retry like for any other faulty component.
log::warn!("{}: commanding device off due to fault", self.id.str());
self.start_transition(DeviceMode::Off, None);
}
}
/// Mode commands from ground or the parent abort a running recovery.
fn handle_mode_command(
&mut self,
target_mode: DeviceMode,
tc_commander: Option<CcsdsPacketIdAndPsc>,
) {
if self.switch_and_mode_helper.power_cycle_active() {
log::warn!(
"{}: mode command aborts power cycle recovery",
self.id.str()
);
// Otherwise, the recovery would restart right away.
self.fdir.recovery_done();
}
self.start_transition(target_mode, tc_commander);
}
fn send_event(&self, event: mgm::Event) {
if let Err(e) = self.event_tx.send((self.id.component_id(), event)) {
log::warn!("{}: failed to send event {:?}: {}", self.id.str(), event, e);
}
}
fn start_transition(
&mut self,
target_mode: DeviceMode,
@@ -502,22 +601,15 @@ impl MgmHandlerLis3Mdl {
fn announce_mode(&self) {
log::info!("{} announcing mode: {:?}", self.id.str(), self.mode());
if let Err(e) = self
.event_tx
.send((self.id.component_id(), mgm::Event::ModeChanged(self.mode())))
{
log::warn!(
"{}: failed to send mode changed event: {}",
self.id.str(),
e
);
}
self.send_event(mgm::Event::ModeChanged(self.mode()));
}
fn report_mode_to_parent(&self) {
self.mode_leaf_helper
.report_tx
.send(ModeResponse::Mode(self.mode()))
.send(ModeResponse::Mode(
self.switch_and_mode_helper.reported_mode(),
))
.unwrap();
}
@@ -534,6 +626,7 @@ mod tests {
};
use arbitrary_int::u11;
use satrs::health::HealthTableProvider;
use satrs::spacepackets::SpacePacketHeader;
use satrs_minisim::acs::lis3mdl::MgmLis3RawValues;
use types::{
@@ -589,7 +682,7 @@ mod tests {
impl MgmTestbench {
pub fn new() -> Self {
let (assembly_mode_request_tx, assembly_mode_request_rx) = mpsc::sync_channel(5);
let (mode_report_tx, mode_report_rx) = mpsc::sync_channel(5);
let (mode_report_tx, mode_report_rx) = mpsc::sync_channel(10);
let mode_leaf_helper = ModeLeafHelper {
request_rx: assembly_mode_request_rx,
report_tx: mode_report_tx,
@@ -603,8 +696,8 @@ mod tests {
let switch_map = SwitchSet::new(switch_map);
let shared_switch_set = SharedSwitchSet::new(Mutex::new(switch_map));
let health_table = HealthTableMapSync::default();
let (event_tx, event_rx) = mpsc::sync_channel(5);
let handler = MgmHandlerLis3Mdl::new(
let (event_tx, event_rx) = mpsc::sync_channel(20);
let mut handler = MgmHandlerLis3Mdl::new(
MgmId::_0,
TmtcQueues { tc_rx, tm_tx },
PowerSwitchHelper::new(switcher_tx, shared_switch_set.clone()),
@@ -615,6 +708,7 @@ mod tests {
health_table.clone(),
event_tx,
);
handler.recovery_off_duration = Duration::ZERO;
Self {
assembly_mode_request_tx,
mode_report_rx,
@@ -645,6 +739,61 @@ mod tests {
assert_eq!(self.handler.mode(), DeviceMode::Normal);
}
pub fn set_switch_state(&self, state: SwitchState) {
self.shared_switch_set
.lock()
.unwrap()
.set_switch_state(SwitchId::Mgm0, state);
}
pub fn inject_stuck_bus(&mut self) {
self.test_spi_interface().next_mgm_data = MgmLis3RawValues {
x: -1,
y: -1,
z: -1,
};
}
/// Drives SPI faults until the SPI fault threshold is exceeded once.
pub fn exceed_spi_fault_threshold(&mut self) {
self.inject_stuck_bus();
for _ in 0..SPI_FAULT_THRESHOLD + 1 {
self.handler.periodic_operation();
}
}
/// Drives a started power cycle recovery to completion, completing both power-switch
/// handshakes.
pub fn complete_power_cycle(&mut self) {
self.handler.periodic_operation();
self.set_switch_state(SwitchState::Off);
self.handler.periodic_operation();
assert_eq!(self.handler.mode(), DeviceMode::Off);
self.handler.periodic_operation();
assert_eq!(
self.handler.switch_and_mode_helper.target(),
Some(DeviceMode::Normal)
);
self.set_switch_state(SwitchState::On);
self.handler.periodic_operation();
assert_eq!(self.handler.mode(), DeviceMode::Normal);
}
pub fn health(&self) -> Option<HealthState> {
self.health_table.health(ComponentId::AcsMgm0.into())
}
pub fn drain_events(&self) -> Vec<mgm::Event> {
self.event_rx.try_iter().map(|(_, event)| event).collect()
}
pub fn drain_switch_requests(&self) -> Vec<SwitchStateBinary> {
self.switch_rx
.try_iter()
.map(|req| req.target_state)
.collect()
}
pub fn test_spi_interface(&mut self) -> &mut TestSpiInterface {
match &mut self.handler.spi_com {
SpiCommunication::Dummy(_) | SpiCommunication::Sim(_) => {
@@ -894,73 +1043,272 @@ mod tests {
}
#[test]
fn test_spi_fault_above_threshold_marks_component_faulty() {
fn test_spi_fault_above_threshold_starts_recovery() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
// Drain the mode changed event emitted by switch_to_normal().
testbench
.event_rx
.try_recv()
.expect("expected mode changed event");
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues {
x: -1,
y: -1,
z: -1,
};
// SPI_FAULT_THRESHOLD is exceeded on the (threshold + 1)-th stuck-bus reading.
for _ in 0..SPI_FAULT_THRESHOLD + 1 {
testbench.handler.periodic_operation();
}
assert_eq!(
testbench.health_table.health(ComponentId::AcsMgm0.into()),
Some(HealthState::Faulty)
);
testbench.drain_events();
testbench.exceed_spi_fault_threshold();
assert_eq!(testbench.health(), Some(HealthState::NeedsRecovery));
assert!(!testbench.handler.shared_mgm_set.lock().unwrap().valid);
let (sender_id, event) = testbench.event_rx.try_recv().expect("expected fault event");
assert_eq!(sender_id, ComponentId::AcsMgm0);
assert!(matches!(event, mgm::Event::SpiFaultThresholdExceeded));
let events = testbench.drain_events();
assert!(matches!(
events[..],
[
mgm::Event::SpiFaultThresholdExceeded,
mgm::Event::Recovery(RecoveryEvent::Started)
]
));
}
#[test]
fn test_spi_fault_above_threshold_commands_device_off() {
fn test_recovery_power_cycles_device() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
// Drain the switch-on request left over from switch_to_normal().
testbench
.switch_rx
.try_recv()
.expect("no switch-on request sent");
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues {
x: -1,
y: -1,
z: -1,
};
testbench.drain_events();
testbench.drain_switch_requests();
testbench.mode_report_rx.try_iter().for_each(drop);
testbench.exceed_spi_fault_threshold();
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues::default();
let call_count = testbench.test_spi_interface().call_count;
testbench.complete_power_cycle();
// The power cycle is hidden from the parent.
assert!(testbench.mode_report_rx.try_recv().is_err());
assert_eq!(testbench.health(), Some(HealthState::Healthy));
assert_eq!(
testbench.drain_switch_requests(),
[SwitchStateBinary::Off, SwitchStateBinary::On]
);
let events = testbench.drain_events();
assert!(matches!(
events[..],
[
mgm::Event::SpiFaultThresholdExceeded,
mgm::Event::Recovery(RecoveryEvent::Started),
mgm::Event::Recovery(RecoveryEvent::Done),
]
));
// The device is not polled during the power cycle, only in the last cycle.
assert_eq!(testbench.test_spi_interface().call_count, call_count + 1);
assert!(testbench.handler.shared_mgm_set.lock().unwrap().valid);
}
#[test]
fn test_repeated_recovery_marks_component_faulty() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench.exceed_spi_fault_threshold();
// Bus is still stuck after the power cycle.
testbench.complete_power_cycle();
testbench.drain_events();
testbench.drain_switch_requests();
// The last cycle of the power cycle already polled once.
for _ in 0..SPI_FAULT_THRESHOLD {
testbench.handler.periodic_operation();
}
assert_eq!(testbench.health(), Some(HealthState::Faulty));
let events = testbench.drain_events();
assert!(matches!(
events[..],
[
mgm::Event::SpiFaultThresholdExceeded,
mgm::Event::Recovery(RecoveryEvent::ThresholdExceeded)
]
));
testbench.handler.periodic_operation();
assert_eq!(testbench.drain_switch_requests(), [SwitchStateBinary::Off]);
testbench.set_switch_state(SwitchState::Off);
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
assert_eq!(testbench.health(), Some(HealthState::Faulty));
}
#[test]
fn test_faults_while_switching_off_keep_component_faulty() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench.exceed_spi_fault_threshold();
testbench.complete_power_cycle();
for _ in 0..SPI_FAULT_THRESHOLD {
testbench.handler.periodic_operation();
}
assert_eq!(testbench.health(), Some(HealthState::Faulty));
// The switch-off takes a while and the device is still polled in the meantime.
for _ in 0..SPI_FAULT_THRESHOLD + 1 {
testbench.handler.periodic_operation();
}
assert_eq!(
testbench.health_table.health(ComponentId::AcsMgm0.into()),
Some(HealthState::Faulty)
);
// The Off transition was only started on the last iteration above, so it has not sent
// its switch-off request yet: drive one more cycle to let it do so.
assert_eq!(testbench.health(), Some(HealthState::Faulty));
testbench.set_switch_state(SwitchState::Off);
testbench.handler.periodic_operation();
let switch_req = testbench
.switch_rx
.try_recv()
.expect("no switch-off request sent after fault");
assert_eq!(switch_req.switch_id, SwitchId::Mgm0);
assert_eq!(switch_req.target_state, SwitchStateBinary::Off);
// Simulate the PCDU acting on the switch-off request.
testbench
.shared_switch_set
.lock()
.unwrap()
.set_switch_state(SwitchId::Mgm0, SwitchState::Off);
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
assert_eq!(testbench.health(), Some(HealthState::Faulty));
}
#[test]
fn test_ground_needs_recovery_power_cycles_device() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench.drain_events();
testbench
.tc_tx
.send(create_request_tc(
MgmSelect::_0,
mgm::request::Request::Health(mgm::request::HealthRequest::SetHealth(
HealthState::NeedsRecovery,
)),
))
.unwrap();
testbench.handler.periodic_operation();
assert_eq!(
testbench.handler.switch_and_mode_helper.target(),
Some(DeviceMode::Off)
);
testbench.set_switch_state(SwitchState::Off);
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
testbench.handler.periodic_operation();
testbench.set_switch_state(SwitchState::On);
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Normal);
assert_eq!(testbench.health(), Some(HealthState::Healthy));
let events = testbench.drain_events();
assert!(matches!(
events[0],
mgm::Event::Recovery(RecoveryEvent::Started)
));
assert!(matches!(
events.last(),
Some(mgm::Event::Recovery(RecoveryEvent::Done))
));
}
#[test]
fn test_needs_recovery_while_off_sets_healthy() {
let mut testbench = MgmTestbench::new();
testbench
.health_table
.set_health(ComponentId::AcsMgm0.into(), HealthState::NeedsRecovery);
testbench.handler.periodic_operation();
assert_eq!(testbench.health(), Some(HealthState::Healthy));
assert!(testbench.drain_switch_requests().is_empty());
}
#[test]
fn test_recovery_switch_on_timeout_marks_component_faulty() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench.exceed_spi_fault_threshold();
testbench.handler.periodic_operation();
testbench.set_switch_state(SwitchState::Off);
testbench.handler.periodic_operation();
testbench.drain_events();
testbench.mode_report_rx.try_iter().for_each(drop);
// The switch never turns on again.
testbench.handler.periodic_operation();
std::thread::sleep(Duration::from_millis(110));
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
assert_eq!(testbench.health(), Some(HealthState::Faulty));
let events = testbench.drain_events();
assert!(matches!(
events[..],
[
mgm::Event::Recovery(RecoveryEvent::Failed),
mgm::Event::ModeChanged(DeviceMode::Off)
]
));
// A failed power cycle is not hidden from the parent.
assert!(matches!(
testbench.mode_report_rx.try_recv(),
Ok(ModeResponse::Mode(DeviceMode::Off))
));
}
#[test]
fn test_recovery_switch_off_timeout_commands_device_off() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench.exceed_spi_fault_threshold();
testbench.handler.periodic_operation();
testbench.drain_events();
testbench.drain_switch_requests();
testbench.mode_report_rx.try_iter().for_each(drop);
// The switch never turns off.
std::thread::sleep(Duration::from_millis(110));
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Normal);
assert_eq!(testbench.health(), Some(HealthState::Faulty));
// The mode did not change, so it is not announced or reported.
let events = testbench.drain_events();
assert!(matches!(
events[..],
[mgm::Event::Recovery(RecoveryEvent::Failed)]
));
assert!(testbench.mode_report_rx.try_recv().is_err());
assert_eq!(
testbench.handler.switch_and_mode_helper.target(),
Some(DeviceMode::Off)
);
testbench.handler.periodic_operation();
assert_eq!(testbench.drain_switch_requests(), [SwitchStateBinary::Off]);
testbench.set_switch_state(SwitchState::Off);
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
assert!(matches!(
testbench.mode_report_rx.try_recv(),
Ok(ModeResponse::Mode(DeviceMode::Off))
));
}
#[test]
fn test_read_mode_during_power_cycle_returns_restored_mode() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench.exceed_spi_fault_threshold();
testbench.handler.periodic_operation();
testbench.set_switch_state(SwitchState::Off);
// Keep the device off until the parent asked for its mode.
testbench.handler.recovery_off_duration = Duration::from_secs(60);
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
testbench.mode_report_rx.try_iter().for_each(drop);
testbench
.assembly_mode_request_tx
.send(ModeRequest::ReadMode)
.unwrap();
testbench.handler.periodic_operation();
assert!(matches!(
testbench.mode_report_rx.try_recv(),
Ok(ModeResponse::Mode(DeviceMode::Normal))
));
}
#[test]
fn test_mode_command_aborts_recovery() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench.exceed_spi_fault_threshold();
testbench.handler.periodic_operation();
testbench.set_switch_state(SwitchState::Off);
testbench
.assembly_mode_request_tx
.send(ModeRequest::SetMode(DeviceMode::Off))
.unwrap();
testbench.handler.periodic_operation();
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
assert_eq!(testbench.handler.switch_and_mode_helper.target(), None);
assert_eq!(testbench.health(), Some(HealthState::Healthy));
}
#[test]
satrs-example/src/device_mode.rs
+356 -2
View File
@@ -1,16 +1,21 @@
use std::time::Duration;
use std::time::{Duration, Instant};
use types::pcdu::SwitchId;
use crate::eps::PowerSwitchHelper;
/// Modes that distinguish a powered-off state from one or more powered-on states, so
/// This is a helper trait required to make [SwitchAndModeHelper] generic.
///
/// It allows distinguish a powered-off state from one or more powered-on states, so
/// [`SwitchAndModeHelper`] knows which way to drive the switch for a given target mode.
pub trait PowerSwitchedMode: Copy + PartialEq {
const OFF: Self;
fn requires_power(&self) -> bool;
}
impl PowerSwitchedMode for types::DeviceMode {
const OFF: Self = types::DeviceMode::Off;
fn requires_power(&self) -> bool {
*self != types::DeviceMode::Off
}
@@ -24,6 +29,14 @@ enum SwitchTransitionState {
Done,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum PowerCycleState<Mode> {
Idle,
SwitchingOff { restore_mode: Mode },
WaitingOff { restore_mode: Mode, since: Instant },
SwitchingOn { restore_mode: Mode },
}
/// Outcome of a pending mode transition, once [`SwitchAndModeHelper::handle_mode_transition`]
/// has driven it to completion. Carries back whichever TC commanded the transition, if any, so
/// the caller can reply to it -- what that reply looks like is handler-specific, so this stays
@@ -31,6 +44,11 @@ enum SwitchTransitionState {
pub enum ModeTransitionEvent {
Reached(Option<satrs::spacepackets::CcsdsPacketIdAndPsc>),
Failed(Option<satrs::spacepackets::CcsdsPacketIdAndPsc>),
/// The power cycle completed and the mode before the power cycle was restored.
PowerCycleDone,
/// Power switching failed during the power cycle. The power cycle is not hidden anymore,
/// so [SwitchAndModeHelper::reported_mode] returns the actual mode again.
PowerCycleFailed,
}
/// Drives the on/off power-switch commanding state machine (Idle -> PowerSwitching -> Done)
@@ -43,6 +61,8 @@ pub struct SwitchAndModeHelper<Mode: PowerSwitchedMode> {
mode_helper: satrs_example::ModeHelper<Mode, SwitchTransitionState>,
switch_helper: PowerSwitchHelper,
switch_id: SwitchId,
power_cycle: PowerCycleState<Mode>,
power_cycle_off_duration: Duration,
}
impl<Mode: PowerSwitchedMode> SwitchAndModeHelper<Mode> {
@@ -56,6 +76,8 @@ impl<Mode: PowerSwitchedMode> SwitchAndModeHelper<Mode> {
mode_helper: satrs_example::ModeHelper::new(init_mode, timeout),
switch_helper,
switch_id,
power_cycle: PowerCycleState::Idle,
power_cycle_off_duration: Duration::ZERO,
}
}
@@ -69,16 +91,88 @@ impl<Mode: PowerSwitchedMode> SwitchAndModeHelper<Mode> {
self.mode_helper.target
}
/// Mode which should be reported to other components. A power cycle is hidden from them,
/// so this is the mode which is restored after the power cycle while one is active.
pub fn reported_mode(&self) -> Mode {
match self.power_cycle {
PowerCycleState::SwitchingOff { restore_mode }
| PowerCycleState::WaitingOff { restore_mode, .. }
| PowerCycleState::SwitchingOn { restore_mode } => restore_mode,
PowerCycleState::Idle => self.mode(),
}
}
#[inline]
pub fn power_cycle_active(&self) -> bool {
self.power_cycle != PowerCycleState::Idle
}
/// Starts a new transition, aborting a running power cycle.
pub fn start_transition(
&mut self,
target_mode: Mode,
tc_commander: Option<satrs::spacepackets::CcsdsPacketIdAndPsc>,
) {
self.power_cycle = PowerCycleState::Idle;
self.start_transition_internal(target_mode, tc_commander);
}
/// Switches the device off, keeps it off for `off_duration` and then restores the current
/// mode. Reaching the intermediate off mode does not generate an event.
pub fn start_power_cycle(&mut self, off_duration: Duration) {
self.power_cycle = PowerCycleState::SwitchingOff {
restore_mode: self.mode(),
};
self.power_cycle_off_duration = off_duration;
self.start_transition_internal(Mode::OFF, None);
}
fn start_transition_internal(
&mut self,
target_mode: Mode,
tc_commander: Option<satrs::spacepackets::CcsdsPacketIdAndPsc>,
) {
self.mode_helper.tc_commander = tc_commander;
self.mode_helper.start(target_mode);
}
pub fn handle_mode_transition(&mut self) -> Option<ModeTransitionEvent> {
if let PowerCycleState::WaitingOff {
restore_mode,
since,
} = self.power_cycle
&& since.elapsed() >= self.power_cycle_off_duration
{
self.power_cycle = PowerCycleState::SwitchingOn { restore_mode };
self.start_transition_internal(restore_mode, None);
}
let event = self.handle_switch_transition()?;
if self.power_cycle == PowerCycleState::Idle {
return Some(event);
}
match event {
ModeTransitionEvent::Reached(_) => match self.power_cycle {
PowerCycleState::SwitchingOff { restore_mode } => {
self.power_cycle = PowerCycleState::WaitingOff {
restore_mode,
since: Instant::now(),
};
None
}
_ => {
self.power_cycle = PowerCycleState::Idle;
Some(ModeTransitionEvent::PowerCycleDone)
}
},
ModeTransitionEvent::Failed(_) => {
self.power_cycle = PowerCycleState::Idle;
Some(ModeTransitionEvent::PowerCycleFailed)
}
event => Some(event),
}
}
fn handle_switch_transition(&mut self) -> Option<ModeTransitionEvent> {
let target_mode = self.mode_helper.target?;
let switch_target_on = target_mode.requires_power();
if self.mode_helper.transition_state == SwitchTransitionState::Idle {
@@ -110,3 +204,263 @@ impl<Mode: PowerSwitchedMode> SwitchAndModeHelper<Mode> {
None
}
}
#[cfg(test)]
mod tests {
use std::sync::{Arc, Mutex, mpsc};
use arbitrary_int::u11;
use satrs::spacepackets::{CcsdsPacketIdAndPsc, SpacePacketHeader};
use types::{
DeviceMode,
pcdu::{SwitchRequest, SwitchState, SwitchStateBinary},
};
use crate::eps::pcdu::{SharedSwitchSet, SwitchMap, SwitchSet};
use super::*;
const TIMEOUT: Duration = Duration::from_millis(50);
struct Testbench {
helper: SwitchAndModeHelper<DeviceMode>,
switch_rx: mpsc::Receiver<SwitchRequest>,
shared_switch_set: SharedSwitchSet,
}
impl Testbench {
fn new() -> Self {
let (switch_tx, switch_rx) = mpsc::sync_channel(10);
let mut switch_map = SwitchMap::new();
switch_map.insert(SwitchId::Mgm0, SwitchState::Off);
let shared_switch_set: SharedSwitchSet =
Arc::new(Mutex::new(SwitchSet::new(switch_map)));
Self {
helper: SwitchAndModeHelper::new(
DeviceMode::Off,
TIMEOUT,
PowerSwitchHelper::new(switch_tx, shared_switch_set.clone()),
SwitchId::Mgm0,
),
switch_rx,
shared_switch_set,
}
}
fn set_switch_state(&self, state: SwitchState) {
self.shared_switch_set
.lock()
.unwrap()
.set_switch_state(SwitchId::Mgm0, state);
}
fn switch_requests(&self) -> Vec<SwitchStateBinary> {
self.switch_rx
.try_iter()
.map(|req| req.target_state)
.collect()
}
/// Drives a transition to `Normal` to completion.
fn switch_to_normal(&mut self) {
self.helper.start_transition(DeviceMode::Normal, None);
self.set_switch_state(SwitchState::On);
assert!(matches!(
self.helper.handle_mode_transition(),
Some(ModeTransitionEvent::Reached(None))
));
self.switch_requests();
}
/// Starts a power cycle from `Normal` and drives it until the device is off.
fn power_cycle_until_off(&mut self, off_duration: Duration) {
self.switch_to_normal();
self.helper.start_power_cycle(off_duration);
assert!(self.helper.handle_mode_transition().is_none());
assert_eq!(self.switch_requests(), [SwitchStateBinary::Off]);
self.set_switch_state(SwitchState::Off);
assert!(self.helper.handle_mode_transition().is_none());
assert_eq!(self.helper.mode(), DeviceMode::Off);
}
}
fn tc_id() -> CcsdsPacketIdAndPsc {
CcsdsPacketIdAndPsc::new_from_ccsds_packet(&SpacePacketHeader::new_from_apid(u11::new(1)))
}
#[test]
fn test_no_transition() {
let mut tb = Testbench::new();
assert_eq!(tb.helper.mode(), DeviceMode::Off);
assert_eq!(tb.helper.target(), None);
assert!(tb.helper.handle_mode_transition().is_none());
assert!(tb.switch_requests().is_empty());
assert!(!tb.helper.power_cycle_active());
assert_eq!(tb.helper.reported_mode(), DeviceMode::Off);
}
#[test]
fn test_switch_on() {
let mut tb = Testbench::new();
tb.helper
.start_transition(DeviceMode::Normal, Some(tc_id()));
assert!(tb.helper.handle_mode_transition().is_none());
assert_eq!(tb.switch_requests(), [SwitchStateBinary::On]);
assert_eq!(tb.helper.mode(), DeviceMode::Off);
assert_eq!(tb.helper.target(), Some(DeviceMode::Normal));
tb.set_switch_state(SwitchState::On);
match tb.helper.handle_mode_transition() {
Some(ModeTransitionEvent::Reached(Some(id))) => assert_eq!(id, tc_id()),
_ => panic!("expected mode reached event with TC commander"),
}
assert_eq!(tb.helper.mode(), DeviceMode::Normal);
assert_eq!(tb.helper.target(), None);
assert!(tb.switch_requests().is_empty());
}
#[test]
fn test_switch_off() {
let mut tb = Testbench::new();
tb.switch_to_normal();
tb.helper.start_transition(DeviceMode::Off, None);
assert!(tb.helper.handle_mode_transition().is_none());
assert_eq!(tb.switch_requests(), [SwitchStateBinary::Off]);
tb.set_switch_state(SwitchState::Off);
assert!(matches!(
tb.helper.handle_mode_transition(),
Some(ModeTransitionEvent::Reached(None))
));
assert_eq!(tb.helper.mode(), DeviceMode::Off);
}
#[test]
fn test_switch_already_in_target_state() {
let mut tb = Testbench::new();
tb.set_switch_state(SwitchState::On);
tb.helper.start_transition(DeviceMode::On, None);
assert!(matches!(
tb.helper.handle_mode_transition(),
Some(ModeTransitionEvent::Reached(None))
));
// The switch command is still sent.
assert_eq!(tb.switch_requests(), [SwitchStateBinary::On]);
assert_eq!(tb.helper.mode(), DeviceMode::On);
}
#[test]
fn test_switch_timeout() {
let mut tb = Testbench::new();
tb.helper
.start_transition(DeviceMode::Normal, Some(tc_id()));
assert!(tb.helper.handle_mode_transition().is_none());
std::thread::sleep(TIMEOUT);
match tb.helper.handle_mode_transition() {
Some(ModeTransitionEvent::Failed(Some(id))) => assert_eq!(id, tc_id()),
_ => panic!("expected mode failed event with TC commander"),
}
assert_eq!(tb.helper.mode(), DeviceMode::Off);
assert_eq!(tb.helper.target(), None);
}
#[test]
fn test_power_cycle() {
let mut tb = Testbench::new();
tb.power_cycle_until_off(Duration::ZERO);
assert!(tb.helper.power_cycle_active());
// The off duration elapsed, so switching on starts right away.
assert!(tb.helper.handle_mode_transition().is_none());
assert_eq!(tb.switch_requests(), [SwitchStateBinary::On]);
assert_eq!(tb.helper.target(), Some(DeviceMode::Normal));
tb.set_switch_state(SwitchState::On);
assert!(matches!(
tb.helper.handle_mode_transition(),
Some(ModeTransitionEvent::PowerCycleDone)
));
assert_eq!(tb.helper.mode(), DeviceMode::Normal);
assert!(!tb.helper.power_cycle_active());
}
#[test]
fn test_power_cycle_reports_restored_mode() {
let mut tb = Testbench::new();
tb.switch_to_normal();
tb.helper.start_power_cycle(Duration::from_secs(60));
assert_eq!(tb.helper.reported_mode(), DeviceMode::Normal);
tb.helper.handle_mode_transition();
tb.set_switch_state(SwitchState::Off);
tb.helper.handle_mode_transition();
assert_eq!(tb.helper.mode(), DeviceMode::Off);
assert_eq!(tb.helper.reported_mode(), DeviceMode::Normal);
}
#[test]
fn test_power_cycle_reports_restored_mode_while_switching_on() {
let mut tb = Testbench::new();
tb.power_cycle_until_off(Duration::ZERO);
assert!(tb.helper.handle_mode_transition().is_none());
assert_eq!(tb.helper.target(), Some(DeviceMode::Normal));
assert_eq!(tb.helper.mode(), DeviceMode::Off);
assert_eq!(tb.helper.reported_mode(), DeviceMode::Normal);
}
#[test]
fn test_power_cycle_waits_off_duration() {
let mut tb = Testbench::new();
tb.power_cycle_until_off(Duration::from_secs(60));
for _ in 0..3 {
assert!(tb.helper.handle_mode_transition().is_none());
}
assert!(tb.switch_requests().is_empty());
assert_eq!(tb.helper.target(), None);
assert!(tb.helper.power_cycle_active());
}
#[test]
fn test_power_cycle_switch_off_timeout() {
let mut tb = Testbench::new();
tb.switch_to_normal();
tb.helper.start_power_cycle(Duration::ZERO);
assert!(tb.helper.handle_mode_transition().is_none());
std::thread::sleep(TIMEOUT);
assert!(matches!(
tb.helper.handle_mode_transition(),
Some(ModeTransitionEvent::PowerCycleFailed)
));
assert_eq!(tb.helper.mode(), DeviceMode::Normal);
assert!(!tb.helper.power_cycle_active());
}
#[test]
fn test_power_cycle_switch_on_timeout() {
let mut tb = Testbench::new();
tb.power_cycle_until_off(Duration::ZERO);
assert!(tb.helper.handle_mode_transition().is_none());
std::thread::sleep(TIMEOUT);
assert!(matches!(
tb.helper.handle_mode_transition(),
Some(ModeTransitionEvent::PowerCycleFailed)
));
assert_eq!(tb.helper.mode(), DeviceMode::Off);
assert!(!tb.helper.power_cycle_active());
// The failed power cycle is not hidden anymore.
assert_eq!(tb.helper.reported_mode(), DeviceMode::Off);
}
#[test]
fn test_transition_aborts_power_cycle() {
let mut tb = Testbench::new();
tb.power_cycle_until_off(Duration::from_secs(60));
tb.helper.start_transition(DeviceMode::On, Some(tc_id()));
assert!(!tb.helper.power_cycle_active());
assert_eq!(tb.helper.reported_mode(), DeviceMode::Off);
tb.set_switch_state(SwitchState::On);
// A regular transition event instead of a power cycle event.
assert!(matches!(
tb.helper.handle_mode_transition(),
Some(ModeTransitionEvent::Reached(Some(_)))
));
assert_eq!(tb.helper.mode(), DeviceMode::On);
}
}
satrs-example/types/src/acs/mgm.rs
+6 -3
View File
@@ -62,11 +62,11 @@ pub struct SensorData {
#[strum_discriminants(derive(num_enum::IntoPrimitive))]
#[repr(u16)]
pub enum Event {
/// The SPI fault counter exceeded its threshold, the component was marked faulty and
/// commanded off.
/// The SPI fault counter exceeded its threshold. Followed by a recovery event.
SpiFaultThresholdExceeded,
/// A commanded or autonomous mode transition completed.
ModeChanged(crate::DeviceMode),
Recovery(satrs::fdir::RecoveryEvent),
}
impl crate::Message for Event {
@@ -77,7 +77,10 @@ impl crate::Message for Event {
impl crate::EventId for Event {
fn event_id(&self) -> u16 {
EventDiscriminants::from(self).into()
match self {
Event::Recovery(event) => crate::recovery_event_id(*event),
_ => EventDiscriminants::from(self).into(),
}
}
}
satrs-example/types/src/lib.rs
+8
View File
@@ -175,6 +175,14 @@ pub trait EventId {
fn event_id(&self) -> u16;
}
/// Start of the event ID range for generic FDIR events, which are embedded into the event types
/// of the components. These events have the same ID for all components.
pub const FDIR_EVENT_ID_BASE: u16 = 0x100;
pub const fn recovery_event_id(event: satrs::fdir::RecoveryEvent) -> u16 {
FDIR_EVENT_ID_BASE + event as u16
}
/// Generic device mode which covers the requirements of most devices.
///
/// The states are related both to the physical and the logical state of the device. Some
satrs/CHANGELOG.md
+3
View File
@@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
- Added `hk` module helpers to track whether a single HK set needs regeneration:
`SingleSetHkHelperStd` (`std`), `SingleSetHkHelperEmbassy` (new `embassy-time` feature),
and `SingleSetHkHelperCountdown`, generic over the existing `Countdown` trait.
- Added `fdir::RecoveryFdir` (`std`), which escalates component faults to a power cycle
recovery first and to a faulty component if it has to be recovered too often.
- Added `fdir::RecoveryEvent` for recovery related events.
# [v0.3.0-alpha.3] 2025-11-06
satrs/src/fdir.rs
+201
View File
@@ -15,8 +15,45 @@
feature = "embassy-time",
doc = "- [FaultCounterEmbassy]: `embassy_time::Instant`, behind the `embassy-time` feature."
)]
//!
//! [RecoveryFdir] builds on top of that. It decides whether a component is power cycled or
//! marked faulty when one of its fault counters exceeds its threshold, and keeps the health
//! table up to date during the recovery. It follows the FSFW `DeviceHandlerFailureIsolation`.
#![deny(missing_docs)]
#[cfg(feature = "std")]
use crate::health::{HealthState, HealthTableProvider};
/// Events related to the recovery of a component. Components are expected to embed this into
/// their own event type.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub enum RecoveryEvent {
/// The component health is [crate::health::HealthState::NeedsRecovery] and it is being power cycled.
Started,
/// The power cycle completed and the component is healthy again.
Done,
/// The power cycle failed, the component was marked faulty.
Failed,
/// The component was recovered too often, it was marked faulty.
ThresholdExceeded,
}
/// Outcome of [RecoveryFdir::handle_fault].
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum FaultResponse {
/// The component is already faulty, recovering or externally controlled, so nothing was
/// changed.
Ignored,
/// The health was set to [crate::health::HealthState::NeedsRecovery]. The component should
/// be power cycled.
Recover,
/// The component was recovered too often and its health was set to [crate::health::HealthState::Faulty].
/// The component should be switched off.
SetFaulty,
}
/// Fault counter backed by [std::time::Instant].
#[cfg(feature = "std")]
#[derive(Debug, Clone)]
@@ -230,6 +267,98 @@ impl FaultCounterEmbassy {
}
}
/// Escalates faults of a component to a power cycle recovery first, and to a faulty
/// component if it has to be recovered too often.
///
/// The component itself runs the power cycle while [Self::needs_recovery] returns `true` and
/// reports the outcome with [Self::recovery_done] or [Self::recovery_failed]. Setting
/// [HealthState::NeedsRecovery] from outside, for example by ground, triggers a recovery as well.
#[cfg(feature = "std")]
#[derive(Debug, Clone)]
pub struct RecoveryFdir<HealthTable: HealthTableProvider> {
id: crate::ComponentId,
health_table: HealthTable,
recovery_counter: FaultCounterStd,
}
#[cfg(feature = "std")]
impl<HealthTable: HealthTableProvider> RecoveryFdir<HealthTable> {
/// Create a new [RecoveryFdir] for component `id`.
///
/// The component is marked faulty when it would be recovered more than `recovery_threshold`
/// times, with the recovery count being decremented every `recovery_decrement_after`.
pub fn new(
id: crate::ComponentId,
health_table: HealthTable,
recovery_threshold: u32,
recovery_decrement_after: core::time::Duration,
) -> Self {
Self {
id,
health_table,
recovery_counter: FaultCounterStd::new(recovery_threshold, recovery_decrement_after),
}
}
/// Health of the component. Absent entries are returned as `None`.
pub fn health(&self) -> Option<HealthState> {
self.health_table.health(self.id)
}
/// Set the health of the component.
pub fn set_health(&mut self, health: HealthState) {
self.health_table.set_health(self.id, health);
}
/// Should be called periodically to decrement the recovery counter.
pub fn periodic_operation(&mut self) {
self.recovery_counter.try_decrement();
}
/// Should be called when a fault counter of the component exceeded its threshold.
pub fn handle_fault(&mut self) -> FaultResponse {
// Ground may have taken manual control, or already given up on this component.
// Autonomous FDIR should not override that decision. An already faulty or recovering
// component must not be escalated again. For example, this would allow a faulty component
// to become healthy again, because the recovery counter was reset when it became faulty.
if matches!(
self.health(),
Some(HealthState::ExternalControl)
| Some(HealthState::PermanentFaulty)
| Some(HealthState::Faulty)
| Some(HealthState::NeedsRecovery)
) {
return FaultResponse::Ignored;
}
if self.recovery_counter.increment_and_check() {
self.set_health(HealthState::Faulty);
return FaultResponse::SetFaulty;
}
self.set_health(HealthState::NeedsRecovery);
FaultResponse::Recover
}
/// The component should be power cycled.
pub fn needs_recovery(&self) -> bool {
self.health() == Some(HealthState::NeedsRecovery)
}
/// The power cycle completed, or was not required. Sets the health back to healthy.
pub fn recovery_done(&mut self) {
// The health might have changed during the recovery.
if self.needs_recovery() {
self.set_health(HealthState::Healthy);
}
}
/// The power cycle failed. Marks the component faulty.
pub fn recovery_failed(&mut self) {
if self.needs_recovery() {
self.set_health(HealthState::Faulty);
}
}
}
#[cfg(all(test, feature = "std"))]
mod tests {
use super::*;
@@ -281,6 +410,78 @@ mod tests {
assert!(!fc.try_decrement());
}
fn recovery_fdir() -> RecoveryFdir<crate::health::HealthTableMapSync> {
RecoveryFdir::new(
1,
crate::health::HealthTableMapSync::default(),
1,
Duration::from_secs(60),
)
}
#[test]
fn first_fault_triggers_recovery() {
let mut fdir = recovery_fdir();
assert_eq!(fdir.handle_fault(), FaultResponse::Recover);
assert!(fdir.needs_recovery());
fdir.recovery_done();
assert_eq!(fdir.health(), Some(HealthState::Healthy));
}
#[test]
fn repeated_recovery_sets_faulty() {
let mut fdir = recovery_fdir();
assert_eq!(fdir.handle_fault(), FaultResponse::Recover);
fdir.recovery_done();
assert_eq!(fdir.handle_fault(), FaultResponse::SetFaulty);
assert_eq!(fdir.health(), Some(HealthState::Faulty));
}
#[test]
fn faulty_component_stays_faulty() {
let mut fdir = recovery_fdir();
fdir.handle_fault();
fdir.recovery_done();
assert_eq!(fdir.handle_fault(), FaultResponse::SetFaulty);
// The recovery counter was reset, but this must not trigger a new recovery.
assert_eq!(fdir.handle_fault(), FaultResponse::Ignored);
assert_eq!(fdir.health(), Some(HealthState::Faulty));
}
#[test]
fn failed_recovery_sets_faulty() {
let mut fdir = recovery_fdir();
fdir.handle_fault();
fdir.recovery_failed();
assert_eq!(fdir.health(), Some(HealthState::Faulty));
}
#[test]
fn health_is_not_overridden() {
let mut fdir = recovery_fdir();
for health in [
HealthState::ExternalControl,
HealthState::PermanentFaulty,
HealthState::Faulty,
HealthState::NeedsRecovery,
] {
fdir.set_health(health);
assert_eq!(fdir.handle_fault(), FaultResponse::Ignored);
assert_eq!(fdir.health(), Some(health));
}
}
#[test]
fn health_changed_during_recovery_is_kept() {
let mut fdir = recovery_fdir();
fdir.handle_fault();
fdir.set_health(HealthState::ExternalControl);
fdir.recovery_done();
assert_eq!(fdir.health(), Some(HealthState::ExternalControl));
fdir.recovery_failed();
assert_eq!(fdir.health(), Some(HealthState::ExternalControl));
}
#[test]
fn clear_resets_state() {
let mut fc = FaultCounterStd::new(1, Duration::from_secs(60));