diff --git a/satrs-example/client/Cargo.toml b/satrs-example/client/Cargo.toml index 47df34b..bd28404 100644 --- a/satrs-example/client/Cargo.toml +++ b/satrs-example/client/Cargo.toml @@ -9,7 +9,10 @@ log = "0.4" fern = "0.7" humantime = "2" serde = { version = "1" } +serde_json = "1" +satrs = { path = "../../satrs" } satrs-example = { path = ".." } +satrs-minisim = { path = "../minisim" } types = { path = "../types" } spacepackets = { version = "0.18", default-features = false } bitbybit = "2" diff --git a/satrs-example/client/src/main.rs b/satrs-example/client/src/main.rs index 4e25ac1..4979574 100644 --- a/satrs-example/client/src/main.rs +++ b/satrs-example/client/src/main.rs @@ -2,9 +2,13 @@ use anyhow::bail; use arbitrary_int::u11; use clap::Parser as _; use satrs_example::config::{OBSW_SERVER_ADDR, SERVER_PORT}; +use satrs_minisim::{ + SerializableSimMsgPayload, SimComponent, SimCtrlReply, SimCtrlRequest, SimMessageProvider, + SimReply, SimRequest, acs::MgmRequestLis3Mdl, acs::SpiFaultMode, udp::SIM_CTRL_PORT, +}; use spacepackets::{CcsdsPacketIdAndPsc, SpacePacketHeader}; use std::{ - net::{IpAddr, SocketAddr, UdpSocket}, + net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, sync::{ Arc, atomic::{AtomicBool, Ordering}, @@ -32,14 +36,40 @@ enum Commands { AcsSubsystem(SubsystemArgs), } -impl Commands { - #[inline] - pub fn target_id(&self) -> types::ComponentId { - match self { - Commands::Mgm0(_mgm_args) => types::ComponentId::AcsMgm0, - Commands::Mgm1(_mgm_args) => types::ComponentId::AcsMgm1, - Commands::MgmAssy(_mgm_assembly_args) => types::ComponentId::AcsMgmAssembly, - Commands::AcsSubsystem(_subsystem_args) => types::ComponentId::AcsSubsystem, +#[derive(Debug, PartialEq, Eq, Clone, Copy, clap::ValueEnum)] +enum SpiFaultModeSelect { + None, + AllZeros, + AllOnes, +} + +impl From for SpiFaultMode { + fn from(mode: SpiFaultModeSelect) -> Self { + match mode { + SpiFaultModeSelect::None => SpiFaultMode::None, + SpiFaultModeSelect::AllZeros => SpiFaultMode::AllZeros, + SpiFaultModeSelect::AllOnes => SpiFaultMode::AllOnes, + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, clap::ValueEnum)] +enum HealthStateSelect { + Healthy, + Faulty, + PermanentFaulty, + ExternalControl, + NeedsRecovery, +} + +impl From for satrs::health::HealthState { + fn from(state: HealthStateSelect) -> Self { + match state { + HealthStateSelect::Healthy => satrs::health::HealthState::Healthy, + HealthStateSelect::Faulty => satrs::health::HealthState::Faulty, + HealthStateSelect::PermanentFaulty => satrs::health::HealthState::PermanentFaulty, + HealthStateSelect::ExternalControl => satrs::health::HealthState::ExternalControl, + HealthStateSelect::NeedsRecovery => satrs::health::HealthState::NeedsRecovery, } } } @@ -52,6 +82,16 @@ struct MgmArgs { request_hk: bool, #[arg(short, long)] mode: Option, + /// Inject (or clear) an SPI bus failure on the simulated device, bypassing the OBSW. + /// + /// Only takes effect for MGM0: minisim always routes this fault to the MGM0 model + /// regardless of which MGM the request names (a pre-existing minisim limitation). + #[arg(long, value_enum)] + spi_fault: Option, + /// Override the device's FDIR health state, for example to clear a `Faulty` state set by + /// the handler after the underlying issue has been fixed or worked around. + #[arg(long, value_enum)] + health: Option, } #[derive(Debug, PartialEq, Eq, Clone, Copy, clap::Parser)] @@ -89,6 +129,93 @@ pub enum SubsystemModeSelect { Safe, } +fn handle_mgm_command( + client: &UdpSocket, + addr: SocketAddr, + target_id: types::ComponentId, + args: MgmArgs, +) -> anyhow::Result<()> { + if let Some(mode) = args.spi_fault { + if target_id != types::ComponentId::AcsMgm0 { + bail!("SPI fault injection is only supported for MGM0 right now (minisim limitation)"); + } + inject_mgm_failure(mode.into())?; + } + if args.ping { + let request = types::ccsds::CcsdsTcPacketOwned::new_with_request( + SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)), + TcHeader::new(target_id, types::MessageType::Ping), + types::acs::mgm::request::Request::Ping, + ); + let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header); + log::info!( + "sending {:?} ping request with TC ID {:#010x}", + target_id, + sent_tc_id.raw() + ); + let request_packet = request.to_vec(); + client.send_to(&request_packet, addr).unwrap(); + } + if args.request_hk { + let request = types::ccsds::CcsdsTcPacketOwned::new_with_request( + SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)), + TcHeader::new(target_id, types::MessageType::Hk), + types::acs::mgm::request::Request::Hk(HkRequest { + id: types::acs::mgm::request::HkId::Sensor, + req_type: types::HkRequestType::OneShot, + }), + ); + let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header); + log::info!( + "sending {:?} HK request with TC ID {:#010x}", + target_id, + sent_tc_id.raw() + ); + let request_packet = request.to_vec(); + client.send_to(&request_packet, addr).unwrap(); + } + if let Some(mode) = args.mode { + let dev_mode = match mode { + DeviceModeSelect::Off => types::DeviceMode::Off, + DeviceModeSelect::Normal => types::DeviceMode::Normal, + }; + + let request = types::ccsds::CcsdsTcPacketOwned::new_with_request( + SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)), + TcHeader::new(target_id, types::MessageType::Mode), + types::acs::mgm::request::Request::Mode( + types::acs::mgm::request::ModeRequest::SetMode(dev_mode), + ), + ); + let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header); + log::info!( + "sending {:?} HK request with TC ID {:#010x}", + target_id, + sent_tc_id.raw() + ); + let request_packet = request.to_vec(); + client.send_to(&request_packet, addr).unwrap(); + } + if let Some(health) = args.health { + let request = types::ccsds::CcsdsTcPacketOwned::new_with_request( + SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)), + TcHeader::new(target_id, types::MessageType::Health), + types::acs::mgm::request::Request::Health( + types::acs::mgm::request::HealthRequest::SetHealth(health.into()), + ), + ); + let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header); + log::info!( + "sending {:?} set-health request with TC ID {:#010x}", + target_id, + sent_tc_id.raw() + ); + let request_packet = request.to_vec(); + client.send_to(&request_packet, addr).unwrap(); + } + Ok(()) +} + fn setup_logger(level: log::LevelFilter) -> Result<(), fern::InitError> { fern::Dispatch::new() .format(|out, message, record| { @@ -145,70 +272,19 @@ fn main() -> anyhow::Result<()> { client.send_to(&request_packet, addr).unwrap(); } if let Some(cmd) = cli.commands { - let target_id = cmd.target_id(); match cmd { - Commands::Mgm0(args) | Commands::Mgm1(args) => { - if args.ping { - let request = types::ccsds::CcsdsTcPacketOwned::new_with_request( - SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)), - TcHeader::new(cmd.target_id(), types::MessageType::Ping), - types::acs::mgm::request::Request::Ping, - ); - let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header); - log::info!( - "sending {:?} ping request with TC ID {:#010x}", - target_id, - sent_tc_id.raw() - ); - let request_packet = request.to_vec(); - client.send_to(&request_packet, addr).unwrap(); - } - if args.request_hk { - let request = types::ccsds::CcsdsTcPacketOwned::new_with_request( - SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)), - TcHeader::new(target_id, types::MessageType::Hk), - types::acs::mgm::request::Request::Hk(HkRequest { - id: types::acs::mgm::request::HkId::Sensor, - req_type: types::HkRequestType::OneShot, - }), - ); - let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header); - log::info!( - "sending {:?} HK request with TC ID {:#010x}", - target_id, - sent_tc_id.raw() - ); - let request_packet = request.to_vec(); - client.send_to(&request_packet, addr).unwrap(); - } - if let Some(mode) = args.mode { - let dev_mode = match mode { - DeviceModeSelect::Off => types::DeviceMode::Off, - DeviceModeSelect::Normal => types::DeviceMode::Normal, - }; - - let request = types::ccsds::CcsdsTcPacketOwned::new_with_request( - SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)), - TcHeader::new(target_id, types::MessageType::Mode), - types::acs::mgm::request::Request::Mode( - types::acs::mgm::request::ModeRequest::SetMode(dev_mode), - ), - ); - let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header); - log::info!( - "sending {:?} HK request with TC ID {:#010x}", - target_id, - sent_tc_id.raw() - ); - let request_packet = request.to_vec(); - client.send_to(&request_packet, addr).unwrap(); - } + Commands::Mgm0(args) => { + handle_mgm_command(&client, addr, types::ComponentId::AcsMgm0, args)? + } + Commands::Mgm1(args) => { + handle_mgm_command(&client, addr, types::ComponentId::AcsMgm1, args)? } Commands::MgmAssy(mgm_assembly_args) => { + let target_id = types::ComponentId::AcsMgmAssembly; if mgm_assembly_args.ping { let request = types::ccsds::CcsdsTcPacketOwned::new_with_request( SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)), - TcHeader::new(cmd.target_id(), types::MessageType::Ping), + TcHeader::new(target_id, types::MessageType::Ping), types::acs::mgm::request::Request::Ping, ); let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header); @@ -251,10 +327,11 @@ fn main() -> anyhow::Result<()> { } } Commands::AcsSubsystem(subsystem_args) => { + let target_id = types::ComponentId::AcsSubsystem; if subsystem_args.ping { let request = types::ccsds::CcsdsTcPacketOwned::new_with_request( SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)), - TcHeader::new(cmd.target_id(), types::MessageType::Ping), + TcHeader::new(target_id, types::MessageType::Ping), types::acs::subsystem::request::Request::Ping, ); let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header); @@ -314,6 +391,49 @@ fn main() -> anyhow::Result<()> { Ok(()) } +/// Injects the given SPI fault mode directly into minisim's MGM0 model, bypassing the OBSW. +/// +/// Confirms the simulator is actually reachable first (same ping/pong check the OBSW's own +/// internal sim client does, see `SimClientUdp::attempt_connection`), since a fire-and-forget +/// UDP send would otherwise silently do nothing if minisim is not running. +fn inject_mgm_failure(mode: SpiFaultMode) -> anyhow::Result<()> { + let sim_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), SIM_CTRL_PORT); + let sim_socket = UdpSocket::bind("127.0.0.1:0")?; + sim_socket.set_read_timeout(Some(Duration::from_millis(200)))?; + + let mut reply_buf = [0u8; 4096]; + let ping = SimRequest::new_with_epoch_time(SimCtrlRequest::Ping); + sim_socket.send_to(&serde_json::to_vec(&ping)?, sim_addr)?; + match sim_socket.recv(&mut reply_buf) { + Ok(len) => { + let reply: SimReply = serde_json::from_slice(&reply_buf[..len])?; + if reply.component() != SimComponent::SimCtrl { + bail!("unexpected reply while checking minisim connectivity: {reply:?}"); + } + match SimCtrlReply::from_sim_message(&reply).expect("invalid SIM reply") { + SimCtrlReply::Pong => {} + SimCtrlReply::InvalidRequest(e) => { + bail!("minisim rejected connectivity ping: {e:?}") + } + } + } + Err(e) + if matches!( + e.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => + { + bail!("minisim not reachable at {sim_addr} (ping timed out) - is it running?"); + } + Err(e) => return Err(e.into()), + } + + let request = SimRequest::new_with_epoch_time(MgmRequestLis3Mdl::SetSpiFault(mode)); + sim_socket.send_to(&serde_json::to_vec(&request)?, sim_addr)?; + log::info!("injected SPI fault mode {mode:?} into minisim MGM0"); + Ok(()) +} + fn handle_raw_tm_packet(data: &[u8]) -> anyhow::Result<()> { match spacepackets::CcsdsPacketReader::new_with_checksum(data) { Ok(packet) => { diff --git a/satrs-example/minisim/src/acs.rs b/satrs-example/minisim/src/acs.rs index fc008e5..9fa671f 100644 --- a/satrs-example/minisim/src/acs.rs +++ b/satrs-example/minisim/src/acs.rs @@ -7,7 +7,7 @@ use nexosim::{ use satrs_minisim::{ acs::{ lis3mdl::MgmLis3MdlReply, MgmReplyCommon, MgmReplyProvider, MgmSensorValuesMicroTesla, - MgtDipole, MgtHkSet, MgtReply, MGT_GEN_MAGNETIC_FIELD, + MgtDipole, MgtHkSet, MgtReply, SpiFaultMode, MGT_GEN_MAGNETIC_FIELD, }, SimReply, }; @@ -34,6 +34,7 @@ pub struct MagnetometerModel { #[allow(dead_code)] pub periodicity: Duration, pub external_mag_field: Option, + pub spi_fault: SpiFaultMode, pub reply_sender: mpsc::Sender, pub phatom: std::marker::PhantomData, } @@ -44,6 +45,7 @@ impl MagnetometerModel { switch_state: SwitchStateBinary::Off, periodicity, external_mag_field: None, + spi_fault: SpiFaultMode::None, reply_sender, phatom: std::marker::PhantomData, } @@ -55,12 +57,21 @@ impl MagnetometerModel { self.switch_state = switch_state; } + /// Force (or clear) a stuck-bus SPI fault, for FDIR testing purposes. + pub async fn set_spi_fault(&mut self, fault_mode: SpiFaultMode) { + self.spi_fault = fault_mode; + } + pub async fn send_sensor_values(&mut self, _: (), scheduler: &mut Context) { self.reply_sender - .send(ReplyProvider::create_mgm_reply(MgmReplyCommon { - switch_state: self.switch_state, - sensor_values: self.calculate_current_mgm_tuple(current_millis(scheduler.time())), - })) + .send(ReplyProvider::create_mgm_reply( + MgmReplyCommon { + switch_state: self.switch_state, + sensor_values: self + .calculate_current_mgm_tuple(current_millis(scheduler.time())), + }, + self.spi_fault, + )) .expect("sending MGM sensor values failed"); } @@ -182,7 +193,7 @@ pub mod tests { use satrs_minisim::{ acs::{ lis3mdl::{self, MgmLis3MdlReply}, - MgmRequestLis3Mdl, MgtDipole, MgtHkSet, MgtReply, MgtRequest, + MgmRequestLis3Mdl, MgtDipole, MgtHkSet, MgtReply, MgtRequest, SpiFaultMode, }, SerializableSimMsgPayload, SimComponent, SimMessageProvider, SimRequest, }; @@ -211,6 +222,37 @@ pub mod tests { assert_eq!(reply.common.sensor_values.z, 0.0); } + #[test] + fn test_mgm_spi_fault_injection_all_ones() { + let mut sim_testbench = SimTestbench::new(); + switch_device_on(&mut sim_testbench, SwitchId::Mgm0); + + let fault_request = + SimRequest::new_with_epoch_time(MgmRequestLis3Mdl::SetSpiFault(SpiFaultMode::AllOnes)); + sim_testbench + .send_request(fault_request) + .expect("sending MGM fault injection request failed"); + sim_testbench.handle_sim_requests_time_agnostic(); + sim_testbench.step().unwrap(); + + let data_request = SimRequest::new_with_epoch_time(MgmRequestLis3Mdl::RequestSensorData); + sim_testbench + .send_request(data_request) + .expect("sending MGM request failed"); + sim_testbench.handle_sim_requests_time_agnostic(); + sim_testbench.step().unwrap(); + let sim_reply = sim_testbench + .try_receive_next_reply() + .expect("no MGM reply received"); + let reply = MgmLis3MdlReply::from_sim_message(&sim_reply) + .expect("failed to deserialize MGM sensor values"); + // Even though the device is switched on, the injected fault forces a stuck-bus reply. + assert_eq!(reply.common.switch_state, SwitchStateBinary::On); + assert_eq!(reply.raw.x, -1); + assert_eq!(reply.raw.y, -1); + assert_eq!(reply.raw.z, -1); + } + #[test] fn test_basic_mgm_request_switched_on() { let mut sim_testbench = SimTestbench::new(); diff --git a/satrs-example/minisim/src/controller.rs b/satrs-example/minisim/src/controller.rs index 60312d4..c805128 100644 --- a/satrs-example/minisim/src/controller.rs +++ b/satrs-example/minisim/src/controller.rs @@ -124,6 +124,7 @@ impl SimController { } match sim_ctrl_request { SimCtrlRequest::Ping => { + log::info!("received ping request, a client is connecting"); self.reply_sender .send(SimReply::new(&SimCtrlReply::Pong)) .expect("sending reply from sim controller failed"); @@ -153,6 +154,18 @@ impl SimController { .process_event(MagnetometerModel::send_sensor_values, (), addr) .expect("event execution error for mgm"); } + MgmRequestLis3Mdl::SetSpiFault(fault_mode) => { + let addr = match mgm_idx { + 0 => &self.addr_wrapper.mgm_0_addr, + 1 => &self.addr_wrapper.mgm_1_addr, + + _ => panic!("invalid mgm index"), + }; + log::info!("MGM{mgm_idx}: setting SPI fault mode to {fault_mode:?}"); + self.simulation + .process_event(MagnetometerModel::set_spi_fault, fault_mode, addr) + .expect("event execution error for mgm"); + } } Ok(()) } diff --git a/satrs-example/minisim/src/lib.rs b/satrs-example/minisim/src/lib.rs index d659d90..f1587d8 100644 --- a/satrs-example/minisim/src/lib.rs +++ b/satrs-example/minisim/src/lib.rs @@ -202,12 +202,27 @@ pub mod acs { use super::*; pub trait MgmReplyProvider: Send + 'static { - fn create_mgm_reply(common: MgmReplyCommon) -> SimReply; + fn create_mgm_reply(common: MgmReplyCommon, fault_mode: SpiFaultMode) -> SimReply; + } + + /// Fault mode injected on the simulated SPI bus, independent of the switch state. + /// + /// Models the classic symptom of a stuck SPI bus: an undriven MISO line commonly reads + /// back as all-1s, a shorted/grounded one as all-0s. + #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] + pub enum SpiFaultMode { + #[default] + None, + AllZeros, + AllOnes, } #[derive(Debug, Copy, Clone, Serialize, Deserialize)] pub enum MgmRequestLis3Mdl { RequestSensorData, + /// Force the raw register reply into a stuck-bus pattern, regardless of switch state. + /// Used to test FDIR handling of SPI bus faults. + SetSpiFault(SpiFaultMode), } impl SerializableSimMsgPayload for MgmRequestLis3Mdl { @@ -236,6 +251,7 @@ pub mod acs { z: 30.0, }; pub const ALL_ONES_SENSOR_VAL: i16 = 0xffff_u16 as i16; + pub const ALL_ZEROS_SENSOR_VAL: i16 = 0; pub mod lis3mdl { use super::*; @@ -263,7 +279,30 @@ pub mod acs { } impl MgmLis3MdlReply { - pub fn new(common: MgmReplyCommon) -> Self { + pub fn new(common: MgmReplyCommon, fault_mode: SpiFaultMode) -> Self { + match fault_mode { + SpiFaultMode::AllZeros => { + return Self { + common, + raw: MgmLis3RawValues { + x: ALL_ZEROS_SENSOR_VAL, + y: ALL_ZEROS_SENSOR_VAL, + z: ALL_ZEROS_SENSOR_VAL, + }, + }; + } + SpiFaultMode::AllOnes => { + return Self { + common, + raw: MgmLis3RawValues { + x: ALL_ONES_SENSOR_VAL, + y: ALL_ONES_SENSOR_VAL, + z: ALL_ONES_SENSOR_VAL, + }, + }; + } + SpiFaultMode::None => (), + } match common.switch_state { SwitchStateBinary::Off => Self { common, @@ -306,8 +345,8 @@ pub mod acs { } impl MgmReplyProvider for MgmLis3MdlReply { - fn create_mgm_reply(common: MgmReplyCommon) -> SimReply { - SimReply::new(&Self::new(common)) + fn create_mgm_reply(common: MgmReplyCommon, fault_mode: SpiFaultMode) -> SimReply { + SimReply::new(&Self::new(common, fault_mode)) } } } diff --git a/satrs-example/src/acs/mgm.rs b/satrs-example/src/acs/mgm.rs index ca4fde7..78e2739 100644 --- a/satrs-example/src/acs/mgm.rs +++ b/satrs-example/src/acs/mgm.rs @@ -1,3 +1,5 @@ +use satrs::fdir::FaultCounterStd; +use satrs::health::{HealthState, HealthTableMapSync, HealthTableProvider}; use satrs::spacepackets::CcsdsPacketIdAndPsc; use satrs_example::{HkHelperSingleSet, ModeHelper, TimestampHelper, TmtcQueues}; use satrs_minisim::acs::MgmRequestLis3Mdl; @@ -14,8 +16,6 @@ use types::acs::mgm::response::ModeResponse; use types::pcdu::SwitchId; use types::{ComponentId, DeviceMode, HkRequestType, acs::mgm}; -use satrs::request::MessageMetadata; - use crate::ccsds::pack_ccsds_tm_packet_for_now; use crate::eps::PowerSwitchHelper; @@ -26,6 +26,15 @@ pub const X_LOWBYTE_IDX: usize = 9; pub const Y_LOWBYTE_IDX: usize = 11; pub const Z_LOWBYTE_IDX: usize = 13; +// FDIR configuration for a stuck SPI bus (data pinned to all-1s). Chosen so a handful of +// transient errors are tolerated but a persistently faulty bus is caught quickly. +// +// SPI itself cannot time out: the master clocks bytes in lockstep, so a transfer always +// completes. An unresponsive or dead device does not withhold a reply, it just leaves the bus +// floating, which is read back as this same all-1s pattern. +pub const SPI_FAULT_THRESHOLD: u32 = 2; +pub const SPI_FAULT_DECREMENT_AFTER: Duration = Duration::from_secs(30); + #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum MgmId { _0, @@ -159,9 +168,12 @@ pub struct MgmHandlerLis3Mdl { hk_helper: HkHelperSingleSet, mode_helpers: ModeHelper, mode_leaf_helper: ModeLeafHelper, + spi_fault_counter: FaultCounterStd, + health_table: HealthTableMapSync, } impl MgmHandlerLis3Mdl { + #[allow(clippy::too_many_arguments)] pub fn new( id: MgmId, tmtc_queues: TmtcQueues, @@ -170,6 +182,7 @@ impl MgmHandlerLis3Mdl { shared_mgm_set: Arc>, mode_leaf_helper: ModeLeafHelper, mode_timeout: Duration, + health_table: HealthTableMapSync, ) -> Self { Self { id, @@ -182,6 +195,8 @@ impl MgmHandlerLis3Mdl { stamp_helper: TimestampHelper::default(), hk_helper: HkHelperSingleSet::new(false, Duration::from_millis(200)), mode_leaf_helper, + spi_fault_counter: FaultCounterStd::new(SPI_FAULT_THRESHOLD, SPI_FAULT_DECREMENT_AFTER), + health_table, } } @@ -254,6 +269,25 @@ impl MgmHandlerLis3Mdl { )), ), }, + mgm::request::Request::Health(health_request) => { + match health_request { + mgm::request::HealthRequest::SetHealth(health_state) => { + log::info!( + "{}: setting health to {:?} via ground command", + self.id.str(), + health_state + ); + self.health_table.set_health( + self.id.component_id().into(), + health_state, + ); + self.send_telemetry( + Some(tc_id), + mgm::response::Response::Ok, + ); + } + } + } } } Err(e) => { @@ -358,6 +392,16 @@ impl MgmHandlerLis3Mdl { .try_into() .unwrap(), ); + // A stuck-high SPI bus (undriven MISO) reads back as all-1s on every register, + // regardless of what was actually requested. This is the pattern this codebase already + // uses for "no real device behind the bus" (see the switched-off MGM sim reply). An + // all-0s reading is not used here, since it collides with a legitimate zero-field + // reading and would cause false positives. + if x_raw == -1 && y_raw == -1 && z_raw == -1 { + self.register_spi_fault(); + return; + } + self.spi_fault_counter.try_decrement(); // Simple scaling to retrieve the float value, assuming the best sensor resolution. let mut mgm_guard = self.shared_mgm_set.lock().unwrap(); mgm_guard.x = x_raw as f32 * GAUSS_TO_MICROTESLA_FACTOR as f32 * FIELD_LSB_PER_GAUSS_4_SENS; @@ -367,6 +411,48 @@ impl MgmHandlerLis3Mdl { drop(mgm_guard); } + /// Registers one stuck-bus SPI fault with the FDIR fault counter, invalidating the current + /// sensor set. If the failure threshold is exceeded, the component is marked faulty in the + /// global health table. + fn register_spi_fault(&mut self) { + log::warn!("{}: stuck-bus SPI fault", self.id.str()); + self.shared_mgm_set.lock().unwrap().valid = false; + if !self.spi_fault_counter.increment_and_check() { + return; + } + // Ground may have taken manual control, or already given up on this component. + // Autonomous FDIR should not override that decision. + let component_id = self.id.component_id().into(); + match self.health_table.health(component_id) { + Some(HealthState::ExternalControl) | Some(HealthState::PermanentFaulty) => { + log::info!( + "{}: SPI fault threshold exceeded, but health is externally controlled, \ + not overriding", + self.id.str() + ); + } + _ => { + log::error!( + "{}: SPI fault threshold exceeded, marking component faulty", + self.id.str() + ); + self.health_table + .set_health(component_id, HealthState::Faulty); + // TODO: Event? Health-table changes are currently invisible to the ground + // except through this log line. Likely applies to other health/mode + // transitions across the example app too, not just this one. + // Do not restart an already pending Off transition: poll_sensor still calls + // this every cycle the fault persists, and current stays Normal until the + // transition completes, so re-triggering here would keep resetting the + // transition state machine before it can ever finish. + if self.mode_helpers.target != Some(DeviceMode::Off) { + log::warn!("{}: commanding device off due to fault", self.id.str()); + self.start_transition(DeviceMode::Off, true); + } + } + } + } + fn start_transition(&mut self, target_mode: DeviceMode, _forced: bool) { log::info!("{}: transitioning to mode {:?}", self.id.str(), target_mode); if target_mode == DeviceMode::Off { @@ -380,29 +466,33 @@ impl MgmHandlerLis3Mdl { return; } let target_mode = self.mode_helpers.target.unwrap(); - if target_mode == DeviceMode::On || target_mode == DeviceMode::Normal { - if self.mode_helpers.transition_state == TransitionState::Idle { - let result = self - .switch_helper - .send_switch_on_cmd(MessageMetadata::new(0, self.id as u32), self.switch_id()); - if result.is_err() { - // Could not send switch command.. still continue with transition. - log::error!("failed to send switch on command"); - } - self.mode_helpers.transition_state = TransitionState::PowerSwitching; + let switch_target_on = target_mode != DeviceMode::Off; + if self.mode_helpers.transition_state == TransitionState::Idle { + let result = if switch_target_on { + self.switch_helper.send_switch_on_cmd(self.switch_id()) + } else { + self.switch_helper.send_switch_off_cmd(self.switch_id()) + }; + if result.is_err() { + // Could not send switch command.. still continue with transition. + log::error!( + "failed to send switch {} command", + if switch_target_on { "on" } else { "off" } + ); } - if self.mode_helpers.transition_state == TransitionState::PowerSwitching { - if self.switch_helper.is_switch_on(self.switch_id()) { - log::info!("switch is on"); - self.mode_helpers.transition_state = TransitionState::Done; - } else if self.mode_helpers.timed_out() { - self.handle_mode_transition_failure(); - } - } - if self.mode_helpers.transition_state == TransitionState::Done { - self.handle_mode_reached(); + self.mode_helpers.transition_state = TransitionState::PowerSwitching; + } + if self.mode_helpers.transition_state == TransitionState::PowerSwitching { + if self.switch_helper.is_switch_on(self.switch_id()) == switch_target_on { + log::info!("switch is {}", if switch_target_on { "on" } else { "off" }); + self.mode_helpers.transition_state = TransitionState::Done; + } else if self.mode_helpers.timed_out() { + self.handle_mode_transition_failure(); } } + if self.mode_helpers.transition_state == TransitionState::Done { + self.handle_mode_reached(); + } } // Should be called to complete a mode transition which failed. @@ -460,7 +550,7 @@ mod tests { }; use arbitrary_int::u11; - use satrs::{request::GenericMessage, spacepackets::SpacePacketHeader}; + use satrs::spacepackets::SpacePacketHeader; use satrs_minisim::acs::lis3mdl::MgmLis3RawValues; use types::{ Apid, ComponentId, TcHeader, @@ -506,7 +596,8 @@ mod tests { pub shared_switch_set: SharedSwitchSet, pub tc_tx: mpsc::SyncSender, pub tm_rx: mpsc::Receiver, - pub switch_rx: mpsc::Receiver>, + pub switch_rx: mpsc::Receiver, + pub health_table: HealthTableMapSync, pub handler: MgmHandlerLis3Mdl, } @@ -526,6 +617,7 @@ mod tests { switch_map.insert(SwitchId::Mgm0, SwitchState::Off); let switch_map = SwitchSet::new(switch_map); let shared_switch_set = SharedSwitchSet::new(Mutex::new(switch_map)); + let health_table = HealthTableMapSync::default(); let handler = MgmHandlerLis3Mdl::new( MgmId::_0, TmtcQueues { tc_rx, tm_tx }, @@ -534,18 +626,37 @@ mod tests { shared_mgm_set, mode_leaf_helper, Duration::from_millis(100), + health_table.clone(), ); Self { assembly_mode_request_tx, mode_report_rx, shared_switch_set, switch_rx, + health_table, handler, tm_rx, tc_tx, } } + /// Switches the MGM to `Normal` mode, completing the power-switch handshake. + pub fn switch_to_normal(&mut self) { + self.tc_tx + .send(create_request_tc( + MgmSelect::_0, + mgm::request::Request::Mode(ModeRequest::SetMode(DeviceMode::Normal)), + )) + .unwrap(); + self.handler.periodic_operation(); + self.shared_switch_set + .lock() + .unwrap() + .set_switch_state(SwitchId::Mgm0, SwitchState::On); + self.handler.periodic_operation(); + assert_eq!(self.handler.mode(), DeviceMode::Normal); + } + pub fn test_spi_interface(&mut self) -> &mut TestSpiInterface { match &mut self.handler.spi_com { SpiCommunication::Dummy(_) | SpiCommunication::Sim(_) => { @@ -582,8 +693,8 @@ mod tests { // Verify power switch handling. let switch_req = testbench.switch_rx.try_recv().expect("no switch request"); - assert_eq!(switch_req.message.switch_id, SwitchId::Mgm0); - assert_eq!(switch_req.message.target_state, SwitchStateBinary::On); + assert_eq!(switch_req.switch_id, SwitchId::Mgm0); + assert_eq!(switch_req.target_state, SwitchStateBinary::On); // This simulates one cycle for the power switch to update. testbench @@ -767,4 +878,129 @@ mod tests { matches!(testbench.tm_rx.try_recv(), Err(TryRecvError::Empty)); } + + #[test] + fn test_spi_fault_below_threshold_stays_healthy() { + let mut testbench = MgmTestbench::new(); + testbench.switch_to_normal(); + testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues { + x: -1, + y: -1, + z: -1, + }; + // One stuck-bus reading should not be enough to trip SPI_FAULT_THRESHOLD. + testbench.handler.periodic_operation(); + assert_eq!( + testbench.health_table.health(ComponentId::AcsMgm0.into()), + None, + "component should not be marked faulty yet" + ); + assert!(!testbench.handler.shared_mgm_set.lock().unwrap().valid); + } + + #[test] + fn test_spi_fault_above_threshold_marks_component_faulty() { + let mut testbench = MgmTestbench::new(); + testbench.switch_to_normal(); + testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues { + x: -1, + y: -1, + z: -1, + }; + // SPI_FAULT_THRESHOLD is exceeded on the (threshold + 1)-th stuck-bus reading. + for _ in 0..SPI_FAULT_THRESHOLD + 1 { + testbench.handler.periodic_operation(); + } + assert_eq!( + testbench.health_table.health(ComponentId::AcsMgm0.into()), + Some(HealthState::Faulty) + ); + assert!(!testbench.handler.shared_mgm_set.lock().unwrap().valid); + } + + #[test] + fn test_spi_fault_above_threshold_commands_device_off() { + let mut testbench = MgmTestbench::new(); + testbench.switch_to_normal(); + // Drain the switch-on request left over from switch_to_normal(). + testbench + .switch_rx + .try_recv() + .expect("no switch-on request sent"); + testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues { + x: -1, + y: -1, + z: -1, + }; + for _ in 0..SPI_FAULT_THRESHOLD + 1 { + testbench.handler.periodic_operation(); + } + assert_eq!( + testbench.health_table.health(ComponentId::AcsMgm0.into()), + Some(HealthState::Faulty) + ); + // The Off transition was only started on the last iteration above, so it has not sent + // its switch-off request yet: drive one more cycle to let it do so. + testbench.handler.periodic_operation(); + let switch_req = testbench + .switch_rx + .try_recv() + .expect("no switch-off request sent after fault"); + assert_eq!(switch_req.switch_id, SwitchId::Mgm0); + assert_eq!(switch_req.target_state, SwitchStateBinary::Off); + + // Simulate the PCDU acting on the switch-off request. + testbench + .shared_switch_set + .lock() + .unwrap() + .set_switch_state(SwitchId::Mgm0, SwitchState::Off); + testbench.handler.periodic_operation(); + + assert_eq!(testbench.handler.mode(), DeviceMode::Off); + } + + #[test] + fn test_spi_fault_does_not_override_external_control() { + let mut testbench = MgmTestbench::new(); + testbench.switch_to_normal(); + testbench + .health_table + .set_health(ComponentId::AcsMgm0.into(), HealthState::ExternalControl); + testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues { + x: -1, + y: -1, + z: -1, + }; + for _ in 0..SPI_FAULT_THRESHOLD + 1 { + testbench.handler.periodic_operation(); + } + // Ground took manual control; autonomous FDIR must not override that decision. + assert_eq!( + testbench.health_table.health(ComponentId::AcsMgm0.into()), + Some(HealthState::ExternalControl) + ); + } + + #[test] + fn test_recovering_from_spi_fault_clears_invalid_data_flag() { + let mut testbench = MgmTestbench::new(); + testbench.switch_to_normal(); + testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues { + x: -1, + y: -1, + z: -1, + }; + testbench.handler.periodic_operation(); + assert!(!testbench.handler.shared_mgm_set.lock().unwrap().valid); + + // Bus recovers before the threshold is exceeded. + testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues::default(); + testbench.handler.periodic_operation(); + assert_eq!( + testbench.health_table.health(ComponentId::AcsMgm0.into()), + None + ); + assert!(testbench.handler.shared_mgm_set.lock().unwrap().valid); + } } diff --git a/satrs-example/src/eps/mod.rs b/satrs-example/src/eps/mod.rs index 8075bf1..6e3b714 100644 --- a/satrs-example/src/eps/mod.rs +++ b/satrs-example/src/eps/mod.rs @@ -2,10 +2,7 @@ use derive_new::new; use std::{cell::RefCell, collections::VecDeque, sync::mpsc, time::Duration}; use types::pcdu::{SwitchId, SwitchRequest, SwitchState, SwitchStateBinary}; -use satrs::{ - queue::GenericSendError, - request::{GenericMessage, MessageMetadata}, -}; +use satrs::{queue::GenericSendError, request::MessageMetadata}; use thiserror::Error; use crate::eps::pcdu::SwitchMapWrapper; @@ -16,7 +13,7 @@ pub mod pcdu; #[derive(new, Clone)] pub struct PowerSwitchHelper { - switcher_tx: mpsc::SyncSender>, + switcher_tx: mpsc::SyncSender, shared_switch_set: SharedSwitchSet, } @@ -37,28 +34,15 @@ pub enum SwitchInfoError { } impl PowerSwitchHelper { - pub fn send_switch_on_cmd( - &self, - requestor_info: satrs::request::MessageMetadata, - switch_id: SwitchId, - ) -> Result<(), GenericSendError> { - self.switcher_tx.send(GenericMessage::new( - requestor_info, - SwitchRequest::new(switch_id, SwitchStateBinary::On), - ))?; + pub fn send_switch_on_cmd(&self, switch_id: SwitchId) -> Result<(), GenericSendError> { + self.switcher_tx + .send(SwitchRequest::new(switch_id, SwitchStateBinary::On))?; Ok(()) } - #[allow(dead_code)] - pub fn send_switch_off_cmd( - &self, - requestor_info: satrs::request::MessageMetadata, - switch_id: SwitchId, - ) -> Result<(), GenericSendError> { - self.switcher_tx.send(GenericMessage::new( - requestor_info, - SwitchRequest::new(switch_id, SwitchStateBinary::Off), - ))?; + pub fn send_switch_off_cmd(&self, switch_id: SwitchId) -> Result<(), GenericSendError> { + self.switcher_tx + .send(SwitchRequest::new(switch_id, SwitchStateBinary::Off))?; Ok(()) } diff --git a/satrs-example/src/eps/pcdu.rs b/satrs-example/src/eps/pcdu.rs index a09dd59..41190cc 100644 --- a/satrs-example/src/eps/pcdu.rs +++ b/satrs-example/src/eps/pcdu.rs @@ -6,7 +6,7 @@ use std::{ use derive_new::new; use num_enum::{IntoPrimitive, TryFromPrimitive}; -use satrs::{request::GenericMessage, spacepackets::CcsdsPacketIdAndPsc}; +use satrs::spacepackets::CcsdsPacketIdAndPsc; use satrs_example::TimestampHelper; use satrs_minisim::{ SerializableSimMsgPayload, SimReply, SimRequest, @@ -264,7 +264,7 @@ pub enum OpCode { #[allow(clippy::too_many_arguments)] pub struct PcduHandler { dev_str: &'static str, - switch_request_rx: mpsc::Receiver>, + switch_request_rx: mpsc::Receiver, tc_rx: std::sync::mpsc::Receiver, tm_tx: mpsc::SyncSender, pub com_interface: ComInterface, @@ -277,7 +277,7 @@ impl PcduHandler { pub fn new( tc_rx: std::sync::mpsc::Receiver, tm_tx: std::sync::mpsc::SyncSender, - switch_request_rx: mpsc::Receiver>, + switch_request_rx: mpsc::Receiver, com_interface: ComInterface, shared_switch_map: Arc>, init_mode: DeviceMode, @@ -488,10 +488,7 @@ impl PcduHandler { loop { match self.switch_request_rx.try_recv() { Ok(switch_req) => { - self.handle_device_switching( - switch_req.message.switch_id(), - switch_req.message.target_state(), - ); + self.handle_device_switching(switch_req.switch_id(), switch_req.target_state()); } Err(e) => match e { mpsc::TryRecvError::Empty => break, @@ -531,10 +528,7 @@ mod tests { use std::sync::mpsc; use arbitrary_int::u11; - use satrs::{ - request::{GenericMessage, MessageMetadata}, - spacepackets::SpacePacketHeader, - }; + use satrs::spacepackets::SpacePacketHeader; use types::{ Apid, TcHeader, pcdu::{SwitchMapBinary, SwitchStateBinary}, @@ -593,7 +587,7 @@ mod tests { pub mode_reply_rx_to_parent: mpsc::Receiver, pub tc_tx: mpsc::SyncSender, pub tm_rx: mpsc::Receiver, - pub switch_request_tx: mpsc::Sender>, + pub switch_request_tx: mpsc::Sender, pub handler: PcduHandler, } @@ -730,10 +724,7 @@ mod tests { .unwrap(); testbench .switch_request_tx - .send(GenericMessage::new( - MessageMetadata::new(0, ComponentId::AcsMgm0 as u32), - SwitchRequest::new(SwitchId::Mgm0, SwitchStateBinary::On), - )) + .send(SwitchRequest::new(SwitchId::Mgm0, SwitchStateBinary::On)) .expect("failed to send switch request"); testbench.handler.periodic_operation(OpCode::RegularOp); testbench diff --git a/satrs-example/src/main.rs b/satrs-example/src/main.rs index 8eeb968..bf4a98a 100644 --- a/satrs-example/src/main.rs +++ b/satrs-example/src/main.rs @@ -156,6 +156,9 @@ fn main() { let (switch_request_tx, switch_request_rx) = mpsc::sync_channel(20); let switch_helper = PowerSwitchHelper::new(switch_request_tx, shared_switch_set.clone()); + // Global FDIR health table, shared by all software objects. + let health_table = satrs::health::HealthTableMapSync::default(); + let shared_mgm_0_set = Arc::default(); let shared_mgm_1_set = Arc::default(); let (mgm_0_spi_interface, mgm_1_spi_interface) = @@ -194,6 +197,7 @@ fn main() { report_tx: mgm_0_mode_report_tx, }, Duration::from_millis(1000), + health_table.clone(), ); let mut mgm_1_handler = mgm::MgmHandlerLis3Mdl::new( mgm::MgmId::_1, @@ -209,6 +213,7 @@ fn main() { report_tx: mgm_1_mode_report_tx, }, Duration::from_millis(1000), + health_table.clone(), ); let mut mgm_assembly = mgm_assembly::Assembly::new( mgm_assembly::ParentQueueHelper { diff --git a/satrs-example/types/Cargo.toml b/satrs-example/types/Cargo.toml index 99b7a90..1911db2 100644 --- a/satrs-example/types/Cargo.toml +++ b/satrs-example/types/Cargo.toml @@ -6,7 +6,7 @@ edition = "2024" [dependencies] serde = { version = "1", features = ["derive"] } spacepackets = { version = "0.18", default-features = false } -satrs = { path = "../../satrs" } +satrs = { path = "../../satrs", features = ["serde"] } num_enum = { version = "0.7" } strum = { version = "0.28", features = ["derive"] } postcard = { version = "1" } diff --git a/satrs-example/types/src/acs/mgm.rs b/satrs-example/types/src/acs/mgm.rs index 8868f30..1f6c147 100644 --- a/satrs-example/types/src/acs/mgm.rs +++ b/satrs-example/types/src/acs/mgm.rs @@ -7,6 +7,13 @@ pub mod request { ReadMode, } + #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Copy, PartialEq, Eq)] + pub enum HealthRequest { + /// Overrides the device's autonomous FDIR health state, for example to clear a `Faulty` + /// state set by the handler after ground has fixed or worked around the underlying issue. + SetHealth(satrs::health::HealthState), + } + #[derive(Debug, PartialEq, Eq, Clone, Copy, serde::Serialize, serde::Deserialize)] pub enum HkId { Sensor, @@ -23,6 +30,7 @@ pub mod request { Ping, Hk(HkRequest), Mode(ModeRequest), + Health(HealthRequest), } impl Request { @@ -31,6 +39,7 @@ pub mod request { Request::Ping => crate::MessageType::Verification, Request::Hk(_hk_request) => crate::MessageType::Hk, Request::Mode(_mode) => crate::MessageType::Mode, + Request::Health(_health) => crate::MessageType::Health, } } } diff --git a/satrs-example/types/src/lib.rs b/satrs-example/types/src/lib.rs index d79afd7..44c370f 100644 --- a/satrs-example/types/src/lib.rs +++ b/satrs-example/types/src/lib.rs @@ -150,6 +150,7 @@ pub enum MessageType { Action, Event, Verification, + Health, } pub trait Message { diff --git a/satrs/Cargo.toml b/satrs/Cargo.toml index 023ea3b..181c7f9 100644 --- a/satrs/Cargo.toml +++ b/satrs/Cargo.toml @@ -72,7 +72,7 @@ alloc = [ ] serde = ["dep:serde", "spacepackets/serde", "satrs-shared/serde"] crossbeam = ["crossbeam-channel"] -defmt = ["dep:defmt", "spacepackets/defmt"] +defmt = ["dep:defmt", "spacepackets/defmt", "embassy-time?/defmt"] embassy-time = ["dep:embassy-time"] test_util = [] diff --git a/satrs/src/fdir.rs b/satrs/src/fdir.rs new file mode 100644 index 0000000..4d33b17 --- /dev/null +++ b/satrs/src/fdir.rs @@ -0,0 +1,292 @@ +//! # FDIR (Fault Detection, Isolation and Recovery) helpers +//! +//! A fault counter tracks a monotonic fault count, decrements it over time when faults stop +//! occurring, and reports when a configured failure threshold has been exceeded. This is the +//! typical building block used to turn a stream of transient error reports into a single +//! "component is faulty" decision without reacting to the first isolated error. +//! +//! The design follows the FSFW `FaultCounter`: +//! +//! +//! Pick a variant based on what clock is available: +//! +//! - [FaultCounterStd]: `std::time::Instant`, behind the `std` feature. +#![cfg_attr( + feature = "embassy-time", + doc = "- [FaultCounterEmbassy]: `embassy_time::Instant`, behind the `embassy-time` feature." +)] +#![deny(missing_docs)] + +/// Fault counter backed by [std::time::Instant]. +#[cfg(feature = "std")] +#[derive(Debug, Clone)] +pub struct FaultCounterStd { + fault_count: u32, + failure_threshold: u32, + decrement_after: core::time::Duration, + last_decrement: Option, +} + +#[cfg(feature = "std")] +impl FaultCounterStd { + /// Create a new [`FaultCounterStd`]. + /// + /// - `failure_threshold`: threshold above which [`Self::above_threshold`] returns `true` and + /// resets the internal count. + /// - `decrement_after`: minimum duration between automatic decrements performed by + /// [`Self::try_decrement`]. + pub fn new(failure_threshold: u32, decrement_after: core::time::Duration) -> Self { + Self { + fault_count: 0, + failure_threshold, + decrement_after, + last_decrement: None, + } + } + + /// Current fault count. + pub fn fault_count(&self) -> u32 { + self.fault_count + } + + /// Increase the fault count by `1`. + /// + /// If the counter was previously `0`, this starts a new decrement clock. + pub fn increment(&mut self) { + if self.fault_count == 0 { + self.last_decrement = Some(std::time::Instant::now()); + } + self.fault_count += 1; + } + + /// Increase the fault count by `n`. + pub fn increment_n(&mut self, n: u32) { + for _ in 0..n { + self.increment(); + } + } + + fn has_decrement_timedout(&self) -> bool { + match self.last_decrement { + Some(last_decrement) => last_decrement.elapsed() >= self.decrement_after, + None => false, + } + } + + /// Decrease the fault count by `1` if the decrement timeout elapsed. + /// + /// Returns `true` if a decrement was performed, `false` otherwise. A decrement is only + /// performed when the counter is non-zero and at least `decrement_after` has elapsed since + /// the last decrement. + pub fn try_decrement(&mut self) -> bool { + if self.fault_count == 0 || !self.has_decrement_timedout() { + return false; + } + self.last_decrement = Some(std::time::Instant::now()); + self.fault_count -= 1; + true + } + + /// Check whether the counter exceeded the failure threshold. + /// + /// Returns `true` when `fault_count > failure_threshold`. In that case, the counter is reset + /// to `0`. + pub fn above_threshold(&mut self) -> bool { + if self.fault_count > self.failure_threshold { + self.fault_count = 0; + return true; + } + false + } + + /// Convenience helper to increment once and immediately check the threshold. + pub fn increment_and_check(&mut self) -> bool { + self.increment(); + self.above_threshold() + } + + /// Clear the counter and decrement timing state. + pub fn clear(&mut self) { + self.fault_count = 0; + self.last_decrement = None; + } + + /// Update the failure threshold used by [`Self::above_threshold`]. + pub fn set_failure_threshold(&mut self, threshold: u32) { + self.failure_threshold = threshold; + } + + /// Update the minimum interval between automatic decrements. + pub fn set_decrement_after(&mut self, duration: core::time::Duration) { + self.decrement_after = duration; + } +} + +/// Fault counter backed by [embassy_time::Instant]. +#[cfg(feature = "embassy-time")] +#[derive(Debug, Clone, Copy)] +#[cfg_attr(feature = "defmt", derive(defmt::Format))] +pub struct FaultCounterEmbassy { + fault_count: u32, + failure_threshold: u32, + decrement_after: embassy_time::Duration, + last_decrement: Option, +} + +#[cfg(feature = "embassy-time")] +impl FaultCounterEmbassy { + /// Create a new [`FaultCounterEmbassy`]. + /// + /// - `failure_threshold`: threshold above which [`Self::above_threshold`] returns `true` and + /// resets the internal count. + /// - `decrement_after`: minimum duration between automatic decrements performed by + /// [`Self::try_decrement`]. + pub fn new(failure_threshold: u32, decrement_after: embassy_time::Duration) -> Self { + Self { + fault_count: 0, + failure_threshold, + decrement_after, + last_decrement: None, + } + } + + /// Current fault count. + pub fn fault_count(&self) -> u32 { + self.fault_count + } + + /// Increase the fault count by `1`. + /// + /// If the counter was previously `0`, this starts a new decrement clock. + pub fn increment(&mut self) { + if self.fault_count == 0 { + self.last_decrement = Some(embassy_time::Instant::now()); + } + self.fault_count += 1; + } + + /// Increase the fault count by `n`. + pub fn increment_n(&mut self, n: u32) { + for _ in 0..n { + self.increment(); + } + } + + fn has_decrement_timedout(&self) -> bool { + match self.last_decrement { + Some(last_decrement) => { + embassy_time::Instant::now().duration_since(last_decrement) >= self.decrement_after + } + None => false, + } + } + + /// Decrease the fault count by `1` if the decrement timeout elapsed. + /// + /// Returns `true` if a decrement was performed, `false` otherwise. A decrement is only + /// performed when the counter is non-zero and at least `decrement_after` has elapsed since + /// the last decrement. + pub fn try_decrement(&mut self) -> bool { + if self.fault_count == 0 || !self.has_decrement_timedout() { + return false; + } + self.last_decrement = Some(embassy_time::Instant::now()); + self.fault_count -= 1; + true + } + + /// Check whether the counter exceeded the failure threshold. + /// + /// Returns `true` when `fault_count > failure_threshold`. In that case, the counter is reset + /// to `0`. + pub fn above_threshold(&mut self) -> bool { + if self.fault_count > self.failure_threshold { + self.fault_count = 0; + return true; + } + false + } + + /// Convenience helper to increment once and immediately check the threshold. + pub fn increment_and_check(&mut self) -> bool { + self.increment(); + self.above_threshold() + } + + /// Clear the counter and decrement timing state. + pub fn clear(&mut self) { + self.fault_count = 0; + self.last_decrement = None; + } + + /// Update the failure threshold used by [`Self::above_threshold`]. + pub fn set_failure_threshold(&mut self, threshold: u32) { + self.failure_threshold = threshold; + } + + /// Update the minimum interval between automatic decrements. + pub fn set_decrement_after(&mut self, duration: embassy_time::Duration) { + self.decrement_after = duration; + } +} + +#[cfg(all(test, feature = "std"))] +mod tests { + use super::*; + use std::thread; + use std::time::Duration; + + #[test] + fn threshold_not_exceeded_below_limit() { + let mut fc = FaultCounterStd::new(2, Duration::from_secs(60)); + assert!(!fc.increment_and_check()); + assert!(!fc.increment_and_check()); + assert_eq!(fc.fault_count(), 2); + } + + #[test] + fn threshold_exceeded_resets_counter() { + let mut fc = FaultCounterStd::new(2, Duration::from_secs(60)); + fc.increment_n(3); + assert!(fc.above_threshold()); + assert_eq!(fc.fault_count(), 0); + assert!(!fc.above_threshold()); + } + + #[test] + fn decrement_only_after_timeout() { + let mut fc = FaultCounterStd::new(5, Duration::from_millis(20)); + fc.increment(); + assert!(!fc.try_decrement()); + thread::sleep(Duration::from_millis(30)); + assert!(fc.try_decrement()); + assert_eq!(fc.fault_count(), 0); + } + + #[test] + fn decrement_noop_when_empty() { + let mut fc = FaultCounterStd::new(5, Duration::from_millis(1)); + thread::sleep(Duration::from_millis(2)); + assert!(!fc.try_decrement()); + } + + #[test] + fn increment_after_empty_resets_decrement_timing() { + let mut fc = FaultCounterStd::new(5, Duration::from_millis(20)); + fc.increment(); + thread::sleep(Duration::from_millis(30)); + assert!(fc.try_decrement()); + // Counter is 0 again, incrementing should require a fresh decrement_after wait. + fc.increment(); + assert!(!fc.try_decrement()); + } + + #[test] + fn clear_resets_state() { + let mut fc = FaultCounterStd::new(1, Duration::from_secs(60)); + fc.increment_n(2); + fc.clear(); + assert_eq!(fc.fault_count(), 0); + assert!(!fc.above_threshold()); + } +} diff --git a/satrs/src/health.rs b/satrs/src/health.rs index 9d32457..20e563b 100644 --- a/satrs/src/health.rs +++ b/satrs/src/health.rs @@ -1,6 +1,7 @@ use crate::ComponentId; #[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum HealthState { Healthy = 1, Faulty = 2, @@ -27,6 +28,15 @@ impl HealthTableMapSync { } } +#[cfg(feature = "std")] +impl Default for HealthTableMapSync { + /// Creates an empty, shared health table. Absent entries are up to the consumer to + /// interpret, for example as [HealthState::Healthy] by default. + fn default() -> Self { + Self::new(hashbrown::HashMap::new()) + } +} + #[cfg(feature = "std")] impl HealthTableProvider for HealthTableMapSync { fn health(&self, id: ComponentId) -> Option { diff --git a/satrs/src/lib.rs b/satrs/src/lib.rs index 70f9dfc..533b880 100644 --- a/satrs/src/lib.rs +++ b/satrs/src/lib.rs @@ -15,6 +15,7 @@ pub mod ccsds; pub mod encoding; #[cfg(feature = "std")] pub mod executable; +pub mod fdir; pub mod hal; pub mod health; /// Helpers to track when housekeeping sets need to be regenerated.