rust/sat-rs
12
1
Fork 0
Code Issues 6 Pull Requests 2 Projects Releases 29 Wiki Activity

feat: add FDIR fault counter and wire it into the MGM device handler

Browse Source 
Add satrs::fdir::FaultCounter, an FSFW-style error threshold counter:
counts faults, decrements over time when faults stop, and reports
when a threshold is exceeded. Two variants for now, mirroring the
hk.rs helper pattern:
- FaultCounterStd, backed by std::time::Instant
- FaultCounterEmbassy, backed by embassy_time::Instant (embassy-time
  feature), with an optional defmt::Format impl gated on the defmt
  feature

Add satrs::health::HealthTableMapSync::default() for easy construction
of a shared, global health table.

Wire both into the example app's MGM device handler as the first real
FDIR use case:
- the minisim MGM model gains SpiFaultMode (None/AllZeros/AllOnes) and
  a SetSpiFault request, so a stuck SPI bus can be injected for testing,
  independent of switch state
- MgmHandlerLis3Mdl::poll_sensor checks the SPI transfer result: a
  comm timeout or an all-1s stuck-bus reply (the same pattern the sim
  already uses for "device off") counts as a fault. Above threshold,
  the component is marked Faulty in a HealthTableMapSync shared from
  main.rs. This logic lives in the device handler, not the SPI comm
  layer, since deciding what a failed transfer means for FDIR is a
  handler concern.
- an all-0s reply is deliberately not treated as a fault, since it
  collides with a legitimate zero-field reading
This commit is contained in:
Robin Mueller
2026-09-16 13:56:47 +02:00
parent 7d6e4ca2f8
commit 86d8528d21
16 changed files with 892 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
satrs-example/client/Cargo.toml
+3
View File
@@ -9,7 +9,10 @@ log = "0.4"
fern = "0.7"
humantime = "2"
serde = { version = "1" }
serde_json = "1"
satrs = { path = "../../satrs" }
satrs-example = { path = ".." }
satrs-minisim = { path = "../minisim" }
types = { path = "../types" }
spacepackets = { version = "0.18", default-features = false }
bitbybit = "2"
satrs-example/client/src/main.rs
+188 -68
View File
@@ -2,9 +2,13 @@ use anyhow::bail;
use arbitrary_int::u11;
use clap::Parser as _;
use satrs_example::config::{OBSW_SERVER_ADDR, SERVER_PORT};
use satrs_minisim::{
SerializableSimMsgPayload, SimComponent, SimCtrlReply, SimCtrlRequest, SimMessageProvider,
SimReply, SimRequest, acs::MgmRequestLis3Mdl, acs::SpiFaultMode, udp::SIM_CTRL_PORT,
};
use spacepackets::{CcsdsPacketIdAndPsc, SpacePacketHeader};
use std::{
net::{IpAddr, SocketAddr, UdpSocket},
net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
sync::{
Arc,
atomic::{AtomicBool, Ordering},
@@ -32,14 +36,40 @@ enum Commands {
AcsSubsystem(SubsystemArgs),
}
impl Commands {
#[inline]
pub fn target_id(&self) -> types::ComponentId {
match self {
Commands::Mgm0(_mgm_args) => types::ComponentId::AcsMgm0,
Commands::Mgm1(_mgm_args) => types::ComponentId::AcsMgm1,
Commands::MgmAssy(_mgm_assembly_args) => types::ComponentId::AcsMgmAssembly,
Commands::AcsSubsystem(_subsystem_args) => types::ComponentId::AcsSubsystem,
#[derive(Debug, PartialEq, Eq, Clone, Copy, clap::ValueEnum)]
enum SpiFaultModeSelect {
None,
AllZeros,
AllOnes,
}
impl From<SpiFaultModeSelect> for SpiFaultMode {
fn from(mode: SpiFaultModeSelect) -> Self {
match mode {
SpiFaultModeSelect::None => SpiFaultMode::None,
SpiFaultModeSelect::AllZeros => SpiFaultMode::AllZeros,
SpiFaultModeSelect::AllOnes => SpiFaultMode::AllOnes,
}
}
}
#[derive(Debug, PartialEq, Eq, Clone, Copy, clap::ValueEnum)]
enum HealthStateSelect {
Healthy,
Faulty,
PermanentFaulty,
ExternalControl,
NeedsRecovery,
}
impl From<HealthStateSelect> for satrs::health::HealthState {
fn from(state: HealthStateSelect) -> Self {
match state {
HealthStateSelect::Healthy => satrs::health::HealthState::Healthy,
HealthStateSelect::Faulty => satrs::health::HealthState::Faulty,
HealthStateSelect::PermanentFaulty => satrs::health::HealthState::PermanentFaulty,
HealthStateSelect::ExternalControl => satrs::health::HealthState::ExternalControl,
HealthStateSelect::NeedsRecovery => satrs::health::HealthState::NeedsRecovery,
}
}
}
@@ -52,6 +82,16 @@ struct MgmArgs {
request_hk: bool,
#[arg(short, long)]
mode: Option<DeviceModeSelect>,
/// Inject (or clear) an SPI bus failure on the simulated device, bypassing the OBSW.
///
/// Only takes effect for MGM0: minisim always routes this fault to the MGM0 model
/// regardless of which MGM the request names (a pre-existing minisim limitation).
#[arg(long, value_enum)]
spi_fault: Option<SpiFaultModeSelect>,
/// Override the device's FDIR health state, for example to clear a `Faulty` state set by
/// the handler after the underlying issue has been fixed or worked around.
#[arg(long, value_enum)]
health: Option<HealthStateSelect>,
}
#[derive(Debug, PartialEq, Eq, Clone, Copy, clap::Parser)]
@@ -89,6 +129,93 @@ pub enum SubsystemModeSelect {
Safe,
}
fn handle_mgm_command(
client: &UdpSocket,
addr: SocketAddr,
target_id: types::ComponentId,
args: MgmArgs,
) -> anyhow::Result<()> {
if let Some(mode) = args.spi_fault {
if target_id != types::ComponentId::AcsMgm0 {
bail!("SPI fault injection is only supported for MGM0 right now (minisim limitation)");
}
inject_mgm_failure(mode.into())?;
}
if args.ping {
let request = types::ccsds::CcsdsTcPacketOwned::new_with_request(
SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)),
TcHeader::new(target_id, types::MessageType::Ping),
types::acs::mgm::request::Request::Ping,
);
let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header);
log::info!(
"sending {:?} ping request with TC ID {:#010x}",
target_id,
sent_tc_id.raw()
);
let request_packet = request.to_vec();
client.send_to(&request_packet, addr).unwrap();
}
if args.request_hk {
let request = types::ccsds::CcsdsTcPacketOwned::new_with_request(
SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)),
TcHeader::new(target_id, types::MessageType::Hk),
types::acs::mgm::request::Request::Hk(HkRequest {
id: types::acs::mgm::request::HkId::Sensor,
req_type: types::HkRequestType::OneShot,
}),
);
let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header);
log::info!(
"sending {:?} HK request with TC ID {:#010x}",
target_id,
sent_tc_id.raw()
);
let request_packet = request.to_vec();
client.send_to(&request_packet, addr).unwrap();
}
if let Some(mode) = args.mode {
let dev_mode = match mode {
DeviceModeSelect::Off => types::DeviceMode::Off,
DeviceModeSelect::Normal => types::DeviceMode::Normal,
};
let request = types::ccsds::CcsdsTcPacketOwned::new_with_request(
SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)),
TcHeader::new(target_id, types::MessageType::Mode),
types::acs::mgm::request::Request::Mode(
types::acs::mgm::request::ModeRequest::SetMode(dev_mode),
),
);
let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header);
log::info!(
"sending {:?} HK request with TC ID {:#010x}",
target_id,
sent_tc_id.raw()
);
let request_packet = request.to_vec();
client.send_to(&request_packet, addr).unwrap();
}
if let Some(health) = args.health {
let request = types::ccsds::CcsdsTcPacketOwned::new_with_request(
SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)),
TcHeader::new(target_id, types::MessageType::Health),
types::acs::mgm::request::Request::Health(
types::acs::mgm::request::HealthRequest::SetHealth(health.into()),
),
);
let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header);
log::info!(
"sending {:?} set-health request with TC ID {:#010x}",
target_id,
sent_tc_id.raw()
);
let request_packet = request.to_vec();
client.send_to(&request_packet, addr).unwrap();
}
Ok(())
}
fn setup_logger(level: log::LevelFilter) -> Result<(), fern::InitError> {
fern::Dispatch::new()
.format(|out, message, record| {
@@ -145,70 +272,19 @@ fn main() -> anyhow::Result<()> {
client.send_to(&request_packet, addr).unwrap();
}
if let Some(cmd) = cli.commands {
let target_id = cmd.target_id();
match cmd {
Commands::Mgm0(args) | Commands::Mgm1(args) => {
if args.ping {
let request = types::ccsds::CcsdsTcPacketOwned::new_with_request(
SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)),
TcHeader::new(cmd.target_id(), types::MessageType::Ping),
types::acs::mgm::request::Request::Ping,
);
let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header);
log::info!(
"sending {:?} ping request with TC ID {:#010x}",
target_id,
sent_tc_id.raw()
);
let request_packet = request.to_vec();
client.send_to(&request_packet, addr).unwrap();
}
if args.request_hk {
let request = types::ccsds::CcsdsTcPacketOwned::new_with_request(
SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)),
TcHeader::new(target_id, types::MessageType::Hk),
types::acs::mgm::request::Request::Hk(HkRequest {
id: types::acs::mgm::request::HkId::Sensor,
req_type: types::HkRequestType::OneShot,
}),
);
let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header);
log::info!(
"sending {:?} HK request with TC ID {:#010x}",
target_id,
sent_tc_id.raw()
);
let request_packet = request.to_vec();
client.send_to(&request_packet, addr).unwrap();
}
if let Some(mode) = args.mode {
let dev_mode = match mode {
DeviceModeSelect::Off => types::DeviceMode::Off,
DeviceModeSelect::Normal => types::DeviceMode::Normal,
};
let request = types::ccsds::CcsdsTcPacketOwned::new_with_request(
SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)),
TcHeader::new(target_id, types::MessageType::Mode),
types::acs::mgm::request::Request::Mode(
types::acs::mgm::request::ModeRequest::SetMode(dev_mode),
),
);
let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header);
log::info!(
"sending {:?} HK request with TC ID {:#010x}",
target_id,
sent_tc_id.raw()
);
let request_packet = request.to_vec();
client.send_to(&request_packet, addr).unwrap();
}
Commands::Mgm0(args) => {
handle_mgm_command(&client, addr, types::ComponentId::AcsMgm0, args)?
}
Commands::Mgm1(args) => {
handle_mgm_command(&client, addr, types::ComponentId::AcsMgm1, args)?
}
Commands::MgmAssy(mgm_assembly_args) => {
let target_id = types::ComponentId::AcsMgmAssembly;
if mgm_assembly_args.ping {
let request = types::ccsds::CcsdsTcPacketOwned::new_with_request(
SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)),
TcHeader::new(cmd.target_id(), types::MessageType::Ping),
TcHeader::new(target_id, types::MessageType::Ping),
types::acs::mgm::request::Request::Ping,
);
let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header);
@@ -251,10 +327,11 @@ fn main() -> anyhow::Result<()> {
}
}
Commands::AcsSubsystem(subsystem_args) => {
let target_id = types::ComponentId::AcsSubsystem;
if subsystem_args.ping {
let request = types::ccsds::CcsdsTcPacketOwned::new_with_request(
SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)),
TcHeader::new(cmd.target_id(), types::MessageType::Ping),
TcHeader::new(target_id, types::MessageType::Ping),
types::acs::subsystem::request::Request::Ping,
);
let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header);
@@ -314,6 +391,49 @@ fn main() -> anyhow::Result<()> {
Ok(())
}
/// Injects the given SPI fault mode directly into minisim's MGM0 model, bypassing the OBSW.
///
/// Confirms the simulator is actually reachable first (same ping/pong check the OBSW's own
/// internal sim client does, see `SimClientUdp::attempt_connection`), since a fire-and-forget
/// UDP send would otherwise silently do nothing if minisim is not running.
fn inject_mgm_failure(mode: SpiFaultMode) -> anyhow::Result<()> {
let sim_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), SIM_CTRL_PORT);
let sim_socket = UdpSocket::bind("127.0.0.1:0")?;
sim_socket.set_read_timeout(Some(Duration::from_millis(200)))?;
let mut reply_buf = [0u8; 4096];
let ping = SimRequest::new_with_epoch_time(SimCtrlRequest::Ping);
sim_socket.send_to(&serde_json::to_vec(&ping)?, sim_addr)?;
match sim_socket.recv(&mut reply_buf) {
Ok(len) => {
let reply: SimReply = serde_json::from_slice(&reply_buf[..len])?;
if reply.component() != SimComponent::SimCtrl {
bail!("unexpected reply while checking minisim connectivity: {reply:?}");
}
match SimCtrlReply::from_sim_message(&reply).expect("invalid SIM reply") {
SimCtrlReply::Pong => {}
SimCtrlReply::InvalidRequest(e) => {
bail!("minisim rejected connectivity ping: {e:?}")
}
}
}
Err(e)
if matches!(
e.kind(),
std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
) =>
{
bail!("minisim not reachable at {sim_addr} (ping timed out) - is it running?");
}
Err(e) => return Err(e.into()),
}
let request = SimRequest::new_with_epoch_time(MgmRequestLis3Mdl::SetSpiFault(mode));
sim_socket.send_to(&serde_json::to_vec(&request)?, sim_addr)?;
log::info!("injected SPI fault mode {mode:?} into minisim MGM0");
Ok(())
}
fn handle_raw_tm_packet(data: &[u8]) -> anyhow::Result<()> {
match spacepackets::CcsdsPacketReader::new_with_checksum(data) {
Ok(packet) => {
satrs-example/minisim/src/acs.rs
+48 -6
View File
@@ -7,7 +7,7 @@ use nexosim::{
use satrs_minisim::{
acs::{
lis3mdl::MgmLis3MdlReply, MgmReplyCommon, MgmReplyProvider, MgmSensorValuesMicroTesla,
MgtDipole, MgtHkSet, MgtReply, MGT_GEN_MAGNETIC_FIELD,
MgtDipole, MgtHkSet, MgtReply, SpiFaultMode, MGT_GEN_MAGNETIC_FIELD,
},
SimReply,
};
@@ -34,6 +34,7 @@ pub struct MagnetometerModel<ReplyProvider: MgmReplyProvider> {
#[allow(dead_code)]
pub periodicity: Duration,
pub external_mag_field: Option<MgmSensorValuesMicroTesla>,
pub spi_fault: SpiFaultMode,
pub reply_sender: mpsc::Sender<SimReply>,
pub phatom: std::marker::PhantomData<ReplyProvider>,
}
@@ -44,6 +45,7 @@ impl MagnetometerModel<MgmLis3MdlReply> {
switch_state: SwitchStateBinary::Off,
periodicity,
external_mag_field: None,
spi_fault: SpiFaultMode::None,
reply_sender,
phatom: std::marker::PhantomData,
}
@@ -55,12 +57,21 @@ impl<ReplyProvider: MgmReplyProvider> MagnetometerModel<ReplyProvider> {
self.switch_state = switch_state;
}
/// Force (or clear) a stuck-bus SPI fault, for FDIR testing purposes.
pub async fn set_spi_fault(&mut self, fault_mode: SpiFaultMode) {
self.spi_fault = fault_mode;
}
pub async fn send_sensor_values(&mut self, _: (), scheduler: &mut Context<Self>) {
self.reply_sender
.send(ReplyProvider::create_mgm_reply(MgmReplyCommon {
switch_state: self.switch_state,
sensor_values: self.calculate_current_mgm_tuple(current_millis(scheduler.time())),
}))
.send(ReplyProvider::create_mgm_reply(
MgmReplyCommon {
switch_state: self.switch_state,
sensor_values: self
.calculate_current_mgm_tuple(current_millis(scheduler.time())),
},
self.spi_fault,
))
.expect("sending MGM sensor values failed");
}
@@ -182,7 +193,7 @@ pub mod tests {
use satrs_minisim::{
acs::{
lis3mdl::{self, MgmLis3MdlReply},
MgmRequestLis3Mdl, MgtDipole, MgtHkSet, MgtReply, MgtRequest,
MgmRequestLis3Mdl, MgtDipole, MgtHkSet, MgtReply, MgtRequest, SpiFaultMode,
},
SerializableSimMsgPayload, SimComponent, SimMessageProvider, SimRequest,
};
@@ -211,6 +222,37 @@ pub mod tests {
assert_eq!(reply.common.sensor_values.z, 0.0);
}
#[test]
fn test_mgm_spi_fault_injection_all_ones() {
let mut sim_testbench = SimTestbench::new();
switch_device_on(&mut sim_testbench, SwitchId::Mgm0);
let fault_request =
SimRequest::new_with_epoch_time(MgmRequestLis3Mdl::SetSpiFault(SpiFaultMode::AllOnes));
sim_testbench
.send_request(fault_request)
.expect("sending MGM fault injection request failed");
sim_testbench.handle_sim_requests_time_agnostic();
sim_testbench.step().unwrap();
let data_request = SimRequest::new_with_epoch_time(MgmRequestLis3Mdl::RequestSensorData);
sim_testbench
.send_request(data_request)
.expect("sending MGM request failed");
sim_testbench.handle_sim_requests_time_agnostic();
sim_testbench.step().unwrap();
let sim_reply = sim_testbench
.try_receive_next_reply()
.expect("no MGM reply received");
let reply = MgmLis3MdlReply::from_sim_message(&sim_reply)
.expect("failed to deserialize MGM sensor values");
// Even though the device is switched on, the injected fault forces a stuck-bus reply.
assert_eq!(reply.common.switch_state, SwitchStateBinary::On);
assert_eq!(reply.raw.x, -1);
assert_eq!(reply.raw.y, -1);
assert_eq!(reply.raw.z, -1);
}
#[test]
fn test_basic_mgm_request_switched_on() {
let mut sim_testbench = SimTestbench::new();
satrs-example/minisim/src/controller.rs
+13
View File
@@ -124,6 +124,7 @@ impl SimController {
}
match sim_ctrl_request {
SimCtrlRequest::Ping => {
log::info!("received ping request, a client is connecting");
self.reply_sender
.send(SimReply::new(&SimCtrlReply::Pong))
.expect("sending reply from sim controller failed");
@@ -153,6 +154,18 @@ impl SimController {
.process_event(MagnetometerModel::send_sensor_values, (), addr)
.expect("event execution error for mgm");
}
MgmRequestLis3Mdl::SetSpiFault(fault_mode) => {
let addr = match mgm_idx {
0 => &self.addr_wrapper.mgm_0_addr,
1 => &self.addr_wrapper.mgm_1_addr,
_ => panic!("invalid mgm index"),
};
log::info!("MGM{mgm_idx}: setting SPI fault mode to {fault_mode:?}");
self.simulation
.process_event(MagnetometerModel::set_spi_fault, fault_mode, addr)
.expect("event execution error for mgm");
}
}
Ok(())
}
satrs-example/minisim/src/lib.rs
+43 -4
View File
@@ -202,12 +202,27 @@ pub mod acs {
use super::*;
pub trait MgmReplyProvider: Send + 'static {
fn create_mgm_reply(common: MgmReplyCommon) -> SimReply;
fn create_mgm_reply(common: MgmReplyCommon, fault_mode: SpiFaultMode) -> SimReply;
}
/// Fault mode injected on the simulated SPI bus, independent of the switch state.
///
/// Models the classic symptom of a stuck SPI bus: an undriven MISO line commonly reads
/// back as all-1s, a shorted/grounded one as all-0s.
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum SpiFaultMode {
#[default]
None,
AllZeros,
AllOnes,
}
#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
pub enum MgmRequestLis3Mdl {
RequestSensorData,
/// Force the raw register reply into a stuck-bus pattern, regardless of switch state.
/// Used to test FDIR handling of SPI bus faults.
SetSpiFault(SpiFaultMode),
}
impl SerializableSimMsgPayload<SimRequest> for MgmRequestLis3Mdl {
@@ -236,6 +251,7 @@ pub mod acs {
z: 30.0,
};
pub const ALL_ONES_SENSOR_VAL: i16 = 0xffff_u16 as i16;
pub const ALL_ZEROS_SENSOR_VAL: i16 = 0;
pub mod lis3mdl {
use super::*;
@@ -263,7 +279,30 @@ pub mod acs {
}
impl MgmLis3MdlReply {
pub fn new(common: MgmReplyCommon) -> Self {
pub fn new(common: MgmReplyCommon, fault_mode: SpiFaultMode) -> Self {
match fault_mode {
SpiFaultMode::AllZeros => {
return Self {
common,
raw: MgmLis3RawValues {
x: ALL_ZEROS_SENSOR_VAL,
y: ALL_ZEROS_SENSOR_VAL,
z: ALL_ZEROS_SENSOR_VAL,
},
};
}
SpiFaultMode::AllOnes => {
return Self {
common,
raw: MgmLis3RawValues {
x: ALL_ONES_SENSOR_VAL,
y: ALL_ONES_SENSOR_VAL,
z: ALL_ONES_SENSOR_VAL,
},
};
}
SpiFaultMode::None => (),
}
match common.switch_state {
SwitchStateBinary::Off => Self {
common,
@@ -306,8 +345,8 @@ pub mod acs {
}
impl MgmReplyProvider for MgmLis3MdlReply {
fn create_mgm_reply(common: MgmReplyCommon) -> SimReply {
SimReply::new(&Self::new(common))
fn create_mgm_reply(common: MgmReplyCommon, fault_mode: SpiFaultMode) -> SimReply {
SimReply::new(&Self::new(common, fault_mode))
}
}
}
satrs-example/src/acs/mgm.rs
+262 -26
View File
@@ -1,3 +1,5 @@
use satrs::fdir::FaultCounterStd;
use satrs::health::{HealthState, HealthTableMapSync, HealthTableProvider};
use satrs::spacepackets::CcsdsPacketIdAndPsc;
use satrs_example::{HkHelperSingleSet, ModeHelper, TimestampHelper, TmtcQueues};
use satrs_minisim::acs::MgmRequestLis3Mdl;
@@ -14,8 +16,6 @@ use types::acs::mgm::response::ModeResponse;
use types::pcdu::SwitchId;
use types::{ComponentId, DeviceMode, HkRequestType, acs::mgm};
use satrs::request::MessageMetadata;
use crate::ccsds::pack_ccsds_tm_packet_for_now;
use crate::eps::PowerSwitchHelper;
@@ -26,6 +26,15 @@ pub const X_LOWBYTE_IDX: usize = 9;
pub const Y_LOWBYTE_IDX: usize = 11;
pub const Z_LOWBYTE_IDX: usize = 13;
// FDIR configuration for a stuck SPI bus (data pinned to all-1s). Chosen so a handful of
// transient errors are tolerated but a persistently faulty bus is caught quickly.
//
// SPI itself cannot time out: the master clocks bytes in lockstep, so a transfer always
// completes. An unresponsive or dead device does not withhold a reply, it just leaves the bus
// floating, which is read back as this same all-1s pattern.
pub const SPI_FAULT_THRESHOLD: u32 = 2;
pub const SPI_FAULT_DECREMENT_AFTER: Duration = Duration::from_secs(30);
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum MgmId {
_0,
@@ -159,9 +168,12 @@ pub struct MgmHandlerLis3Mdl {
hk_helper: HkHelperSingleSet,
mode_helpers: ModeHelper<DeviceMode, TransitionState>,
mode_leaf_helper: ModeLeafHelper,
spi_fault_counter: FaultCounterStd,
health_table: HealthTableMapSync,
}
impl MgmHandlerLis3Mdl {
#[allow(clippy::too_many_arguments)]
pub fn new(
id: MgmId,
tmtc_queues: TmtcQueues,
@@ -170,6 +182,7 @@ impl MgmHandlerLis3Mdl {
shared_mgm_set: Arc<Mutex<SensorData>>,
mode_leaf_helper: ModeLeafHelper,
mode_timeout: Duration,
health_table: HealthTableMapSync,
) -> Self {
Self {
id,
@@ -182,6 +195,8 @@ impl MgmHandlerLis3Mdl {
stamp_helper: TimestampHelper::default(),
hk_helper: HkHelperSingleSet::new(false, Duration::from_millis(200)),
mode_leaf_helper,
spi_fault_counter: FaultCounterStd::new(SPI_FAULT_THRESHOLD, SPI_FAULT_DECREMENT_AFTER),
health_table,
}
}
@@ -254,6 +269,25 @@ impl MgmHandlerLis3Mdl {
)),
),
},
mgm::request::Request::Health(health_request) => {
match health_request {
mgm::request::HealthRequest::SetHealth(health_state) => {
log::info!(
"{}: setting health to {:?} via ground command",
self.id.str(),
health_state
);
self.health_table.set_health(
self.id.component_id().into(),
health_state,
);
self.send_telemetry(
Some(tc_id),
mgm::response::Response::Ok,
);
}
}
}
}
}
Err(e) => {
@@ -358,6 +392,16 @@ impl MgmHandlerLis3Mdl {
.try_into()
.unwrap(),
);
// A stuck-high SPI bus (undriven MISO) reads back as all-1s on every register,
// regardless of what was actually requested. This is the pattern this codebase already
// uses for "no real device behind the bus" (see the switched-off MGM sim reply). An
// all-0s reading is not used here, since it collides with a legitimate zero-field
// reading and would cause false positives.
if x_raw == -1 && y_raw == -1 && z_raw == -1 {
self.register_spi_fault();
return;
}
self.spi_fault_counter.try_decrement();
// Simple scaling to retrieve the float value, assuming the best sensor resolution.
let mut mgm_guard = self.shared_mgm_set.lock().unwrap();
mgm_guard.x = x_raw as f32 * GAUSS_TO_MICROTESLA_FACTOR as f32 * FIELD_LSB_PER_GAUSS_4_SENS;
@@ -367,6 +411,48 @@ impl MgmHandlerLis3Mdl {
drop(mgm_guard);
}
/// Registers one stuck-bus SPI fault with the FDIR fault counter, invalidating the current
/// sensor set. If the failure threshold is exceeded, the component is marked faulty in the
/// global health table.
fn register_spi_fault(&mut self) {
log::warn!("{}: stuck-bus SPI fault", self.id.str());
self.shared_mgm_set.lock().unwrap().valid = false;
if !self.spi_fault_counter.increment_and_check() {
return;
}
// Ground may have taken manual control, or already given up on this component.
// Autonomous FDIR should not override that decision.
let component_id = self.id.component_id().into();
match self.health_table.health(component_id) {
Some(HealthState::ExternalControl) | Some(HealthState::PermanentFaulty) => {
log::info!(
"{}: SPI fault threshold exceeded, but health is externally controlled, \
not overriding",
self.id.str()
);
}
_ => {
log::error!(
"{}: SPI fault threshold exceeded, marking component faulty",
self.id.str()
);
self.health_table
.set_health(component_id, HealthState::Faulty);
// TODO: Event? Health-table changes are currently invisible to the ground
// except through this log line. Likely applies to other health/mode
// transitions across the example app too, not just this one.
// Do not restart an already pending Off transition: poll_sensor still calls
// this every cycle the fault persists, and current stays Normal until the
// transition completes, so re-triggering here would keep resetting the
// transition state machine before it can ever finish.
if self.mode_helpers.target != Some(DeviceMode::Off) {
log::warn!("{}: commanding device off due to fault", self.id.str());
self.start_transition(DeviceMode::Off, true);
}
}
}
}
fn start_transition(&mut self, target_mode: DeviceMode, _forced: bool) {
log::info!("{}: transitioning to mode {:?}", self.id.str(), target_mode);
if target_mode == DeviceMode::Off {
@@ -380,29 +466,33 @@ impl MgmHandlerLis3Mdl {
return;
}
let target_mode = self.mode_helpers.target.unwrap();
if target_mode == DeviceMode::On || target_mode == DeviceMode::Normal {
if self.mode_helpers.transition_state == TransitionState::Idle {
let result = self
.switch_helper
.send_switch_on_cmd(MessageMetadata::new(0, self.id as u32), self.switch_id());
if result.is_err() {
// Could not send switch command.. still continue with transition.
log::error!("failed to send switch on command");
}
self.mode_helpers.transition_state = TransitionState::PowerSwitching;
let switch_target_on = target_mode != DeviceMode::Off;
if self.mode_helpers.transition_state == TransitionState::Idle {
let result = if switch_target_on {
self.switch_helper.send_switch_on_cmd(self.switch_id())
} else {
self.switch_helper.send_switch_off_cmd(self.switch_id())
};
if result.is_err() {
// Could not send switch command.. still continue with transition.
log::error!(
"failed to send switch {} command",
if switch_target_on { "on" } else { "off" }
);
}
if self.mode_helpers.transition_state == TransitionState::PowerSwitching {
if self.switch_helper.is_switch_on(self.switch_id()) {
log::info!("switch is on");
self.mode_helpers.transition_state = TransitionState::Done;
} else if self.mode_helpers.timed_out() {
self.handle_mode_transition_failure();
}
}
if self.mode_helpers.transition_state == TransitionState::Done {
self.handle_mode_reached();
self.mode_helpers.transition_state = TransitionState::PowerSwitching;
}
if self.mode_helpers.transition_state == TransitionState::PowerSwitching {
if self.switch_helper.is_switch_on(self.switch_id()) == switch_target_on {
log::info!("switch is {}", if switch_target_on { "on" } else { "off" });
self.mode_helpers.transition_state = TransitionState::Done;
} else if self.mode_helpers.timed_out() {
self.handle_mode_transition_failure();
}
}
if self.mode_helpers.transition_state == TransitionState::Done {
self.handle_mode_reached();
}
}
// Should be called to complete a mode transition which failed.
@@ -460,7 +550,7 @@ mod tests {
};
use arbitrary_int::u11;
use satrs::{request::GenericMessage, spacepackets::SpacePacketHeader};
use satrs::spacepackets::SpacePacketHeader;
use satrs_minisim::acs::lis3mdl::MgmLis3RawValues;
use types::{
Apid, ComponentId, TcHeader,
@@ -506,7 +596,8 @@ mod tests {
pub shared_switch_set: SharedSwitchSet,
pub tc_tx: mpsc::SyncSender<CcsdsTcPacketOwned>,
pub tm_rx: mpsc::Receiver<CcsdsTmPacketOwned>,
pub switch_rx: mpsc::Receiver<GenericMessage<SwitchRequest>>,
pub switch_rx: mpsc::Receiver<SwitchRequest>,
pub health_table: HealthTableMapSync,
pub handler: MgmHandlerLis3Mdl,
}
@@ -526,6 +617,7 @@ mod tests {
switch_map.insert(SwitchId::Mgm0, SwitchState::Off);
let switch_map = SwitchSet::new(switch_map);
let shared_switch_set = SharedSwitchSet::new(Mutex::new(switch_map));
let health_table = HealthTableMapSync::default();
let handler = MgmHandlerLis3Mdl::new(
MgmId::_0,
TmtcQueues { tc_rx, tm_tx },
@@ -534,18 +626,37 @@ mod tests {
shared_mgm_set,
mode_leaf_helper,
Duration::from_millis(100),
health_table.clone(),
);
Self {
assembly_mode_request_tx,
mode_report_rx,
shared_switch_set,
switch_rx,
health_table,
handler,
tm_rx,
tc_tx,
}
}
/// Switches the MGM to `Normal` mode, completing the power-switch handshake.
pub fn switch_to_normal(&mut self) {
self.tc_tx
.send(create_request_tc(
MgmSelect::_0,
mgm::request::Request::Mode(ModeRequest::SetMode(DeviceMode::Normal)),
))
.unwrap();
self.handler.periodic_operation();
self.shared_switch_set
.lock()
.unwrap()
.set_switch_state(SwitchId::Mgm0, SwitchState::On);
self.handler.periodic_operation();
assert_eq!(self.handler.mode(), DeviceMode::Normal);
}
pub fn test_spi_interface(&mut self) -> &mut TestSpiInterface {
match &mut self.handler.spi_com {
SpiCommunication::Dummy(_) | SpiCommunication::Sim(_) => {
@@ -582,8 +693,8 @@ mod tests {
// Verify power switch handling.
let switch_req = testbench.switch_rx.try_recv().expect("no switch request");
assert_eq!(switch_req.message.switch_id, SwitchId::Mgm0);
assert_eq!(switch_req.message.target_state, SwitchStateBinary::On);
assert_eq!(switch_req.switch_id, SwitchId::Mgm0);
assert_eq!(switch_req.target_state, SwitchStateBinary::On);
// This simulates one cycle for the power switch to update.
testbench
@@ -767,4 +878,129 @@ mod tests {
matches!(testbench.tm_rx.try_recv(), Err(TryRecvError::Empty));
}
#[test]
fn test_spi_fault_below_threshold_stays_healthy() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues {
x: -1,
y: -1,
z: -1,
};
// One stuck-bus reading should not be enough to trip SPI_FAULT_THRESHOLD.
testbench.handler.periodic_operation();
assert_eq!(
testbench.health_table.health(ComponentId::AcsMgm0.into()),
None,
"component should not be marked faulty yet"
);
assert!(!testbench.handler.shared_mgm_set.lock().unwrap().valid);
}
#[test]
fn test_spi_fault_above_threshold_marks_component_faulty() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues {
x: -1,
y: -1,
z: -1,
};
// SPI_FAULT_THRESHOLD is exceeded on the (threshold + 1)-th stuck-bus reading.
for _ in 0..SPI_FAULT_THRESHOLD + 1 {
testbench.handler.periodic_operation();
}
assert_eq!(
testbench.health_table.health(ComponentId::AcsMgm0.into()),
Some(HealthState::Faulty)
);
assert!(!testbench.handler.shared_mgm_set.lock().unwrap().valid);
}
#[test]
fn test_spi_fault_above_threshold_commands_device_off() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
// Drain the switch-on request left over from switch_to_normal().
testbench
.switch_rx
.try_recv()
.expect("no switch-on request sent");
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues {
x: -1,
y: -1,
z: -1,
};
for _ in 0..SPI_FAULT_THRESHOLD + 1 {
testbench.handler.periodic_operation();
}
assert_eq!(
testbench.health_table.health(ComponentId::AcsMgm0.into()),
Some(HealthState::Faulty)
);
// The Off transition was only started on the last iteration above, so it has not sent
// its switch-off request yet: drive one more cycle to let it do so.
testbench.handler.periodic_operation();
let switch_req = testbench
.switch_rx
.try_recv()
.expect("no switch-off request sent after fault");
assert_eq!(switch_req.switch_id, SwitchId::Mgm0);
assert_eq!(switch_req.target_state, SwitchStateBinary::Off);
// Simulate the PCDU acting on the switch-off request.
testbench
.shared_switch_set
.lock()
.unwrap()
.set_switch_state(SwitchId::Mgm0, SwitchState::Off);
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
}
#[test]
fn test_spi_fault_does_not_override_external_control() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench
.health_table
.set_health(ComponentId::AcsMgm0.into(), HealthState::ExternalControl);
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues {
x: -1,
y: -1,
z: -1,
};
for _ in 0..SPI_FAULT_THRESHOLD + 1 {
testbench.handler.periodic_operation();
}
// Ground took manual control; autonomous FDIR must not override that decision.
assert_eq!(
testbench.health_table.health(ComponentId::AcsMgm0.into()),
Some(HealthState::ExternalControl)
);
}
#[test]
fn test_recovering_from_spi_fault_clears_invalid_data_flag() {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues {
x: -1,
y: -1,
z: -1,
};
testbench.handler.periodic_operation();
assert!(!testbench.handler.shared_mgm_set.lock().unwrap().valid);
// Bus recovers before the threshold is exceeded.
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues::default();
testbench.handler.periodic_operation();
assert_eq!(
testbench.health_table.health(ComponentId::AcsMgm0.into()),
None
);
assert!(testbench.handler.shared_mgm_set.lock().unwrap().valid);
}
}
satrs-example/src/eps/mod.rs
+8 -24
View File
@@ -2,10 +2,7 @@ use derive_new::new;
use std::{cell::RefCell, collections::VecDeque, sync::mpsc, time::Duration};
use types::pcdu::{SwitchId, SwitchRequest, SwitchState, SwitchStateBinary};
use satrs::{
queue::GenericSendError,
request::{GenericMessage, MessageMetadata},
};
use satrs::{queue::GenericSendError, request::MessageMetadata};
use thiserror::Error;
use crate::eps::pcdu::SwitchMapWrapper;
@@ -16,7 +13,7 @@ pub mod pcdu;
#[derive(new, Clone)]
pub struct PowerSwitchHelper {
switcher_tx: mpsc::SyncSender<GenericMessage<SwitchRequest>>,
switcher_tx: mpsc::SyncSender<SwitchRequest>,
shared_switch_set: SharedSwitchSet,
}
@@ -37,28 +34,15 @@ pub enum SwitchInfoError {
}
impl PowerSwitchHelper {
pub fn send_switch_on_cmd(
&self,
requestor_info: satrs::request::MessageMetadata,
switch_id: SwitchId,
) -> Result<(), GenericSendError> {
self.switcher_tx.send(GenericMessage::new(
requestor_info,
SwitchRequest::new(switch_id, SwitchStateBinary::On),
))?;
pub fn send_switch_on_cmd(&self, switch_id: SwitchId) -> Result<(), GenericSendError> {
self.switcher_tx
.send(SwitchRequest::new(switch_id, SwitchStateBinary::On))?;
Ok(())
}
#[allow(dead_code)]
pub fn send_switch_off_cmd(
&self,
requestor_info: satrs::request::MessageMetadata,
switch_id: SwitchId,
) -> Result<(), GenericSendError> {
self.switcher_tx.send(GenericMessage::new(
requestor_info,
SwitchRequest::new(switch_id, SwitchStateBinary::Off),
))?;
pub fn send_switch_off_cmd(&self, switch_id: SwitchId) -> Result<(), GenericSendError> {
self.switcher_tx
.send(SwitchRequest::new(switch_id, SwitchStateBinary::Off))?;
Ok(())
}
satrs-example/src/eps/pcdu.rs
+7 -16
View File
@@ -6,7 +6,7 @@ use std::{
use derive_new::new;
use num_enum::{IntoPrimitive, TryFromPrimitive};
use satrs::{request::GenericMessage, spacepackets::CcsdsPacketIdAndPsc};
use satrs::spacepackets::CcsdsPacketIdAndPsc;
use satrs_example::TimestampHelper;
use satrs_minisim::{
SerializableSimMsgPayload, SimReply, SimRequest,
@@ -264,7 +264,7 @@ pub enum OpCode {
#[allow(clippy::too_many_arguments)]
pub struct PcduHandler<ComInterface: SerialInterface> {
dev_str: &'static str,
switch_request_rx: mpsc::Receiver<GenericMessage<SwitchRequest>>,
switch_request_rx: mpsc::Receiver<SwitchRequest>,
tc_rx: std::sync::mpsc::Receiver<CcsdsTcPacketOwned>,
tm_tx: mpsc::SyncSender<CcsdsTmPacketOwned>,
pub com_interface: ComInterface,
@@ -277,7 +277,7 @@ impl<ComInterface: SerialInterface> PcduHandler<ComInterface> {
pub fn new(
tc_rx: std::sync::mpsc::Receiver<CcsdsTcPacketOwned>,
tm_tx: std::sync::mpsc::SyncSender<CcsdsTmPacketOwned>,
switch_request_rx: mpsc::Receiver<GenericMessage<SwitchRequest>>,
switch_request_rx: mpsc::Receiver<SwitchRequest>,
com_interface: ComInterface,
shared_switch_map: Arc<Mutex<SwitchSet>>,
init_mode: DeviceMode,
@@ -488,10 +488,7 @@ impl<ComInterface: SerialInterface> PcduHandler<ComInterface> {
loop {
match self.switch_request_rx.try_recv() {
Ok(switch_req) => {
self.handle_device_switching(
switch_req.message.switch_id(),
switch_req.message.target_state(),
);
self.handle_device_switching(switch_req.switch_id(), switch_req.target_state());
}
Err(e) => match e {
mpsc::TryRecvError::Empty => break,
@@ -531,10 +528,7 @@ mod tests {
use std::sync::mpsc;
use arbitrary_int::u11;
use satrs::{
request::{GenericMessage, MessageMetadata},
spacepackets::SpacePacketHeader,
};
use satrs::spacepackets::SpacePacketHeader;
use types::{
Apid, TcHeader,
pcdu::{SwitchMapBinary, SwitchStateBinary},
@@ -593,7 +587,7 @@ mod tests {
pub mode_reply_rx_to_parent: mpsc::Receiver<types::pcdu::response::Response>,
pub tc_tx: mpsc::SyncSender<CcsdsTcPacketOwned>,
pub tm_rx: mpsc::Receiver<CcsdsTmPacketOwned>,
pub switch_request_tx: mpsc::Sender<GenericMessage<SwitchRequest>>,
pub switch_request_tx: mpsc::Sender<SwitchRequest>,
pub handler: PcduHandler<SerialInterfaceTest>,
}
@@ -730,10 +724,7 @@ mod tests {
.unwrap();
testbench
.switch_request_tx
.send(GenericMessage::new(
MessageMetadata::new(0, ComponentId::AcsMgm0 as u32),
SwitchRequest::new(SwitchId::Mgm0, SwitchStateBinary::On),
))
.send(SwitchRequest::new(SwitchId::Mgm0, SwitchStateBinary::On))
.expect("failed to send switch request");
testbench.handler.periodic_operation(OpCode::RegularOp);
testbench
satrs-example/src/main.rs
+5
View File
@@ -156,6 +156,9 @@ fn main() {
let (switch_request_tx, switch_request_rx) = mpsc::sync_channel(20);
let switch_helper = PowerSwitchHelper::new(switch_request_tx, shared_switch_set.clone());
// Global FDIR health table, shared by all software objects.
let health_table = satrs::health::HealthTableMapSync::default();
let shared_mgm_0_set = Arc::default();
let shared_mgm_1_set = Arc::default();
let (mgm_0_spi_interface, mgm_1_spi_interface) =
@@ -194,6 +197,7 @@ fn main() {
report_tx: mgm_0_mode_report_tx,
},
Duration::from_millis(1000),
health_table.clone(),
);
let mut mgm_1_handler = mgm::MgmHandlerLis3Mdl::new(
mgm::MgmId::_1,
@@ -209,6 +213,7 @@ fn main() {
report_tx: mgm_1_mode_report_tx,
},
Duration::from_millis(1000),
health_table.clone(),
);
let mut mgm_assembly = mgm_assembly::Assembly::new(
mgm_assembly::ParentQueueHelper {
satrs-example/types/Cargo.toml
+1 -1
View File
@@ -6,7 +6,7 @@ edition = "2024"
[dependencies]
serde = { version = "1", features = ["derive"] }
spacepackets = { version = "0.18", default-features = false }
satrs = { path = "../../satrs" }
satrs = { path = "../../satrs", features = ["serde"] }
num_enum = { version = "0.7" }
strum = { version = "0.28", features = ["derive"] }
postcard = { version = "1" }
satrs-example/types/src/acs/mgm.rs
+9
View File
@@ -7,6 +7,13 @@ pub mod request {
ReadMode,
}
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
pub enum HealthRequest {
/// Overrides the device's autonomous FDIR health state, for example to clear a `Faulty`
/// state set by the handler after ground has fixed or worked around the underlying issue.
SetHealth(satrs::health::HealthState),
}
#[derive(Debug, PartialEq, Eq, Clone, Copy, serde::Serialize, serde::Deserialize)]
pub enum HkId {
Sensor,
@@ -23,6 +30,7 @@ pub mod request {
Ping,
Hk(HkRequest),
Mode(ModeRequest),
Health(HealthRequest),
}
impl Request {
@@ -31,6 +39,7 @@ pub mod request {
Request::Ping => crate::MessageType::Verification,
Request::Hk(_hk_request) => crate::MessageType::Hk,
Request::Mode(_mode) => crate::MessageType::Mode,
Request::Health(_health) => crate::MessageType::Health,
}
}
}
satrs-example/types/src/lib.rs
+1
View File
@@ -150,6 +150,7 @@ pub enum MessageType {
Action,
Event,
Verification,
Health,
}
pub trait Message {
satrs/Cargo.toml
+1 -1
View File
@@ -72,7 +72,7 @@ alloc = [
]
serde = ["dep:serde", "spacepackets/serde", "satrs-shared/serde"]
crossbeam = ["crossbeam-channel"]
defmt = ["dep:defmt", "spacepackets/defmt"]
defmt = ["dep:defmt", "spacepackets/defmt", "embassy-time?/defmt"]
embassy-time = ["dep:embassy-time"]
test_util = []
satrs/src/fdir.rs
+292
View File
@@ -0,0 +1,292 @@
//! # FDIR (Fault Detection, Isolation and Recovery) helpers
//!
//! A fault counter tracks a monotonic fault count, decrements it over time when faults stop
//! occurring, and reports when a configured failure threshold has been exceeded. This is the
//! typical building block used to turn a stream of transient error reports into a single
//! "component is faulty" decision without reacting to the first isolated error.
//!
//! The design follows the FSFW `FaultCounter`:
//! <https://egit.irs.uni-stuttgart.de/KSat/fsfw/src/branch/main/src/fsfw/fdir/FaultCounter.h>
//!
//! Pick a variant based on what clock is available:
//!
//! - [FaultCounterStd]: `std::time::Instant`, behind the `std` feature.
#![cfg_attr(
feature = "embassy-time",
doc = "- [FaultCounterEmbassy]: `embassy_time::Instant`, behind the `embassy-time` feature."
)]
#![deny(missing_docs)]
/// Fault counter backed by [std::time::Instant].
#[cfg(feature = "std")]
#[derive(Debug, Clone)]
pub struct FaultCounterStd {
fault_count: u32,
failure_threshold: u32,
decrement_after: core::time::Duration,
last_decrement: Option<std::time::Instant>,
}
#[cfg(feature = "std")]
impl FaultCounterStd {
/// Create a new [`FaultCounterStd`].
///
/// - `failure_threshold`: threshold above which [`Self::above_threshold`] returns `true` and
/// resets the internal count.
/// - `decrement_after`: minimum duration between automatic decrements performed by
/// [`Self::try_decrement`].
pub fn new(failure_threshold: u32, decrement_after: core::time::Duration) -> Self {
Self {
fault_count: 0,
failure_threshold,
decrement_after,
last_decrement: None,
}
}
/// Current fault count.
pub fn fault_count(&self) -> u32 {
self.fault_count
}
/// Increase the fault count by `1`.
///
/// If the counter was previously `0`, this starts a new decrement clock.
pub fn increment(&mut self) {
if self.fault_count == 0 {
self.last_decrement = Some(std::time::Instant::now());
}
self.fault_count += 1;
}
/// Increase the fault count by `n`.
pub fn increment_n(&mut self, n: u32) {
for _ in 0..n {
self.increment();
}
}
fn has_decrement_timedout(&self) -> bool {
match self.last_decrement {
Some(last_decrement) => last_decrement.elapsed() >= self.decrement_after,
None => false,
}
}
/// Decrease the fault count by `1` if the decrement timeout elapsed.
///
/// Returns `true` if a decrement was performed, `false` otherwise. A decrement is only
/// performed when the counter is non-zero and at least `decrement_after` has elapsed since
/// the last decrement.
pub fn try_decrement(&mut self) -> bool {
if self.fault_count == 0 || !self.has_decrement_timedout() {
return false;
}
self.last_decrement = Some(std::time::Instant::now());
self.fault_count -= 1;
true
}
/// Check whether the counter exceeded the failure threshold.
///
/// Returns `true` when `fault_count > failure_threshold`. In that case, the counter is reset
/// to `0`.
pub fn above_threshold(&mut self) -> bool {
if self.fault_count > self.failure_threshold {
self.fault_count = 0;
return true;
}
false
}
/// Convenience helper to increment once and immediately check the threshold.
pub fn increment_and_check(&mut self) -> bool {
self.increment();
self.above_threshold()
}
/// Clear the counter and decrement timing state.
pub fn clear(&mut self) {
self.fault_count = 0;
self.last_decrement = None;
}
/// Update the failure threshold used by [`Self::above_threshold`].
pub fn set_failure_threshold(&mut self, threshold: u32) {
self.failure_threshold = threshold;
}
/// Update the minimum interval between automatic decrements.
pub fn set_decrement_after(&mut self, duration: core::time::Duration) {
self.decrement_after = duration;
}
}
/// Fault counter backed by [embassy_time::Instant].
#[cfg(feature = "embassy-time")]
#[derive(Debug, Clone, Copy)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub struct FaultCounterEmbassy {
fault_count: u32,
failure_threshold: u32,
decrement_after: embassy_time::Duration,
last_decrement: Option<embassy_time::Instant>,
}
#[cfg(feature = "embassy-time")]
impl FaultCounterEmbassy {
/// Create a new [`FaultCounterEmbassy`].
///
/// - `failure_threshold`: threshold above which [`Self::above_threshold`] returns `true` and
/// resets the internal count.
/// - `decrement_after`: minimum duration between automatic decrements performed by
/// [`Self::try_decrement`].
pub fn new(failure_threshold: u32, decrement_after: embassy_time::Duration) -> Self {
Self {
fault_count: 0,
failure_threshold,
decrement_after,
last_decrement: None,
}
}
/// Current fault count.
pub fn fault_count(&self) -> u32 {
self.fault_count
}
/// Increase the fault count by `1`.
///
/// If the counter was previously `0`, this starts a new decrement clock.
pub fn increment(&mut self) {
if self.fault_count == 0 {
self.last_decrement = Some(embassy_time::Instant::now());
}
self.fault_count += 1;
}
/// Increase the fault count by `n`.
pub fn increment_n(&mut self, n: u32) {
for _ in 0..n {
self.increment();
}
}
fn has_decrement_timedout(&self) -> bool {
match self.last_decrement {
Some(last_decrement) => {
embassy_time::Instant::now().duration_since(last_decrement) >= self.decrement_after
}
None => false,
}
}
/// Decrease the fault count by `1` if the decrement timeout elapsed.
///
/// Returns `true` if a decrement was performed, `false` otherwise. A decrement is only
/// performed when the counter is non-zero and at least `decrement_after` has elapsed since
/// the last decrement.
pub fn try_decrement(&mut self) -> bool {
if self.fault_count == 0 || !self.has_decrement_timedout() {
return false;
}
self.last_decrement = Some(embassy_time::Instant::now());
self.fault_count -= 1;
true
}
/// Check whether the counter exceeded the failure threshold.
///
/// Returns `true` when `fault_count > failure_threshold`. In that case, the counter is reset
/// to `0`.
pub fn above_threshold(&mut self) -> bool {
if self.fault_count > self.failure_threshold {
self.fault_count = 0;
return true;
}
false
}
/// Convenience helper to increment once and immediately check the threshold.
pub fn increment_and_check(&mut self) -> bool {
self.increment();
self.above_threshold()
}
/// Clear the counter and decrement timing state.
pub fn clear(&mut self) {
self.fault_count = 0;
self.last_decrement = None;
}
/// Update the failure threshold used by [`Self::above_threshold`].
pub fn set_failure_threshold(&mut self, threshold: u32) {
self.failure_threshold = threshold;
}
/// Update the minimum interval between automatic decrements.
pub fn set_decrement_after(&mut self, duration: embassy_time::Duration) {
self.decrement_after = duration;
}
}
#[cfg(all(test, feature = "std"))]
mod tests {
use super::*;
use std::thread;
use std::time::Duration;
#[test]
fn threshold_not_exceeded_below_limit() {
let mut fc = FaultCounterStd::new(2, Duration::from_secs(60));
assert!(!fc.increment_and_check());
assert!(!fc.increment_and_check());
assert_eq!(fc.fault_count(), 2);
}
#[test]
fn threshold_exceeded_resets_counter() {
let mut fc = FaultCounterStd::new(2, Duration::from_secs(60));
fc.increment_n(3);
assert!(fc.above_threshold());
assert_eq!(fc.fault_count(), 0);
assert!(!fc.above_threshold());
}
#[test]
fn decrement_only_after_timeout() {
let mut fc = FaultCounterStd::new(5, Duration::from_millis(20));
fc.increment();
assert!(!fc.try_decrement());
thread::sleep(Duration::from_millis(30));
assert!(fc.try_decrement());
assert_eq!(fc.fault_count(), 0);
}
#[test]
fn decrement_noop_when_empty() {
let mut fc = FaultCounterStd::new(5, Duration::from_millis(1));
thread::sleep(Duration::from_millis(2));
assert!(!fc.try_decrement());
}
#[test]
fn increment_after_empty_resets_decrement_timing() {
let mut fc = FaultCounterStd::new(5, Duration::from_millis(20));
fc.increment();
thread::sleep(Duration::from_millis(30));
assert!(fc.try_decrement());
// Counter is 0 again, incrementing should require a fresh decrement_after wait.
fc.increment();
assert!(!fc.try_decrement());
}
#[test]
fn clear_resets_state() {
let mut fc = FaultCounterStd::new(1, Duration::from_secs(60));
fc.increment_n(2);
fc.clear();
assert_eq!(fc.fault_count(), 0);
assert!(!fc.above_threshold());
}
}
satrs/src/health.rs
+10
View File
@@ -1,6 +1,7 @@
use crate::ComponentId;
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum HealthState {
Healthy = 1,
Faulty = 2,
@@ -27,6 +28,15 @@ impl HealthTableMapSync {
}
}
#[cfg(feature = "std")]
impl Default for HealthTableMapSync {
/// Creates an empty, shared health table. Absent entries are up to the consumer to
/// interpret, for example as [HealthState::Healthy] by default.
fn default() -> Self {
Self::new(hashbrown::HashMap::new())
}
}
#[cfg(feature = "std")]
impl HealthTableProvider for HealthTableMapSync {
fn health(&self, id: ComponentId) -> Option<HealthState> {
satrs/src/lib.rs
+1
View File
@@ -15,6 +15,7 @@ pub mod ccsds;
pub mod encoding;
#[cfg(feature = "std")]
pub mod executable;
pub mod fdir;
pub mod hal;
pub mod health;
/// Helpers to track when housekeeping sets need to be regenerated.