rust/sat-rs
Archived
12
1 Fork 0
Code Issues 6 Projects Releases 29 Wiki Activity

Merge pull request 'MGT fdir' (#292) from mgt-fdir into main

Browse Source 
Reviewed-on: #292
This commit was merged in pull request #292.
This commit is contained in:
muellerr committed 2026-09-29 11:50:39 +02:00
2 parents cb6c8b84bb 23ac7cf0ad
commit bb8decfc74
13 files changed
+969 -305
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

satrs-example/Cargo.toml
-1
View File
@@ -3,7 +3,6 @@ name = "satrs-example"
version = "0.1.1"
edition = "2024"
default-run = "satrs-example"
homepage = "https://egit.irs.uni-stuttgart.de/rust/sat-rs"
repository = "https://egit.irs.uni-stuttgart.de/rust/sat-rs"
[dependencies]
satrs-example/client/src/main.rs
+11 -1
View File
@@ -189,6 +189,9 @@ struct MgtArgs {
torque: Option<types::acs::mgt::Dipole>,
#[arg(long, default_value_t = 1000)]
torque_duration_ms: u64,
/// Override the device's FDIR health state, for example to clear a `Faulty` state.
#[arg(long, value_enum)]
health: Option<HealthStateSelect>,
}
#[derive(Debug, PartialEq, Eq, Clone, Copy, clap::Parser)]
@@ -295,6 +298,13 @@ fn handle_mgt_command(client: &UdpSocket, addr: SocketAddr, args: MgtArgs) -> an
};
send_mgt_request(client, addr, request);
}
if let Some(health) = args.health {
send_mgt_request(
client,
addr,
Request::Health(types::HealthRequest::SetHealth(health.into())),
);
}
Ok(())
}
@@ -374,7 +384,7 @@ fn handle_mgm_command(
SpacePacketHeader::new_from_apid(u11::new(Apid::Acs as u16)),
TcHeader::new(target_id, types::MessageType::Health),
types::acs::mgm::request::Request::Health(
types::acs::mgm::request::HealthRequest::SetHealth(health.into()),
types::HealthRequest::SetHealth(health.into()),
),
);
let sent_tc_id = CcsdsPacketIdAndPsc::new_from_ccsds_packet(&request.sp_header);
satrs-example/minisim/src/acs/mgt.rs
+62 -23
View File
@@ -10,6 +10,9 @@ use serde::{Deserialize, Serialize};
use std::time::Duration;
use types::pcdu::SwitchStateBinary;
/// Time the device needs to answer a command.
const REPLY_DELAY: Duration = Duration::from_millis(15);
/// Simple magnetorquer simulation model.
#[derive(Serialize, Deserialize)]
pub struct MgtModel {
@@ -39,6 +42,9 @@ impl MgtModel {
duration_and_dipole: (Duration, mgt::Dipole),
cx: &Context<Self>,
) {
if self.switch_state != SwitchStateBinary::On {
return;
}
self.torque_dipole = duration_and_dipole.1;
self.torquing = true;
if cx
@@ -48,6 +54,7 @@ impl MgtModel {
log::warn!("torque clearing can only be set for a future time.");
}
self.generate_magnetic_field(()).await;
self.schedule_reply(mgt::Reply::Ack, cx);
}
#[nexosim(schedulable)]
@@ -69,22 +76,22 @@ impl MgtModel {
if self.switch_state != SwitchStateBinary::On {
return;
}
cx.schedule_event(
Duration::from_millis(15),
schedulable!(Self::send_housekeeping_data),
(),
)
.expect("requesting housekeeping data failed")
// The HK is sampled when the command is processed, not when the reply is sent.
let hk = mgt::HkSet {
dipole: self.torque_dipole,
torquing: self.torquing,
};
self.schedule_reply(mgt::Reply::Hk(hk), cx);
}
fn schedule_reply(&self, reply: mgt::Reply, cx: &Context<Self>) {
cx.schedule_event(REPLY_DELAY, schedulable!(Self::send_reply), reply)
.expect("scheduling MGT reply failed")
}
#[nexosim(schedulable)]
async fn send_housekeeping_data(&mut self) {
self.reply
.send(SimReply::from(mgt::Reply::Hk(mgt::HkSet {
dipole: self.torque_dipole,
torquing: self.torquing,
})))
.await;
async fn send_reply(&mut self, reply: mgt::Reply) {
self.reply.send(SimReply::from(reply)).await;
}
fn calc_magnetic_field(&self, _: mgt::Dipole) -> mgm::SensorValuesMicroTesla {
@@ -118,10 +125,17 @@ mod tests {
use crate::{eps::tests::switch_device_on, test_helpers::SimTestbench};
fn decode_reply(sim_reply: SimReply) -> mgt::Reply {
let SimReply::Mgt(frame) = sim_reply else {
panic!("unexpected reply {sim_reply:?}");
};
mgt::Reply::from_frame(&frame).expect("invalid MGT reply frame")
}
fn request_hk(sim_testbench: &mut SimTestbench) -> Option<mgt::HkSet> {
let sim_reply = sim_testbench.request_reply(mgt::Request::RequestHk)?;
let SimReply::Mgt(mgt::Reply::Hk(hk)) = sim_reply else {
panic!("unexpected reply {sim_reply:?}");
let mgt::Reply::Hk(hk) = decode_reply(sim_reply) else {
panic!("unexpected MGT reply");
};
Some(hk)
}
@@ -162,7 +176,11 @@ mod tests {
.send_request(request)
.expect("sending MGT request failed");
sim_testbench.handle_sim_requests_time_agnostic();
sim_testbench.step_until(Duration::from_millis(5)).unwrap();
sim_testbench.step_until(Duration::from_millis(20)).unwrap();
let ack = sim_testbench
.try_receive_next_reply()
.expect("no torque command ack");
assert_eq!(decode_reply(ack), mgt::Reply::Ack);
assert_eq!(
request_hk(&mut sim_testbench),
@@ -183,6 +201,25 @@ mod tests {
);
}
#[test]
fn test_torque_command_not_acked_when_off() {
let mut sim_testbench = SimTestbench::new();
let reply = sim_testbench.request_reply(mgt::Request::ApplyTorque {
duration: Duration::from_millis(100),
dipole: mgt::Dipole { x: 1, y: 2, z: 3 },
});
assert!(reply.is_none());
}
#[test]
fn test_invalid_frame_is_dropped() {
let mut sim_testbench = SimTestbench::new();
switch_device_on(&mut sim_testbench, SwitchId::Mgt);
assert!(sim_testbench
.request_reply(SimRequest::Mgt(vec![0x01, 0x00]))
.is_none());
}
/// Processes the request without stepping, so scheduled events like the torque clearing do
/// not fire.
fn process_without_step(sim_testbench: &mut SimTestbench, request: impl Into<SimRequest>) {
@@ -200,13 +237,15 @@ mod tests {
request: mgm::Request::RequestSensorData,
},
);
let sim_reply = sim_testbench
.try_receive_next_reply()
.expect("no MGM reply received");
let SimReply::Mgm { reply, .. } = sim_reply else {
panic!("unexpected reply {sim_reply:?}");
};
reply.sensor_values
// Skips pending MGT replies, for example torque command acks.
loop {
let sim_reply = sim_testbench
.try_receive_next_reply()
.expect("no MGM reply received");
if let SimReply::Mgm { reply, .. } = sim_reply {
return reply.sensor_values;
}
}
}
fn start_torquing(sim_testbench: &mut SimTestbench, duration: Duration) {
satrs-example/minisim/src/controller.rs
+8 -1
View File
@@ -268,7 +268,14 @@ impl SimController {
}
}
fn handle_mgt_request(&mut self, mgt_request: mgt::Request) {
fn handle_mgt_request(&mut self, frame: Vec<u8>) {
let mgt_request = match mgt::Request::from_frame(&frame) {
Ok(request) => request,
Err(e) => {
log::warn!("dropping invalid MGT frame {frame:02x?}: {e}");
return;
}
};
if MGT_REQ_WIRETAPPING {
log::info!("received MGT request: {mgt_request:?}");
}
satrs-example/minisim/src/lib.rs
+197 -17
View File
@@ -19,8 +19,12 @@ pub enum ComponentId {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum SimRequest {
SimCtrl(SimCtrlRequest),
Mgm { id: mgm::Id, request: mgm::Request },
Mgt(mgt::Request),
Mgm {
id: mgm::Id,
request: mgm::Request,
},
/// Raw frame of the MGT serial protocol.
Mgt(Vec<u8>),
Pcdu(PcduRequest),
}
@@ -32,7 +36,7 @@ impl From<SimCtrlRequest> for SimRequest {
impl From<mgt::Request> for SimRequest {
fn from(request: mgt::Request) -> Self {
Self::Mgt(request)
Self::Mgt(request.to_frame())
}
}
@@ -64,8 +68,12 @@ impl SimRequestWithTime {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum SimReply {
SimCtrl(SimCtrlReply),
Mgm { id: mgm::Id, reply: mgm::Reply },
Mgt(mgt::Reply),
Mgm {
id: mgm::Id,
reply: mgm::Reply,
},
/// Raw frame of the MGT serial protocol.
Mgt(Vec<u8>),
Pcdu(PcduReply),
}
@@ -88,7 +96,7 @@ impl From<SimCtrlReply> for SimReply {
impl From<mgt::Reply> for SimReply {
fn from(reply: mgt::Reply) -> Self {
Self::Mgt(reply)
Self::Mgt(reply.to_frame())
}
}
@@ -263,34 +271,206 @@ pub mod acs {
}
}
/// Simple serial protocol of the magnetorquer.
///
/// The first byte of each frame is the packet ID. The high bit of the ID is set for replies.
/// All fields are big endian. Every command is answered with exactly one reply, but only
/// if the device is powered. The device drops invalid frames.
///
/// A data link layer is deliberately skipped for simplicity. A real serial link would need
/// framing and error detection, for example COBS encoding and a CRC. Here, the transport
/// always delivers complete and intact frames.
pub mod mgt {
use std::time::Duration;
use serde::{Deserialize, Serialize};
use num_enum::{IntoPrimitive, TryFromPrimitive};
pub use types::acs::mgt::Dipole;
// Simple model using i16 values.
#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Dipole {
pub x: i16,
pub y: i16,
pub z: i16,
#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
#[repr(u8)]
pub enum RequestId {
RequestHk = 0x01,
/// Payload: dipole (3 x i16), duration in milliseconds (u32).
ApplyTorque = 0x02,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
#[repr(u8)]
pub enum ReplyId {
/// Payload: dipole (3 x i16), torquing flag (u8).
Hk = 0x81,
/// Reply to [RequestId::ApplyTorque].
Ack = 0x82,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, thiserror::Error)]
pub enum FrameError {
#[error("empty frame")]
Empty,
#[error("unknown packet ID {0:#04x}")]
UnknownPacketId(u8),
#[error("invalid length {len} for packet ID {id:#04x}")]
InvalidLength { id: u8, len: usize },
}
fn split_packet_id(frame: &[u8]) -> Result<(u8, &[u8]), FrameError> {
let (&id, payload) = frame.split_first().ok_or(FrameError::Empty)?;
Ok((id, payload))
}
fn payload_array<const N: usize>(id: u8, payload: &[u8]) -> Result<&[u8; N], FrameError> {
payload.try_into().map_err(|_| FrameError::InvalidLength {
id,
len: payload.len() + 1,
})
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum Request {
ApplyTorque { duration: Duration, dipole: Dipole },
/// The duration has millisecond resolution on the wire.
ApplyTorque {
duration: Duration,
dipole: Dipole,
},
RequestHk,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
impl Request {
pub fn to_frame(&self) -> Vec<u8> {
match self {
Request::RequestHk => vec![RequestId::RequestHk.into()],
Request::ApplyTorque { duration, dipole } => {
let mut frame = vec![RequestId::ApplyTorque.into()];
frame.extend_from_slice(&dipole.to_be_bytes());
let duration_ms = u32::try_from(duration.as_millis()).unwrap_or(u32::MAX);
frame.extend_from_slice(&duration_ms.to_be_bytes());
frame
}
}
}
pub fn from_frame(frame: &[u8]) -> Result<Self, FrameError> {
let (id, payload) = split_packet_id(frame)?;
match RequestId::try_from(id).map_err(|_| FrameError::UnknownPacketId(id))? {
RequestId::RequestHk => {
payload_array::<0>(id, payload)?;
Ok(Request::RequestHk)
}
RequestId::ApplyTorque => {
let payload: &[u8; Dipole::LEN + 4] = payload_array(id, payload)?;
let [dipole @ .., d0, d1, d2, d3] = *payload;
Ok(Request::ApplyTorque {
duration: Duration::from_millis(
u32::from_be_bytes([d0, d1, d2, d3]).into(),
),
dipole: Dipole::from_be_bytes(&dipole),
})
}
}
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct HkSet {
pub dipole: Dipole,
pub torquing: bool,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum Reply {
Hk(HkSet),
Ack,
}
impl Reply {
pub fn to_frame(&self) -> Vec<u8> {
match self {
Reply::Hk(hk) => {
let mut frame = vec![ReplyId::Hk.into()];
frame.extend_from_slice(&hk.dipole.to_be_bytes());
frame.push(hk.torquing as u8);
frame
}
Reply::Ack => vec![ReplyId::Ack.into()],
}
}
pub fn from_frame(frame: &[u8]) -> Result<Self, FrameError> {
let (id, payload) = split_packet_id(frame)?;
match ReplyId::try_from(id).map_err(|_| FrameError::UnknownPacketId(id))? {
ReplyId::Hk => {
let payload: &[u8; Dipole::LEN + 1] = payload_array(id, payload)?;
let [dipole @ .., torquing] = *payload;
Ok(Reply::Hk(HkSet {
dipole: Dipole::from_be_bytes(&dipole),
torquing: torquing != 0,
}))
}
ReplyId::Ack => {
payload_array::<0>(id, payload)?;
Ok(Reply::Ack)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_apply_torque_frame() {
let request = Request::ApplyTorque {
duration: Duration::from_millis(0x0102_0304),
dipole: Dipole {
x: -2,
y: 0x0506,
z: 0x0708,
},
};
let frame = request.to_frame();
assert_eq!(
frame,
[0x02, 0xff, 0xfe, 0x05, 0x06, 0x07, 0x08, 0x01, 0x02, 0x03, 0x04]
);
assert_eq!(Request::from_frame(&frame), Ok(request));
}
#[test]
fn test_request_hk_frame() {
assert_eq!(Request::RequestHk.to_frame(), [0x01]);
assert_eq!(Request::from_frame(&[0x01]), Ok(Request::RequestHk));
}
#[test]
fn test_reply_frames() {
let hk = Reply::Hk(HkSet {
dipole: Dipole { x: 1, y: 2, z: 3 },
torquing: true,
});
let frame = hk.to_frame();
assert_eq!(frame, [0x81, 0, 1, 0, 2, 0, 3, 1]);
assert_eq!(Reply::from_frame(&frame), Ok(hk));
assert_eq!(Reply::Ack.to_frame(), [0x82]);
assert_eq!(Reply::from_frame(&[0x82]), Ok(Reply::Ack));
}
#[test]
fn test_invalid_frames() {
assert_eq!(Request::from_frame(&[]), Err(FrameError::Empty));
assert_eq!(
Request::from_frame(&[0x81]),
Err(FrameError::UnknownPacketId(0x81))
);
assert_eq!(
Request::from_frame(&[0x01, 0x00]),
Err(FrameError::InvalidLength { id: 0x01, len: 2 })
);
assert_eq!(
Reply::from_frame(&[0x81, 0, 1]),
Err(FrameError::InvalidLength { id: 0x81, len: 3 })
);
}
}
}
}
satrs-example/src/acs/mgm.rs
+59 -143
View File
@@ -1,4 +1,4 @@
use satrs::fdir::{FaultCounterStd, FaultResponse, RecoveryEvent, RecoveryFdir};
use satrs::fdir::{FaultCounterStd, RecoveryEvent};
use satrs::health::HealthTableMapSync;
use satrs::spacepackets::CcsdsPacketIdAndPsc;
use satrs_example::{HkHelperSingleSet, TimestampHelper, TmtcQueues};
@@ -15,6 +15,7 @@ use types::pcdu::SwitchId;
use types::{ComponentId, DeviceMode, HkRequestType, acs::mgm};
use crate::ccsds::pack_ccsds_tm_packet_for_now;
use crate::device_fdir::{DeviceFdir, FdirEvent};
use crate::device_mode::{ModeTransitionEvent, SwitchAndModeHelper};
use crate::eps::PowerSwitchHelper;
@@ -34,13 +35,6 @@ pub const Z_LOWBYTE_IDX: usize = 13;
pub const SPI_FAULT_THRESHOLD: u32 = 2;
pub const SPI_FAULT_DECREMENT_AFTER: Duration = Duration::from_secs(30);
// FDIR configuration for power cycle recoveries. The component is marked faulty if it would be
// recovered more than RECOVERY_THRESHOLD times before the counter is decremented again.
pub const RECOVERY_THRESHOLD: u32 = 2;
pub const RECOVERY_DECREMENT_AFTER: Duration = Duration::from_secs(60);
/// Time the device stays unpowered during a power cycle, so it can fully discharge.
pub const RECOVERY_OFF_DURATION: Duration = Duration::from_millis(500);
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum MgmId {
_0,
@@ -48,7 +42,7 @@ pub enum MgmId {
}
impl MgmId {
pub const fn str(&self) -> &str {
pub const fn str(&self) -> &'static str {
match self {
MgmId::_0 => "MGM 0",
MgmId::_1 => "MGM 1",
@@ -176,11 +170,13 @@ pub struct ModeLeafHelper {
///
/// This device handler includes several components beyond the scope of reading sensor values:
///
/// - FDIR handling on communication issues.
/// - FDIR helper for power cycling the device on communication issues.
/// - Event generation for certain events like communication issues.
/// - HK helper for periodic data generation.
/// - Mode leaf helper to allow integration into a full ACS mode tree
/// - The [SpiCommunication] structure models different communication interfaces to the
/// physical device.
/// - The device manages and commands its own power switch using the [SwitchAndModeHelper].
/// - The device FDIR is integrated directly into the device handler using the [DeviceFdir]
/// helper. It power cycles the device on communication issues and generates events for them.
/// - Periodic HK is generated using the [HkHelperSingleSet] helper.
/// - The device is a mode leaf in the ACS tree and has a [ModeLeafHelper] for this.
pub struct MgmHandlerLis3Mdl {
id: MgmId,
tmtc_queues: TmtcQueues,
@@ -191,9 +187,7 @@ pub struct MgmHandlerLis3Mdl {
hk_helper: HkHelperSingleSet,
switch_and_mode_helper: SwitchAndModeHelper<DeviceMode>,
mode_leaf_helper: ModeLeafHelper,
spi_fault_counter: FaultCounterStd,
fdir: RecoveryFdir<HealthTableMapSync>,
recovery_off_duration: Duration,
fdir: DeviceFdir,
event_tx: mpsc::SyncSender<(ComponentId, mgm::Event)>,
}
@@ -225,14 +219,12 @@ impl MgmHandlerLis3Mdl {
stamp_helper: TimestampHelper::default(),
hk_helper: HkHelperSingleSet::new(false, Duration::from_millis(200)),
mode_leaf_helper,
spi_fault_counter: FaultCounterStd::new(SPI_FAULT_THRESHOLD, SPI_FAULT_DECREMENT_AFTER),
fdir: RecoveryFdir::new(
id.component_id().into(),
fdir: DeviceFdir::new(
id.str(),
id.component_id(),
health_table,
RECOVERY_THRESHOLD,
RECOVERY_DECREMENT_AFTER,
FaultCounterStd::new(SPI_FAULT_THRESHOLD, SPI_FAULT_DECREMENT_AFTER),
),
recovery_off_duration: RECOVERY_OFF_DURATION,
event_tx,
}
}
@@ -253,8 +245,9 @@ impl MgmHandlerLis3Mdl {
// Handle assembly related messages.
self.handle_mode_leaf_handling();
self.fdir.periodic_operation();
self.check_needs_recovery();
self.fdir
.periodic_operation(&mut self.switch_and_mode_helper);
self.handle_fdir_events();
// Handle mode transitions first. This also takes care of recoveries required by FDIR.
if let Some(event) = self.switch_and_mode_helper.handle_mode_transition() {
@@ -266,9 +259,14 @@ impl MgmHandlerLis3Mdl {
self.handle_mode_transition_failure(tc_commander)
}
// The mode did not change for other components, so there is nothing to report.
ModeTransitionEvent::PowerCycleDone => self.handle_recovery_done(),
ModeTransitionEvent::PowerCycleDone => {
self.fdir.handle_power_cycle_done();
self.handle_fdir_events();
}
ModeTransitionEvent::PowerCycleFailed { restore_mode } => {
self.handle_recovery_failure(restore_mode)
self.fdir
.handle_power_cycle_failed(&mut self.switch_and_mode_helper, restore_mode);
self.handle_fdir_events();
}
}
}
@@ -319,7 +317,7 @@ impl MgmHandlerLis3Mdl {
},
mgm::request::Request::Health(health_request) => {
match health_request {
mgm::request::HealthRequest::SetHealth(health_state) => {
types::HealthRequest::SetHealth(health_state) => {
log::info!(
"{}: setting health to {:?} via ground command",
self.id.str(),
@@ -453,7 +451,7 @@ impl MgmHandlerLis3Mdl {
return;
}
// Successfull readout, so we can decrement the counter.
self.spi_fault_counter.try_decrement();
self.fdir.register_success();
// Simple scaling to retrieve the float value, assuming the best sensor resolution.
let mut mgm_guard = self.shared_mgm_set.lock().unwrap();
mgm_guard.x = x_raw as f32 * GAUSS_TO_MICROTESLA_FACTOR as f32 * FIELD_LSB_PER_GAUSS_4_SENS;
@@ -463,122 +461,39 @@ impl MgmHandlerLis3Mdl {
drop(mgm_guard);
}
/// Registers one SPI fault with the FDIR fault counter, invalidating the current
/// sensor set. If the failure threshold is exceeded, the device is power cycled. If it was
/// power cycled too often, the component is marked faulty and commanded off instead.
/// Registers one SPI fault with the FDIR, invalidating the current sensor set.
fn register_spi_fault(&mut self) {
log::warn!("{}: stuck-bus SPI fault", self.id.str());
self.shared_mgm_set.lock().unwrap().valid = false;
if !self.spi_fault_counter.increment_and_check() {
return;
}
match self.fdir.handle_fault() {
FaultResponse::Ignored => {
log::info!(
"{}: SPI fault threshold exceeded, but component is already faulty, \
recovering or externally controlled",
self.id.str()
);
}
FaultResponse::Recover => {
log::warn!(
"{}: SPI fault threshold exceeded, power cycling device",
self.id.str()
);
self.send_event(mgm::Event::SpiFaultThresholdExceeded);
self.check_needs_recovery();
}
FaultResponse::SetFaulty => {
log::error!(
"{}: SPI fault threshold exceeded after too many recoveries, marking \
component faulty",
self.id.str()
);
self.send_event(mgm::Event::SpiFaultThresholdExceeded);
self.send_event(mgm::Event::Recovery(RecoveryEvent::ThresholdExceeded));
self.switch_off_faulty_device();
}
self.fdir.register_fault(&mut self.switch_and_mode_helper);
self.handle_fdir_events();
}
fn handle_fdir_events(&mut self) {
while let Some(event) = self.fdir.next_event() {
let event = match event {
FdirEvent::FaultThresholdExceeded => mgm::Event::SpiFaultThresholdExceeded,
FdirEvent::Recovery(recovery_event) => {
// The device is power cycled or switched off.
if matches!(
recovery_event,
RecoveryEvent::Started | RecoveryEvent::ThresholdExceeded
) {
self.shared_mgm_set.lock().unwrap().valid = false;
}
mgm::Event::Recovery(recovery_event)
}
};
self.send_event(event);
}
}
fn switch_off_faulty_device(&mut self) {
// Do not restart an already pending Off transition, which would reset the transition
// state machine before it can finish.
if self.switch_and_mode_helper.target() != Some(DeviceMode::Off) {
log::warn!("{}: commanding device off due to fault", self.id.str());
self.start_transition(DeviceMode::Off, None);
}
}
/// Starts a power cycle if the health is [satrs::health::HealthState::NeedsRecovery]. The health is set
/// either by the FDIR or by ground.
fn check_needs_recovery(&mut self) {
if self.switch_and_mode_helper.power_cycle_active()
|| self.switch_and_mode_helper.target().is_some()
|| !self.fdir.needs_recovery()
{
return;
}
if self.mode() == DeviceMode::Off {
// Nothing to power cycle, the next switch-on is a fresh start anyway.
log::info!("{}: device is off, no recovery required", self.id.str());
self.fdir.recovery_done();
return;
}
self.start_recovery(self.mode());
}
fn start_recovery(&mut self, restore_mode: DeviceMode) {
log::warn!("{}: starting power cycle recovery", self.id.str());
self.shared_mgm_set.lock().unwrap().valid = false;
self.switch_and_mode_helper
.start_power_cycle(restore_mode, self.recovery_off_duration);
self.send_event(mgm::Event::Recovery(RecoveryEvent::Started));
}
fn handle_recovery_done(&mut self) {
log::info!("{}: power cycle recovery done", self.id.str());
// Faults registered while the device was switched off do not count anymore.
self.spi_fault_counter.clear();
self.fdir.recovery_done();
self.send_event(mgm::Event::Recovery(RecoveryEvent::Done));
}
/// A failed power cycle costs a recovery attempt like any other fault.
fn handle_recovery_failure(&mut self, restore_mode: DeviceMode) {
self.send_event(mgm::Event::Recovery(RecoveryEvent::Failed));
match self.fdir.recovery_failed() {
FaultResponse::Recover => {
log::warn!("{}: power cycle recovery failed, retrying", self.id.str());
self.start_recovery(restore_mode);
}
FaultResponse::SetFaulty => {
log::error!(
"{}: power cycle recovery failed too often, marking component faulty",
self.id.str()
);
self.send_event(mgm::Event::Recovery(RecoveryEvent::ThresholdExceeded));
self.switch_off_faulty_device();
}
// Ground changed the health during the recovery and is in charge now.
FaultResponse::Ignored => (),
}
}
/// Mode commands from ground or the parent abort a running recovery.
fn handle_mode_command(
&mut self,
target_mode: DeviceMode,
tc_commander: Option<CcsdsPacketIdAndPsc>,
) {
if self.switch_and_mode_helper.power_cycle_active() {
log::warn!(
"{}: mode command aborts power cycle recovery",
self.id.str()
);
// Otherwise, the recovery would restart right away.
self.fdir.recovery_done();
}
self.fdir.handle_mode_command(&self.switch_and_mode_helper);
self.start_transition(target_mode, tc_commander);
}
@@ -662,6 +577,7 @@ mod tests {
pcdu::{SwitchRequest, SwitchState, SwitchStateBinary},
};
use crate::device_fdir::RECOVERY_THRESHOLD;
use crate::eps::pcdu::{SharedSwitchSet, SwitchMap, SwitchSet};
use super::*;
@@ -734,7 +650,7 @@ mod tests {
health_table.clone(),
event_tx,
);
handler.recovery_off_duration = Duration::ZERO;
handler.fdir.recovery_off_duration = Duration::ZERO;
Self {
assembly_mode_request_tx,
mode_report_rx,
@@ -1106,7 +1022,7 @@ mod tests {
testbench.switch_to_normal();
testbench.drain_events();
testbench.drain_switch_requests();
testbench.mode_report_rx.try_iter().for_each(drop);
while testbench.mode_report_rx.try_recv().is_ok() {}
testbench.exceed_spi_fault_threshold();
testbench.test_spi_interface().next_mgm_data = sim_mgm::RawValues::default();
let call_count = testbench.test_spi_interface().call_count;
@@ -1132,7 +1048,7 @@ mod tests {
mgm::Event::Recovery(RecoveryEvent::Done),
]
));
assert_eq!(testbench.handler.spi_fault_counter.fault_count(), 0);
assert_eq!(testbench.handler.fdir.fault_count(), 0);
assert!(testbench.handler.shared_mgm_set.lock().unwrap().valid);
}
@@ -1189,7 +1105,7 @@ mod tests {
.tc_tx
.send(create_request_tc(
MgmSelect::_0,
mgm::request::Request::Health(mgm::request::HealthRequest::SetHealth(
mgm::request::Request::Health(types::HealthRequest::SetHealth(
HealthState::NeedsRecovery,
)),
))
@@ -1234,7 +1150,7 @@ mod tests {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench.drain_events();
testbench.mode_report_rx.try_iter().for_each(drop);
while testbench.mode_report_rx.try_recv().is_ok() {}
testbench.exceed_spi_fault_threshold();
// The switch never turns on again. Every failed power cycle costs a recovery attempt.
@@ -1279,7 +1195,7 @@ mod tests {
let mut testbench = MgmTestbench::new();
testbench.switch_to_normal();
testbench.drain_events();
testbench.mode_report_rx.try_iter().for_each(drop);
while testbench.mode_report_rx.try_recv().is_ok() {}
testbench.exceed_spi_fault_threshold();
testbench.test_spi_interface().next_mgm_data = sim_mgm::RawValues::default();
@@ -1324,7 +1240,7 @@ mod tests {
.tc_tx
.send(create_request_tc(
MgmSelect::_0,
mgm::request::Request::Health(mgm::request::HealthRequest::SetHealth(
mgm::request::Request::Health(types::HealthRequest::SetHealth(
HealthState::ExternalControl,
)),
))
@@ -1343,10 +1259,10 @@ mod tests {
testbench.handler.periodic_operation();
testbench.set_switch_state(SwitchState::Off);
// Keep the device off until the parent asked for its mode.
testbench.handler.recovery_off_duration = Duration::from_secs(60);
testbench.handler.fdir.recovery_off_duration = Duration::from_secs(60);
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
testbench.mode_report_rx.try_iter().for_each(drop);
while testbench.mode_report_rx.try_recv().is_ok() {}
testbench
.assembly_mode_request_tx
satrs-example/src/acs/mgt.rs
+387 -107
View File
@@ -2,62 +2,67 @@ use std::collections::VecDeque;
use std::sync::mpsc;
use std::time::{Duration, Instant};
use satrs::fdir::{FaultCounterStd, RecoveryEvent};
use satrs::health::HealthTableMapSync;
use satrs::spacepackets::CcsdsPacketIdAndPsc;
use satrs_example::{HkHelperSingleSet, TmtcQueues};
use satrs_minisim::acs::mgt as sim_mgt;
use satrs_minisim::{SimReply, SimRequestWithTime};
use satrs_minisim::{SimReply, SimRequest, SimRequestWithTime};
use types::acs::mgt::{
self, HkSet,
request::{ModeRequest, Request},
response::{ModeResponse, Response},
};
use types::pcdu::SwitchId;
use types::{ComponentId, DeviceMode, HkRequestType};
use types::{ComponentId, DeviceMode, HealthRequest, HkRequestType};
use crate::ccsds::pack_ccsds_tm_packet_for_now;
use crate::device_fdir::{DeviceFdir, FdirEvent};
use crate::device_mode::{ModeTransitionEvent, SwitchAndModeHelper};
use crate::eps::PowerSwitchHelper;
// The handler blocks while waiting for a reply, so this must be well below the cycle time
// of the ACS thread.
pub const REPLY_TIMEOUT: Duration = Duration::from_millis(50);
pub const REPLY_FAULT_THRESHOLD: u32 = 3;
pub const REPLY_FAULT_DECREMENT_AFTER: Duration = Duration::from_secs(30);
/// Interface for ideal device which never fails.
#[derive(Default)]
pub struct DummyInterface {
dipole: sim_mgt::Dipole,
dipole: mgt::Dipole,
torque_end: Option<Instant>,
hk_requested: bool,
}
impl DummyInterface {
fn send(&mut self, request: sim_mgt::Request) {
match request {
fn transfer(&mut self, frame: &[u8]) -> Option<Vec<u8>> {
let reply = match sim_mgt::Request::from_frame(frame).ok()? {
sim_mgt::Request::ApplyTorque { duration, dipole } => {
self.dipole = dipole;
self.torque_end = Some(Instant::now() + duration);
sim_mgt::Reply::Ack
}
sim_mgt::Request::RequestHk => self.hk_requested = true,
}
}
fn try_recv_hk(&mut self) -> Option<sim_mgt::HkSet> {
if !std::mem::take(&mut self.hk_requested) {
return None;
}
let torquing = self.torque_end.is_some_and(|end| Instant::now() < end);
Some(sim_mgt::HkSet {
dipole: if torquing {
self.dipole
} else {
sim_mgt::Dipole::default()
},
torquing,
})
sim_mgt::Request::RequestHk => {
let torquing = self.torque_end.is_some_and(|end| Instant::now() < end);
sim_mgt::Reply::Hk(sim_mgt::HkSet {
dipole: if torquing {
self.dipole
} else {
mgt::Dipole::default()
},
torquing,
})
}
};
Some(reply.to_frame())
}
}
/// Records all requests and returns injected HK replies.
/// Records all sent frames and returns injected reply frames.
#[derive(Default)]
pub struct TestInterface {
pub sent_requests: Vec<sim_mgt::Request>,
pub hk_replies: VecDeque<sim_mgt::HkSet>,
pub sent_frames: Vec<Vec<u8>>,
pub replies: VecDeque<Vec<u8>>,
}
pub struct SimInterface {
@@ -66,20 +71,21 @@ pub struct SimInterface {
}
impl SimInterface {
fn send(&mut self, request: sim_mgt::Request) {
fn transfer(&mut self, frame: &[u8]) -> Option<Vec<u8>> {
// Replies which arrived after a previous timeout must not be mistaken for this reply.
while self.sim_reply_rx.try_recv().is_ok() {}
if let Err(e) = self
.sim_request_tx
.send(SimRequestWithTime::new_with_epoch_time(request))
.send(SimRequestWithTime::new_with_epoch_time(SimRequest::Mgt(
frame.to_vec(),
)))
{
log::error!("failed to send MGT SIM request: {e}");
return None;
}
}
fn try_recv_hk(&mut self) -> Option<sim_mgt::HkSet> {
let sim_reply = self.sim_reply_rx.try_recv().ok()?;
match sim_reply {
SimReply::Mgt(sim_mgt::Reply::Hk(hk)) => Some(hk),
_ => {
match self.sim_reply_rx.recv_timeout(REPLY_TIMEOUT).ok()? {
SimReply::Mgt(frame) => Some(frame),
sim_reply => {
log::warn!("unexpected MGT SIM reply: {sim_reply:?}");
None
}
@@ -87,6 +93,7 @@ impl SimInterface {
}
}
/// Frame based transport to the device. The handler implements the protocol on top of it.
pub enum MgtCommunication {
Dummy(DummyInterface),
Sim(SimInterface),
@@ -95,19 +102,16 @@ pub enum MgtCommunication {
}
impl MgtCommunication {
fn send(&mut self, request: sim_mgt::Request) {
/// Sends a request frame and blocks until the reply frame arrives. Returns [None] if there
/// was no reply in time.
fn transfer(&mut self, frame: &[u8]) -> Option<Vec<u8>> {
match self {
MgtCommunication::Dummy(dummy) => dummy.send(request),
MgtCommunication::Sim(sim) => sim.send(request),
MgtCommunication::Test(test) => test.sent_requests.push(request),
}
}
fn try_recv_hk(&mut self) -> Option<sim_mgt::HkSet> {
match self {
MgtCommunication::Dummy(dummy) => dummy.try_recv_hk(),
MgtCommunication::Sim(sim) => sim.try_recv_hk(),
MgtCommunication::Test(test) => test.hk_replies.pop_front(),
MgtCommunication::Dummy(dummy) => dummy.transfer(frame),
MgtCommunication::Sim(sim) => sim.transfer(frame),
MgtCommunication::Test(test) => {
test.sent_frames.push(frame.to_vec());
test.replies.pop_front()
}
}
}
}
@@ -122,8 +126,19 @@ pub struct ModeLeafHelper {
/// Magnetorquer (MGT) device handler.
///
/// The device is powered through the PCDU and only accepts torque commands in normal mode.
/// In normal mode, the device HK is polled every cycle. The replies arrive asynchronously and
/// are cached as the HK set of the handler.
/// In normal mode, the device HK is polled every cycle and cached as the HK set of the handler.
/// Every request is answered with exactly one reply. A missing, invalid or unexpected reply is
/// a fault which is handled by the FDIR.
///
/// This device handler includes several components beyond the scope of commanding the device:
///
/// - The [MgtCommunication] structure models different communication interfaces to the
/// physical device.
/// - The device manages and commands its own power switch using the [SwitchAndModeHelper].
/// - The device FDIR is integrated directly into the device handler using the [DeviceFdir]
/// helper.
/// - Periodic HK is generated using the [HkHelperSingleSet] helper.
/// - The device is a mode leaf in the ACS tree and has a [ModeLeafHelper] for this.
pub struct MgtHandler {
tmtc_queues: TmtcQueues,
pub com: MgtCommunication,
@@ -131,6 +146,7 @@ pub struct MgtHandler {
hk_helper: HkHelperSingleSet,
switch_and_mode_helper: SwitchAndModeHelper<DeviceMode>,
mode_leaf_helper: ModeLeafHelper,
fdir: DeviceFdir,
event_tx: mpsc::SyncSender<mgt::Event>,
}
@@ -141,6 +157,7 @@ impl MgtHandler {
com: MgtCommunication,
mode_leaf_helper: ModeLeafHelper,
mode_timeout: Duration,
health_table: HealthTableMapSync,
event_tx: mpsc::SyncSender<mgt::Event>,
) -> Self {
Self {
@@ -155,6 +172,12 @@ impl MgtHandler {
SwitchId::Mgt,
),
mode_leaf_helper,
fdir: DeviceFdir::new(
"MGT",
ComponentId::AcsMgt,
health_table,
FaultCounterStd::new(REPLY_FAULT_THRESHOLD, REPLY_FAULT_DECREMENT_AFTER),
),
event_tx,
}
}
@@ -168,6 +191,10 @@ impl MgtHandler {
self.handle_telecommands();
self.handle_mode_leaf_handling();
self.fdir
.periodic_operation(&mut self.switch_and_mode_helper);
self.handle_fdir_events();
if let Some(event) = self.switch_and_mode_helper.handle_mode_transition() {
match event {
ModeTransitionEvent::Reached(tc_commander) => {
@@ -176,25 +203,20 @@ impl MgtHandler {
ModeTransitionEvent::Failed(tc_commander) => {
self.handle_mode_transition_failure(tc_commander)
}
// No power cycles are started without FDIR.
ModeTransitionEvent::PowerCycleDone
| ModeTransitionEvent::PowerCycleFailed { .. } => (),
ModeTransitionEvent::PowerCycleDone => {
self.fdir.handle_power_cycle_done();
self.handle_fdir_events();
}
ModeTransitionEvent::PowerCycleFailed { restore_mode } => {
self.fdir
.handle_power_cycle_failed(&mut self.switch_and_mode_helper, restore_mode);
self.handle_fdir_events();
}
}
}
if self.ready_for_commanding() {
self.com.send(sim_mgt::Request::RequestHk);
}
while let Some(hk) = self.com.try_recv_hk() {
self.hk_set = HkSet {
valid: true,
dipole: types::acs::mgt::Dipole {
x: hk.dipole.x,
y: hk.dipole.y,
z: hk.dipole.z,
},
torquing: hk.torquing,
};
self.poll_hk();
}
if self.hk_helper.needs_generation() {
@@ -202,6 +224,54 @@ impl MgtHandler {
}
}
fn poll_hk(&mut self) {
match self.transfer(sim_mgt::Request::RequestHk) {
Some(sim_mgt::Reply::Hk(hk)) => {
self.fdir.register_success();
self.hk_set = HkSet {
valid: true,
dipole: hk.dipole,
torquing: hk.torquing,
};
}
reply => self.register_reply_fault(reply),
}
}
/// Returns [None] if there was no reply in time or the reply frame was invalid.
fn transfer(&mut self, request: sim_mgt::Request) -> Option<sim_mgt::Reply> {
let frame = self.com.transfer(&request.to_frame())?;
sim_mgt::Reply::from_frame(&frame)
.inspect_err(|e| log::warn!("MGT: invalid reply frame {frame:02x?}: {e}"))
.ok()
}
fn register_reply_fault(&mut self, reply: Option<sim_mgt::Reply>) {
log::warn!("MGT: missing or unexpected reply {reply:?}");
self.hk_set.valid = false;
self.fdir.register_fault(&mut self.switch_and_mode_helper);
self.handle_fdir_events();
}
fn handle_fdir_events(&mut self) {
while let Some(event) = self.fdir.next_event() {
let event = match event {
FdirEvent::FaultThresholdExceeded => mgt::Event::ReplyFaultThresholdExceeded,
FdirEvent::Recovery(recovery_event) => {
// The device is power cycled or switched off.
if matches!(
recovery_event,
RecoveryEvent::Started | RecoveryEvent::ThresholdExceeded
) {
self.hk_set = HkSet::default();
}
mgt::Event::Recovery(recovery_event)
}
};
self.send_event(event);
}
}
fn ready_for_commanding(&self) -> bool {
self.mode() == DeviceMode::Normal && self.switch_and_mode_helper.target().is_none()
}
@@ -227,11 +297,20 @@ impl MgtHandler {
Request::Mode(ModeRequest::SetMode(mode)) => {
self.start_transition(mode, Some(tc_id))
}
Request::Mode(ModeRequest::ReadMode) => self
.send_telemetry(Some(tc_id), Response::Mode(ModeResponse::Mode(self.mode()))),
Request::Mode(ModeRequest::ReadMode) => self.send_telemetry(
Some(tc_id),
Response::Mode(ModeResponse::Mode(
self.switch_and_mode_helper.reported_mode(),
)),
),
Request::ApplyTorque { dipole, duration } => {
self.handle_torque_command(tc_id, dipole, duration)
}
Request::Health(HealthRequest::SetHealth(health)) => {
log::info!("MGT: setting health to {health:?} via ground command");
self.fdir.set_health(health);
self.send_telemetry(Some(tc_id), Response::Ok);
}
}
}
}
@@ -263,7 +342,7 @@ impl MgtHandler {
fn handle_torque_command(
&mut self,
tc_id: CcsdsPacketIdAndPsc,
dipole: types::acs::mgt::Dipole,
dipole: mgt::Dipole,
duration: Duration,
) {
if !self.ready_for_commanding() {
@@ -271,15 +350,16 @@ impl MgtHandler {
self.send_telemetry(Some(tc_id), Response::NotInNormalMode);
return;
}
self.com.send(sim_mgt::Request::ApplyTorque {
duration,
dipole: sim_mgt::Dipole {
x: dipole.x,
y: dipole.y,
z: dipole.z,
},
});
self.send_telemetry(Some(tc_id), Response::Ok);
match self.transfer(sim_mgt::Request::ApplyTorque { duration, dipole }) {
Some(sim_mgt::Reply::Ack) => {
self.fdir.register_success();
self.send_telemetry(Some(tc_id), Response::Ok);
}
reply => {
self.register_reply_fault(reply);
self.send_telemetry(Some(tc_id), Response::ReplyTimeout);
}
}
}
fn start_transition(
@@ -288,6 +368,7 @@ impl MgtHandler {
tc_commander: Option<CcsdsPacketIdAndPsc>,
) {
log::info!("MGT: transitioning to mode {:?}", target_mode);
self.fdir.handle_mode_command(&self.switch_and_mode_helper);
if target_mode == DeviceMode::Off {
self.hk_set = HkSet::default();
}
@@ -317,7 +398,9 @@ impl MgtHandler {
fn report_mode_to_parent(&self) {
self.mode_leaf_helper
.report_tx
.send(ModeResponse::Mode(self.mode()))
.send(ModeResponse::Mode(
self.switch_and_mode_helper.reported_mode(),
))
.unwrap();
}
@@ -344,6 +427,7 @@ mod tests {
use std::sync::Mutex;
use arbitrary_int::u11;
use satrs::health::{HealthState, HealthTableProvider};
use satrs::spacepackets::SpacePacketHeader;
use types::{
Apid, Message as _, TcHeader,
@@ -351,10 +435,24 @@ mod tests {
pcdu::{SwitchRequest, SwitchState, SwitchStateBinary},
};
use crate::device_fdir::RECOVERY_THRESHOLD;
use crate::eps::pcdu::{SharedSwitchSet, SwitchMap, SwitchSet};
use super::*;
impl TestInterface {
fn sent_requests(&self) -> Vec<sim_mgt::Request> {
self.sent_frames
.iter()
.map(|frame| sim_mgt::Request::from_frame(frame).unwrap())
.collect()
}
fn push_reply(&mut self, reply: sim_mgt::Reply) {
self.replies.push_back(reply.to_frame());
}
}
struct MgtTestbench {
parent_request_tx: mpsc::SyncSender<ModeRequest>,
parent_report_rx: mpsc::Receiver<ModeResponse>,
@@ -363,6 +461,7 @@ mod tests {
tc_tx: mpsc::SyncSender<CcsdsTcPacketOwned>,
tm_rx: mpsc::Receiver<CcsdsTmPacketOwned>,
event_rx: mpsc::Receiver<mgt::Event>,
health_table: HealthTableMapSync,
handler: MgtHandler,
}
@@ -373,11 +472,12 @@ mod tests {
let (tc_tx, tc_rx) = mpsc::sync_channel(10);
let (tm_tx, tm_rx) = mpsc::sync_channel(10);
let (switch_tx, switch_rx) = mpsc::sync_channel(10);
let (event_tx, event_rx) = mpsc::sync_channel(10);
let (event_tx, event_rx) = mpsc::sync_channel(20);
let mut switch_map = SwitchMap::new();
switch_map.insert(SwitchId::Mgt, SwitchState::Off);
let shared_switch_set = SharedSwitchSet::new(Mutex::new(SwitchSet::new(switch_map)));
let handler = MgtHandler::new(
let health_table = HealthTableMapSync::default();
let mut handler = MgtHandler::new(
TmtcQueues { tc_rx, tm_tx },
PowerSwitchHelper::new(switch_tx, shared_switch_set.clone()),
MgtCommunication::Test(TestInterface::default()),
@@ -386,8 +486,10 @@ mod tests {
report_tx,
},
Duration::from_millis(100),
health_table.clone(),
event_tx,
);
handler.fdir.recovery_off_duration = Duration::ZERO;
Self {
parent_request_tx,
parent_report_rx,
@@ -396,6 +498,7 @@ mod tests {
tc_tx,
tm_rx,
event_rx,
health_table,
handler,
}
}
@@ -420,10 +523,7 @@ mod tests {
fn switch_to_normal(&mut self) {
self.send_tc(Request::Mode(ModeRequest::SetMode(DeviceMode::Normal)));
self.handler.periodic_operation();
self.shared_switch_set
.lock()
.unwrap()
.set_switch_state(SwitchId::Mgt, SwitchState::On);
self.set_switch_state(SwitchState::On);
self.handler.periodic_operation();
assert_eq!(self.handler.mode(), DeviceMode::Normal);
assert_eq!(self.next_response(), Response::Ok);
@@ -435,6 +535,42 @@ mod tests {
_ => panic!("unexpected MGT interface"),
}
}
fn set_switch_state(&self, state: SwitchState) {
self.shared_switch_set
.lock()
.unwrap()
.set_switch_state(SwitchId::Mgt, state);
}
fn health(&self) -> Option<HealthState> {
self.health_table.health(ComponentId::AcsMgt.into())
}
fn drain_events(&self) -> Vec<mgt::Event> {
self.event_rx.try_iter().collect()
}
/// No replies are injected, so every HK poll is a fault. The poll in the cycle which
/// reached normal mode already registered the first fault.
fn exceed_reply_fault_threshold(&mut self) {
for _ in 0..REPLY_FAULT_THRESHOLD {
self.handler.periodic_operation();
}
}
/// Drives a started power cycle recovery to completion, completing both power switch
/// handshakes.
fn complete_power_cycle(&mut self) {
self.handler.periodic_operation();
self.set_switch_state(SwitchState::Off);
self.handler.periodic_operation();
assert_eq!(self.handler.mode(), DeviceMode::Off);
self.handler.periodic_operation();
self.set_switch_state(SwitchState::On);
self.handler.periodic_operation();
assert_eq!(self.handler.mode(), DeviceMode::Normal);
}
}
#[test]
@@ -442,7 +578,7 @@ mod tests {
let mut testbench = MgtTestbench::new();
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
assert!(testbench.test_interface().sent_requests.is_empty());
assert!(testbench.test_interface().sent_frames.is_empty());
}
#[test]
@@ -455,11 +591,7 @@ mod tests {
assert_eq!(switch_request.target_state, SwitchStateBinary::On);
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
testbench
.shared_switch_set
.lock()
.unwrap()
.set_switch_state(SwitchId::Mgt, SwitchState::On);
testbench.set_switch_state(SwitchState::On);
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Normal);
assert_eq!(testbench.next_response(), Response::Ok);
@@ -481,11 +613,7 @@ mod tests {
.send(ModeRequest::SetMode(DeviceMode::Normal))
.unwrap();
testbench.handler.periodic_operation();
testbench
.shared_switch_set
.lock()
.unwrap()
.set_switch_state(SwitchId::Mgt, SwitchState::On);
testbench.set_switch_state(SwitchState::On);
testbench.handler.periodic_operation();
assert_eq!(
testbench.parent_report_rx.try_recv(),
@@ -504,14 +632,15 @@ mod tests {
});
testbench.handler.periodic_operation();
assert_eq!(testbench.next_response(), Response::NotInNormalMode);
assert!(testbench.test_interface().sent_requests.is_empty());
assert!(testbench.test_interface().sent_frames.is_empty());
}
#[test]
fn test_torque_command_forwarded_in_normal_mode() {
let mut testbench = MgtTestbench::new();
testbench.switch_to_normal();
testbench.test_interface().sent_requests.clear();
testbench.test_interface().sent_frames.clear();
testbench.test_interface().push_reply(sim_mgt::Reply::Ack);
testbench.send_tc(Request::ApplyTorque {
dipole: mgt::Dipole { x: 1, y: 2, z: 3 },
@@ -520,14 +649,26 @@ mod tests {
testbench.handler.periodic_operation();
assert_eq!(testbench.next_response(), Response::Ok);
assert_eq!(
testbench.test_interface().sent_requests.first(),
testbench.test_interface().sent_requests().first(),
Some(&sim_mgt::Request::ApplyTorque {
duration: Duration::from_millis(100),
dipole: sim_mgt::Dipole { x: 1, y: 2, z: 3 },
dipole: mgt::Dipole { x: 1, y: 2, z: 3 },
})
);
}
#[test]
fn test_torque_command_without_ack() {
let mut testbench = MgtTestbench::new();
testbench.switch_to_normal();
testbench.send_tc(Request::ApplyTorque {
dipole: mgt::Dipole { x: 1, y: 2, z: 3 },
duration: Duration::from_millis(100),
});
testbench.handler.periodic_operation();
assert_eq!(testbench.next_response(), Response::ReplyTimeout);
}
#[test]
fn test_hk_polling_updates_hk_set() {
let mut testbench = MgtTestbench::new();
@@ -535,17 +676,16 @@ mod tests {
assert!(
testbench
.test_interface()
.sent_requests
.sent_requests()
.contains(&sim_mgt::Request::RequestHk)
);
testbench
.test_interface()
.hk_replies
.push_back(sim_mgt::HkSet {
dipole: sim_mgt::Dipole { x: 1, y: 2, z: 3 },
.push_reply(sim_mgt::Reply::Hk(sim_mgt::HkSet {
dipole: mgt::Dipole { x: 1, y: 2, z: 3 },
torquing: true,
});
}));
testbench.handler.periodic_operation();
testbench.send_tc(Request::Hk(HkRequestType::OneShot));
testbench.handler.periodic_operation();
@@ -559,17 +699,45 @@ mod tests {
);
}
#[test]
fn test_switch_off() {
let mut testbench = MgtTestbench::new();
testbench.switch_to_normal();
testbench.drain_events();
while testbench.parent_report_rx.try_recv().is_ok() {}
testbench.send_tc(Request::Mode(ModeRequest::SetMode(DeviceMode::Off)));
testbench.handler.periodic_operation();
let switch_request = testbench
.switch_rx
.try_iter()
.last()
.expect("no switch request");
assert_eq!(switch_request.target_state, SwitchStateBinary::Off);
testbench.set_switch_state(SwitchState::Off);
testbench.handler.periodic_operation();
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
assert_eq!(testbench.next_response(), Response::Ok);
assert!(matches!(
testbench.drain_events().as_slice(),
[mgt::Event::ModeChanged(DeviceMode::Off)]
));
assert_eq!(
testbench.parent_report_rx.try_recv(),
Ok(ModeResponse::Mode(DeviceMode::Off))
);
}
#[test]
fn test_hk_set_invalid_after_switch_off() {
let mut testbench = MgtTestbench::new();
testbench.switch_to_normal();
testbench
.test_interface()
.hk_replies
.push_back(sim_mgt::HkSet {
dipole: sim_mgt::Dipole::default(),
.push_reply(sim_mgt::Reply::Hk(sim_mgt::HkSet {
dipole: mgt::Dipole::default(),
torquing: false,
});
}));
testbench.handler.periodic_operation();
testbench.send_tc(Request::Mode(ModeRequest::SetMode(DeviceMode::Off)));
@@ -593,4 +761,116 @@ mod tests {
testbench.handler.periodic_operation();
assert!(testbench.tm_rx.try_recv().is_err());
}
#[test]
fn test_valid_reply_no_fault() {
let mut testbench = MgtTestbench::new();
testbench
.test_interface()
.push_reply(sim_mgt::Reply::Hk(sim_mgt::HkSet {
dipole: mgt::Dipole::default(),
torquing: false,
}));
testbench.switch_to_normal();
assert_eq!(testbench.handler.fdir.fault_count(), 0);
assert!(testbench.handler.hk_set.valid);
}
#[test]
fn test_missing_reply_registers_fault() {
let mut testbench = MgtTestbench::new();
testbench.switch_to_normal();
assert_eq!(testbench.handler.fdir.fault_count(), 1);
assert!(!testbench.handler.hk_set.valid);
assert!(matches!(
testbench.drain_events().as_slice(),
[mgt::Event::ModeChanged(DeviceMode::Normal)]
));
}
#[test]
fn test_unexpected_reply_registers_fault() {
let mut testbench = MgtTestbench::new();
testbench.test_interface().push_reply(sim_mgt::Reply::Ack);
testbench.switch_to_normal();
assert_eq!(testbench.handler.fdir.fault_count(), 1);
assert!(matches!(
testbench.drain_events().as_slice(),
[mgt::Event::ModeChanged(DeviceMode::Normal)]
));
}
#[test]
fn test_invalid_reply_registers_fault() {
let mut testbench = MgtTestbench::new();
testbench.test_interface().replies.push_back(vec![0xff]);
testbench.switch_to_normal();
assert_eq!(testbench.handler.fdir.fault_count(), 1);
assert!(matches!(
testbench.drain_events().as_slice(),
[mgt::Event::ModeChanged(DeviceMode::Normal)]
));
}
#[test]
fn test_reply_fault_threshold_starts_recovery() {
let mut testbench = MgtTestbench::new();
testbench.switch_to_normal();
testbench.drain_events();
while testbench.parent_report_rx.try_recv().is_ok() {}
testbench.exceed_reply_fault_threshold();
assert_eq!(testbench.health(), Some(HealthState::NeedsRecovery));
assert!(matches!(
testbench.drain_events().as_slice(),
[
mgt::Event::ReplyFaultThresholdExceeded,
mgt::Event::Recovery(RecoveryEvent::Started)
]
));
testbench.complete_power_cycle();
assert_eq!(testbench.health(), Some(HealthState::Healthy));
assert!(matches!(
testbench.drain_events().as_slice(),
[mgt::Event::Recovery(RecoveryEvent::Done)]
));
// The power cycle is hidden from the parent.
assert!(testbench.parent_report_rx.try_recv().is_err());
}
#[test]
fn test_unresponsive_device_marked_faulty() {
let mut testbench = MgtTestbench::new();
testbench.switch_to_normal();
for _ in 0..RECOVERY_THRESHOLD {
testbench.exceed_reply_fault_threshold();
assert_eq!(testbench.health(), Some(HealthState::NeedsRecovery));
testbench.complete_power_cycle();
}
testbench.drain_events();
testbench.exceed_reply_fault_threshold();
assert_eq!(testbench.health(), Some(HealthState::Faulty));
assert!(matches!(
testbench.drain_events().as_slice(),
[
mgt::Event::ReplyFaultThresholdExceeded,
mgt::Event::Recovery(RecoveryEvent::ThresholdExceeded)
]
));
assert_eq!(
testbench.handler.switch_and_mode_helper.target(),
Some(DeviceMode::Off)
);
}
#[test]
fn test_set_health() {
let mut testbench = MgtTestbench::new();
testbench.send_tc(Request::Health(HealthRequest::SetHealth(
HealthState::Faulty,
)));
testbench.handler.periodic_operation();
assert_eq!(testbench.next_response(), Response::Ok);
assert_eq!(testbench.health(), Some(HealthState::Faulty));
}
}
satrs-example/src/device_fdir.rs
+200
View File
@@ -0,0 +1,200 @@
use std::collections::VecDeque;
use std::time::Duration;
use satrs::fdir::{FaultCounterStd, FaultResponse, RecoveryEvent, RecoveryFdir};
use satrs::health::{HealthState, HealthTableMapSync};
use types::{ComponentId, DeviceMode};
use crate::device_mode::SwitchAndModeHelper;
// The component is marked faulty if it would be recovered more than RECOVERY_THRESHOLD times
// before the counter is decremented again.
pub const RECOVERY_THRESHOLD: u32 = 2;
pub const RECOVERY_DECREMENT_AFTER: Duration = Duration::from_secs(60);
/// Time the device stays unpowered during a power cycle, so it can fully discharge.
pub const RECOVERY_OFF_DURATION: Duration = Duration::from_millis(500);
/// Generic FDIR events. The device handler maps them to its own event type.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum FdirEvent {
FaultThresholdExceeded,
Recovery(RecoveryEvent),
}
/// Fault counting and power cycle recovery for device handlers which own the power switch of
/// their device.
///
/// The handler detects faults itself and reports them with [Self::register_fault]. This helper
/// then decides whether the device is power cycled, or marked faulty and switched off, and drives
/// the [SwitchAndModeHelper] of the handler accordingly. The handler retrieves the resulting
/// events with [Self::next_event].
pub struct DeviceFdir {
name: &'static str,
fault_counter: FaultCounterStd,
recovery: RecoveryFdir<HealthTableMapSync>,
pub recovery_off_duration: Duration,
events: VecDeque<FdirEvent>,
}
impl DeviceFdir {
pub fn new(
name: &'static str,
component_id: ComponentId,
health_table: HealthTableMapSync,
fault_counter: FaultCounterStd,
) -> Self {
Self {
name,
fault_counter,
recovery: RecoveryFdir::new(
component_id.into(),
health_table,
RECOVERY_THRESHOLD,
RECOVERY_DECREMENT_AFTER,
),
recovery_off_duration: RECOVERY_OFF_DURATION,
events: VecDeque::new(),
}
}
#[cfg(test)]
pub fn fault_count(&self) -> u32 {
self.fault_counter.fault_count()
}
pub fn set_health(&mut self, health: HealthState) {
self.recovery.set_health(health);
}
pub fn next_event(&mut self) -> Option<FdirEvent> {
self.events.pop_front()
}
/// Should be called once per cycle, before the mode transition is handled. Starts a power
/// cycle if the health was set to [HealthState::NeedsRecovery] by the FDIR or by ground.
pub fn periodic_operation(&mut self, modes: &mut SwitchAndModeHelper<DeviceMode>) {
self.recovery.periodic_operation();
self.check_needs_recovery(modes);
}
pub fn register_success(&mut self) {
self.fault_counter.try_decrement();
}
/// If the fault threshold is exceeded, the device is power cycled. If it was power cycled
/// too often, the component is marked faulty and commanded off instead.
pub fn register_fault(&mut self, modes: &mut SwitchAndModeHelper<DeviceMode>) {
if !self.fault_counter.increment_and_check() {
return;
}
match self.recovery.handle_fault() {
FaultResponse::Ignored => {
log::info!(
"{}: fault threshold exceeded, but component is already faulty, \
recovering or externally controlled",
self.name
);
}
FaultResponse::Recover => {
log::warn!(
"{}: fault threshold exceeded, power cycling device",
self.name
);
self.events.push_back(FdirEvent::FaultThresholdExceeded);
self.check_needs_recovery(modes);
}
FaultResponse::SetFaulty => {
log::error!(
"{}: fault threshold exceeded after too many recoveries, marking \
component faulty",
self.name
);
self.events.push_back(FdirEvent::FaultThresholdExceeded);
self.events
.push_back(FdirEvent::Recovery(RecoveryEvent::ThresholdExceeded));
self.switch_off_faulty_device(modes);
}
}
}
/// Mode commands from ground or the parent abort a running recovery. Must be called before
/// the commanded transition is started.
pub fn handle_mode_command(&mut self, modes: &SwitchAndModeHelper<DeviceMode>) {
if modes.power_cycle_active() {
log::warn!("{}: mode command aborts power cycle recovery", self.name);
// Otherwise, the recovery would restart right away.
self.recovery.recovery_done();
}
}
pub fn handle_power_cycle_done(&mut self) {
log::info!("{}: power cycle recovery done", self.name);
// Faults registered while the device was switched off do not count anymore.
self.fault_counter.clear();
self.recovery.recovery_done();
self.events
.push_back(FdirEvent::Recovery(RecoveryEvent::Done));
}
/// A failed power cycle costs a recovery attempt like any other fault.
pub fn handle_power_cycle_failed(
&mut self,
modes: &mut SwitchAndModeHelper<DeviceMode>,
restore_mode: DeviceMode,
) {
self.events
.push_back(FdirEvent::Recovery(RecoveryEvent::Failed));
match self.recovery.recovery_failed() {
FaultResponse::Recover => {
log::warn!("{}: power cycle recovery failed, retrying", self.name);
self.start_recovery(modes, restore_mode);
}
FaultResponse::SetFaulty => {
log::error!(
"{}: power cycle recovery failed too often, marking component faulty",
self.name
);
self.events
.push_back(FdirEvent::Recovery(RecoveryEvent::ThresholdExceeded));
self.switch_off_faulty_device(modes);
}
// Ground changed the health during the recovery and is in charge now.
FaultResponse::Ignored => (),
}
}
fn check_needs_recovery(&mut self, modes: &mut SwitchAndModeHelper<DeviceMode>) {
if modes.power_cycle_active() || modes.target().is_some() || !self.recovery.needs_recovery()
{
return;
}
if modes.mode() == DeviceMode::Off {
// Nothing to power cycle, the next switch-on is a fresh start anyway.
log::info!("{}: device is off, no recovery required", self.name);
self.recovery.recovery_done();
return;
}
let restore_mode = modes.mode();
self.start_recovery(modes, restore_mode);
}
fn start_recovery(
&mut self,
modes: &mut SwitchAndModeHelper<DeviceMode>,
restore_mode: DeviceMode,
) {
log::warn!("{}: starting power cycle recovery", self.name);
modes.start_power_cycle(restore_mode, self.recovery_off_duration);
self.events
.push_back(FdirEvent::Recovery(RecoveryEvent::Started));
}
fn switch_off_faulty_device(&mut self, modes: &mut SwitchAndModeHelper<DeviceMode>) {
// Do not restart an already pending Off transition, which would reset the transition
// state machine before it can finish.
if modes.target() != Some(DeviceMode::Off) {
log::warn!("{}: commanding device off due to fault", self.name);
modes.start_transition(DeviceMode::Off, None);
}
}
}
satrs-example/src/main.rs
+2
View File
@@ -48,6 +48,7 @@ use crate::{
mod acs;
mod ccsds;
mod controller;
mod device_fdir;
mod device_mode;
mod eps;
mod event_manager;
@@ -278,6 +279,7 @@ fn main() {
report_tx: mgt_report_tx,
},
Duration::from_millis(1000),
health_table.clone(),
mgt_event_tx,
);
satrs-example/types/src/acs/mgm.rs
+1 -8
View File
@@ -1,5 +1,5 @@
pub mod request {
use crate::{DeviceMode, HkRequestType, Message};
use crate::{DeviceMode, HealthRequest, HkRequestType, Message};
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
pub enum ModeRequest {
@@ -7,13 +7,6 @@ pub mod request {
ReadMode,
}
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
pub enum HealthRequest {
/// Overrides the device's autonomous FDIR health state, for example to clear a `Faulty`
/// state set by the handler after ground has fixed or worked around the underlying issue.
SetHealth(satrs::health::HealthState),
}
#[derive(Debug, PartialEq, Eq, Clone, Copy, serde::Serialize, serde::Deserialize)]
pub enum HkId {
Sensor,
satrs-example/types/src/acs/mgt.rs
+35 -3
View File
@@ -8,6 +8,26 @@ pub struct Dipole {
pub z: i16,
}
impl Dipole {
pub const LEN: usize = 6;
pub fn to_be_bytes(&self) -> [u8; Self::LEN] {
let [x0, x1] = self.x.to_be_bytes();
let [y0, y1] = self.y.to_be_bytes();
let [z0, z1] = self.z.to_be_bytes();
[x0, x1, y0, y1, z0, z1]
}
pub fn from_be_bytes(bytes: &[u8; Self::LEN]) -> Self {
let [x0, x1, y0, y1, z0, z1] = *bytes;
Self {
x: i16::from_be_bytes([x0, x1]),
y: i16::from_be_bytes([y0, y1]),
z: i16::from_be_bytes([z0, z1]),
}
}
}
#[derive(serde::Serialize, serde::Deserialize, Default, Debug, Clone, Copy, PartialEq, Eq)]
pub struct HkSet {
pub valid: bool,
@@ -16,7 +36,7 @@ pub struct HkSet {
}
pub mod request {
use crate::{DeviceMode, HkRequestType, Message};
use crate::{DeviceMode, HealthRequest, HkRequestType, Message};
use super::Dipole;
@@ -36,6 +56,7 @@ pub mod request {
dipole: Dipole,
duration: core::time::Duration,
},
Health(HealthRequest),
}
impl Message for Request {
@@ -45,6 +66,7 @@ pub mod request {
Request::Hk(_) => crate::MessageType::Hk,
Request::Mode(_) => crate::MessageType::Mode,
Request::ApplyTorque { .. } => crate::MessageType::Action,
Request::Health(_) => crate::MessageType::Health,
}
}
}
@@ -56,6 +78,9 @@ pub mod request {
pub enum Event {
/// A commanded mode transition completed.
ModeChanged(DeviceMode),
/// Too many requests were not answered correctly. Followed by a recovery event.
ReplyFaultThresholdExceeded,
Recovery(satrs::fdir::RecoveryEvent),
}
impl crate::Message for Event {
@@ -66,7 +91,10 @@ impl crate::Message for Event {
impl crate::EventId for Event {
fn event_id(&self) -> u16 {
EventDiscriminants::from(self).into()
match self {
Event::Recovery(event) => crate::recovery_event_id(*event),
_ => EventDiscriminants::from(self).into(),
}
}
}
@@ -90,12 +118,16 @@ pub mod response {
Mode(ModeResponse),
/// The command requires the device to be in normal mode.
NotInNormalMode,
/// The device did not answer the command with a valid reply in time.
ReplyTimeout,
}
impl Message for Response {
fn message_type(&self) -> crate::MessageType {
match self {
Response::Ok | Response::NotInNormalMode => crate::MessageType::Verification,
Response::Ok | Response::NotInNormalMode | Response::ReplyTimeout => {
crate::MessageType::Verification
}
Response::Hk(_) => crate::MessageType::Hk,
Response::Mode(_) => crate::MessageType::Mode,
}
satrs-example/types/src/lib.rs
+7
View File
@@ -220,6 +220,13 @@ impl FromStr for DeviceMode {
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum HealthRequest {
/// Overrides the device's autonomous FDIR health state, for example to clear a `Faulty`
/// state set by the handler after ground has fixed or worked around the underlying issue.
SetHealth(satrs::health::HealthState),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[non_exhaustive]
pub enum HkRequestType {
satrs-mib/codegen/Cargo.toml
-1
View File
@@ -3,7 +3,6 @@ name = "satrs-mib-codegen"
version = "0.1.2"
edition = "2021"
description = "satrs-mib proc macro implementation"
homepage = "https://egit.irs.uni-stuttgart.de/rust/sat-rs"
repository = "https://egit.irs.uni-stuttgart.de/rust/sat-rs"
license = "Apache-2.0"