FDIR extensions and improvements for MGM device handler
- try power cycling now instead of going to faulty immediately - after too many power cycles in a short time frame, go to faulty - new FDIR/recovery helper which is generic - new failure variants for fault injection: transient failures to test that a power cycles could fix the issue
This commit is contained in:
Robin Mueller
+517
-113
@@ -1,5 +1,5 @@
|
||||
use satrs::fdir::FaultCounterStd;
|
||||
use satrs::health::{HealthState, HealthTableMapSync, HealthTableProvider};
|
||||
use satrs::fdir::{FaultCounterStd, FaultResponse, RecoveryEvent, RecoveryFdir};
|
||||
use satrs::health::HealthTableMapSync;
|
||||
use satrs::spacepackets::CcsdsPacketIdAndPsc;
|
||||
use satrs_example::{HkHelperSingleSet, TimestampHelper, TmtcQueues};
|
||||
use satrs_minisim::acs::MgmRequestLis3Mdl;
|
||||
@@ -36,6 +36,13 @@ pub const Z_LOWBYTE_IDX: usize = 13;
|
||||
pub const SPI_FAULT_THRESHOLD: u32 = 2;
|
||||
pub const SPI_FAULT_DECREMENT_AFTER: Duration = Duration::from_secs(30);
|
||||
|
||||
// FDIR configuration for power cycle recoveries. The component is marked faulty if it would be
|
||||
// recovered more than RECOVERY_THRESHOLD times before the counter is decremented again.
|
||||
pub const RECOVERY_THRESHOLD: u32 = 2;
|
||||
pub const RECOVERY_DECREMENT_AFTER: Duration = Duration::from_secs(60);
|
||||
/// Time the device stays unpowered during a power cycle, so it can fully discharge.
|
||||
pub const RECOVERY_OFF_DURATION: Duration = Duration::from_millis(500);
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
|
||||
pub enum MgmId {
|
||||
_0,
|
||||
@@ -158,6 +165,14 @@ pub struct ModeLeafHelper {
|
||||
}
|
||||
|
||||
/// Example MGM device handler strongly based on the LIS3MDL MEMS device.
|
||||
///
|
||||
/// This device handler includes several components beyond the scope of reading sensor values:
|
||||
///
|
||||
/// - FDIR handling on communication issues.
|
||||
/// - FDIR helper for power cycling the device on communication issues.
|
||||
/// - Event generation for certain events like communication issues.
|
||||
/// - HK helper for periodic data generation.
|
||||
/// - Mode leaf helper to allow integration into a full ACS mode tree
|
||||
pub struct MgmHandlerLis3Mdl {
|
||||
id: MgmId,
|
||||
tmtc_queues: TmtcQueues,
|
||||
@@ -169,7 +184,8 @@ pub struct MgmHandlerLis3Mdl {
|
||||
switch_and_mode_helper: SwitchAndModeHelper<DeviceMode>,
|
||||
mode_leaf_helper: ModeLeafHelper,
|
||||
spi_fault_counter: FaultCounterStd,
|
||||
health_table: HealthTableMapSync,
|
||||
fdir: RecoveryFdir<HealthTableMapSync>,
|
||||
recovery_off_duration: Duration,
|
||||
event_tx: mpsc::SyncSender<(ComponentId, mgm::Event)>,
|
||||
}
|
||||
|
||||
@@ -202,7 +218,13 @@ impl MgmHandlerLis3Mdl {
|
||||
hk_helper: HkHelperSingleSet::new(false, Duration::from_millis(200)),
|
||||
mode_leaf_helper,
|
||||
spi_fault_counter: FaultCounterStd::new(SPI_FAULT_THRESHOLD, SPI_FAULT_DECREMENT_AFTER),
|
||||
health_table,
|
||||
fdir: RecoveryFdir::new(
|
||||
id.component_id().into(),
|
||||
health_table,
|
||||
RECOVERY_THRESHOLD,
|
||||
RECOVERY_DECREMENT_AFTER,
|
||||
),
|
||||
recovery_off_duration: RECOVERY_OFF_DURATION,
|
||||
event_tx,
|
||||
}
|
||||
}
|
||||
@@ -212,6 +234,7 @@ impl MgmHandlerLis3Mdl {
|
||||
self.switch_and_mode_helper.mode()
|
||||
}
|
||||
|
||||
/// Core function called periodically to drive the handler.
|
||||
pub fn periodic_operation(&mut self) {
|
||||
// Update current time.
|
||||
self.stamp_helper.update_from_now();
|
||||
@@ -222,7 +245,10 @@ impl MgmHandlerLis3Mdl {
|
||||
// Handle assembly related messages.
|
||||
self.handle_mode_leaf_handling();
|
||||
|
||||
// Handle mode transitions first.
|
||||
self.fdir.periodic_operation();
|
||||
self.check_needs_recovery();
|
||||
|
||||
// Handle mode transitions first. This also takes care of recoveries required by FDIR.
|
||||
if let Some(event) = self.switch_and_mode_helper.handle_mode_transition() {
|
||||
match event {
|
||||
ModeTransitionEvent::Reached(tc_commander) => {
|
||||
@@ -231,11 +257,18 @@ impl MgmHandlerLis3Mdl {
|
||||
ModeTransitionEvent::Failed(tc_commander) => {
|
||||
self.handle_mode_transition_failure(tc_commander)
|
||||
}
|
||||
// The mode did not change for other components, so there is nothing to report.
|
||||
ModeTransitionEvent::PowerCycleDone => self.handle_recovery_done(),
|
||||
ModeTransitionEvent::PowerCycleFailed { restore_mode } => {
|
||||
self.handle_recovery_failure(restore_mode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Poll sensor before checking and generating HK.
|
||||
if self.mode() == DeviceMode::Normal {
|
||||
// Poll sensor before checking and generating HK. The device is not polled during mode
|
||||
// transitions, which includes all FDIR actions like power cycling or switching off a
|
||||
// faulty device. Faults are expected then, and polling would only add noise.
|
||||
if self.mode() == DeviceMode::Normal && self.switch_and_mode_helper.target().is_none() {
|
||||
log::trace!("polling LIS3MDL sensor {}", self.id.str());
|
||||
self.poll_sensor();
|
||||
}
|
||||
@@ -267,12 +300,12 @@ impl MgmHandlerLis3Mdl {
|
||||
}
|
||||
mgm::request::Request::Mode(device_mode) => match device_mode {
|
||||
ModeRequest::SetMode(device_mode) => {
|
||||
self.start_transition(device_mode, Some(tc_id));
|
||||
self.handle_mode_command(device_mode, Some(tc_id));
|
||||
}
|
||||
ModeRequest::ReadMode => self.send_telemetry(
|
||||
Some(tc_id),
|
||||
mgm::response::Response::Mode(ModeResponse::Mode(
|
||||
self.mode(),
|
||||
self.switch_and_mode_helper.reported_mode(),
|
||||
)),
|
||||
),
|
||||
},
|
||||
@@ -284,10 +317,7 @@ impl MgmHandlerLis3Mdl {
|
||||
self.id.str(),
|
||||
health_state
|
||||
);
|
||||
self.health_table.set_health(
|
||||
self.id.component_id().into(),
|
||||
health_state,
|
||||
);
|
||||
self.fdir.set_health(health_state);
|
||||
self.send_telemetry(
|
||||
Some(tc_id),
|
||||
mgm::response::Response::Ok,
|
||||
@@ -316,7 +346,9 @@ impl MgmHandlerLis3Mdl {
|
||||
loop {
|
||||
match self.mode_leaf_helper.request_rx.try_recv() {
|
||||
Ok(request) => match request {
|
||||
ModeRequest::SetMode(device_mode) => self.start_transition(device_mode, None),
|
||||
ModeRequest::SetMode(device_mode) => {
|
||||
self.handle_mode_command(device_mode, None)
|
||||
}
|
||||
ModeRequest::ReadMode => self.report_mode_to_parent(),
|
||||
},
|
||||
Err(e) => match e {
|
||||
@@ -400,14 +432,17 @@ impl MgmHandlerLis3Mdl {
|
||||
.unwrap(),
|
||||
);
|
||||
// A stuck-high SPI bus (undriven MISO) reads back as all-1s on every register,
|
||||
// regardless of what was actually requested. This is the pattern this codebase already
|
||||
// uses for "no real device behind the bus" (see the switched-off MGM sim reply). An
|
||||
// all-0s reading is not used here, since it collides with a legitimate zero-field
|
||||
// reading and would cause false positives.
|
||||
// regardless of what was actually requested.
|
||||
// If our sensor was broken, this is what we would probably see.
|
||||
// An all zeroes reading is ignored for now. the sensor could theoretically return this.
|
||||
// In a production app, we also need to check whether the sensor data never varies, which is
|
||||
// also a fault. We ignore this in this example because the handler is already complex
|
||||
// enough.
|
||||
if x_raw == -1 && y_raw == -1 && z_raw == -1 {
|
||||
self.register_spi_fault();
|
||||
return;
|
||||
}
|
||||
// Successfull readout, so we can decrement the counter.
|
||||
self.spi_fault_counter.try_decrement();
|
||||
// Simple scaling to retrieve the float value, assuming the best sensor resolution.
|
||||
let mut mgm_guard = self.shared_mgm_set.lock().unwrap();
|
||||
@@ -418,51 +453,131 @@ impl MgmHandlerLis3Mdl {
|
||||
drop(mgm_guard);
|
||||
}
|
||||
|
||||
/// Registers one stuck-bus SPI fault with the FDIR fault counter, invalidating the current
|
||||
/// sensor set. If the failure threshold is exceeded, the component is marked faulty in the
|
||||
/// global health table.
|
||||
/// Registers one SPI fault with the FDIR fault counter, invalidating the current
|
||||
/// sensor set. If the failure threshold is exceeded, the device is power cycled. If it was
|
||||
/// power cycled too often, the component is marked faulty and commanded off instead.
|
||||
fn register_spi_fault(&mut self) {
|
||||
log::warn!("{}: stuck-bus SPI fault", self.id.str());
|
||||
self.shared_mgm_set.lock().unwrap().valid = false;
|
||||
if !self.spi_fault_counter.increment_and_check() {
|
||||
return;
|
||||
}
|
||||
// Ground may have taken manual control, or already given up on this component.
|
||||
// Autonomous FDIR should not override that decision.
|
||||
let component_id = self.id.component_id().into();
|
||||
match self.health_table.health(component_id) {
|
||||
Some(HealthState::ExternalControl) | Some(HealthState::PermanentFaulty) => {
|
||||
match self.fdir.handle_fault() {
|
||||
FaultResponse::Ignored => {
|
||||
log::info!(
|
||||
"{}: SPI fault threshold exceeded, but health is externally controlled, \
|
||||
not overriding",
|
||||
"{}: SPI fault threshold exceeded, but component is already faulty, \
|
||||
recovering or externally controlled",
|
||||
self.id.str()
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
FaultResponse::Recover => {
|
||||
log::warn!(
|
||||
"{}: SPI fault threshold exceeded, power cycling device",
|
||||
self.id.str()
|
||||
);
|
||||
self.send_event(mgm::Event::SpiFaultThresholdExceeded);
|
||||
self.check_needs_recovery();
|
||||
}
|
||||
FaultResponse::SetFaulty => {
|
||||
log::error!(
|
||||
"{}: SPI fault threshold exceeded, marking component faulty",
|
||||
"{}: SPI fault threshold exceeded after too many recoveries, marking \
|
||||
component faulty",
|
||||
self.id.str()
|
||||
);
|
||||
self.health_table
|
||||
.set_health(component_id, HealthState::Faulty);
|
||||
if let Err(e) = self.event_tx.send((
|
||||
self.id.component_id(),
|
||||
mgm::Event::SpiFaultThresholdExceeded,
|
||||
)) {
|
||||
log::warn!("{}: failed to send fault event: {}", self.id.str(), e);
|
||||
}
|
||||
// Do not restart an already pending Off transition: poll_sensor still calls
|
||||
// this every cycle the fault persists, and current stays Normal until the
|
||||
// transition completes, so re-triggering here would keep resetting the
|
||||
// transition state machine before it can ever finish.
|
||||
if self.switch_and_mode_helper.target() != Some(DeviceMode::Off) {
|
||||
log::warn!("{}: commanding device off due to fault", self.id.str());
|
||||
self.start_transition(DeviceMode::Off, None);
|
||||
}
|
||||
self.send_event(mgm::Event::SpiFaultThresholdExceeded);
|
||||
self.send_event(mgm::Event::Recovery(RecoveryEvent::ThresholdExceeded));
|
||||
self.switch_off_faulty_device();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn switch_off_faulty_device(&mut self) {
|
||||
// Do not restart an already pending Off transition, which would reset the transition
|
||||
// state machine before it can finish.
|
||||
if self.switch_and_mode_helper.target() != Some(DeviceMode::Off) {
|
||||
log::warn!("{}: commanding device off due to fault", self.id.str());
|
||||
self.start_transition(DeviceMode::Off, None);
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts a power cycle if the health is [satrs::health::HealthState::NeedsRecovery]. The health is set
|
||||
/// either by the FDIR or by ground.
|
||||
fn check_needs_recovery(&mut self) {
|
||||
if self.switch_and_mode_helper.power_cycle_active()
|
||||
|| self.switch_and_mode_helper.target().is_some()
|
||||
|| !self.fdir.needs_recovery()
|
||||
{
|
||||
return;
|
||||
}
|
||||
if self.mode() == DeviceMode::Off {
|
||||
// Nothing to power cycle, the next switch-on is a fresh start anyway.
|
||||
log::info!("{}: device is off, no recovery required", self.id.str());
|
||||
self.fdir.recovery_done();
|
||||
return;
|
||||
}
|
||||
self.start_recovery(self.mode());
|
||||
}
|
||||
|
||||
fn start_recovery(&mut self, restore_mode: DeviceMode) {
|
||||
log::warn!("{}: starting power cycle recovery", self.id.str());
|
||||
self.shared_mgm_set.lock().unwrap().valid = false;
|
||||
self.switch_and_mode_helper
|
||||
.start_power_cycle(restore_mode, self.recovery_off_duration);
|
||||
self.send_event(mgm::Event::Recovery(RecoveryEvent::Started));
|
||||
}
|
||||
|
||||
fn handle_recovery_done(&mut self) {
|
||||
log::info!("{}: power cycle recovery done", self.id.str());
|
||||
// Faults registered while the device was switched off do not count anymore.
|
||||
self.spi_fault_counter.clear();
|
||||
self.fdir.recovery_done();
|
||||
self.send_event(mgm::Event::Recovery(RecoveryEvent::Done));
|
||||
}
|
||||
|
||||
/// A failed power cycle costs a recovery attempt like any other fault.
|
||||
fn handle_recovery_failure(&mut self, restore_mode: DeviceMode) {
|
||||
self.send_event(mgm::Event::Recovery(RecoveryEvent::Failed));
|
||||
match self.fdir.recovery_failed() {
|
||||
FaultResponse::Recover => {
|
||||
log::warn!("{}: power cycle recovery failed, retrying", self.id.str());
|
||||
self.start_recovery(restore_mode);
|
||||
}
|
||||
FaultResponse::SetFaulty => {
|
||||
log::error!(
|
||||
"{}: power cycle recovery failed too often, marking component faulty",
|
||||
self.id.str()
|
||||
);
|
||||
self.send_event(mgm::Event::Recovery(RecoveryEvent::ThresholdExceeded));
|
||||
self.switch_off_faulty_device();
|
||||
}
|
||||
// Ground changed the health during the recovery and is in charge now.
|
||||
FaultResponse::Ignored => (),
|
||||
}
|
||||
}
|
||||
|
||||
/// Mode commands from ground or the parent abort a running recovery.
|
||||
fn handle_mode_command(
|
||||
&mut self,
|
||||
target_mode: DeviceMode,
|
||||
tc_commander: Option<CcsdsPacketIdAndPsc>,
|
||||
) {
|
||||
if self.switch_and_mode_helper.power_cycle_active() {
|
||||
log::warn!(
|
||||
"{}: mode command aborts power cycle recovery",
|
||||
self.id.str()
|
||||
);
|
||||
// Otherwise, the recovery would restart right away.
|
||||
self.fdir.recovery_done();
|
||||
}
|
||||
self.start_transition(target_mode, tc_commander);
|
||||
}
|
||||
|
||||
fn send_event(&self, event: mgm::Event) {
|
||||
if let Err(e) = self.event_tx.send((self.id.component_id(), event)) {
|
||||
log::warn!("{}: failed to send event {:?}: {}", self.id.str(), event, e);
|
||||
}
|
||||
}
|
||||
|
||||
fn start_transition(
|
||||
&mut self,
|
||||
target_mode: DeviceMode,
|
||||
@@ -502,22 +617,15 @@ impl MgmHandlerLis3Mdl {
|
||||
|
||||
fn announce_mode(&self) {
|
||||
log::info!("{} announcing mode: {:?}", self.id.str(), self.mode());
|
||||
if let Err(e) = self
|
||||
.event_tx
|
||||
.send((self.id.component_id(), mgm::Event::ModeChanged(self.mode())))
|
||||
{
|
||||
log::warn!(
|
||||
"{}: failed to send mode changed event: {}",
|
||||
self.id.str(),
|
||||
e
|
||||
);
|
||||
}
|
||||
self.send_event(mgm::Event::ModeChanged(self.mode()));
|
||||
}
|
||||
|
||||
fn report_mode_to_parent(&self) {
|
||||
self.mode_leaf_helper
|
||||
.report_tx
|
||||
.send(ModeResponse::Mode(self.mode()))
|
||||
.send(ModeResponse::Mode(
|
||||
self.switch_and_mode_helper.reported_mode(),
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
@@ -534,6 +642,7 @@ mod tests {
|
||||
};
|
||||
|
||||
use arbitrary_int::u11;
|
||||
use satrs::health::{HealthState, HealthTableProvider};
|
||||
use satrs::spacepackets::SpacePacketHeader;
|
||||
use satrs_minisim::acs::lis3mdl::MgmLis3RawValues;
|
||||
use types::{
|
||||
@@ -589,7 +698,7 @@ mod tests {
|
||||
impl MgmTestbench {
|
||||
pub fn new() -> Self {
|
||||
let (assembly_mode_request_tx, assembly_mode_request_rx) = mpsc::sync_channel(5);
|
||||
let (mode_report_tx, mode_report_rx) = mpsc::sync_channel(5);
|
||||
let (mode_report_tx, mode_report_rx) = mpsc::sync_channel(10);
|
||||
let mode_leaf_helper = ModeLeafHelper {
|
||||
request_rx: assembly_mode_request_rx,
|
||||
report_tx: mode_report_tx,
|
||||
@@ -603,8 +712,8 @@ mod tests {
|
||||
let switch_map = SwitchSet::new(switch_map);
|
||||
let shared_switch_set = SharedSwitchSet::new(Mutex::new(switch_map));
|
||||
let health_table = HealthTableMapSync::default();
|
||||
let (event_tx, event_rx) = mpsc::sync_channel(5);
|
||||
let handler = MgmHandlerLis3Mdl::new(
|
||||
let (event_tx, event_rx) = mpsc::sync_channel(20);
|
||||
let mut handler = MgmHandlerLis3Mdl::new(
|
||||
MgmId::_0,
|
||||
TmtcQueues { tc_rx, tm_tx },
|
||||
PowerSwitchHelper::new(switcher_tx, shared_switch_set.clone()),
|
||||
@@ -615,6 +724,7 @@ mod tests {
|
||||
health_table.clone(),
|
||||
event_tx,
|
||||
);
|
||||
handler.recovery_off_duration = Duration::ZERO;
|
||||
Self {
|
||||
assembly_mode_request_tx,
|
||||
mode_report_rx,
|
||||
@@ -645,6 +755,75 @@ mod tests {
|
||||
assert_eq!(self.handler.mode(), DeviceMode::Normal);
|
||||
}
|
||||
|
||||
pub fn set_switch_state(&self, state: SwitchState) {
|
||||
self.shared_switch_set
|
||||
.lock()
|
||||
.unwrap()
|
||||
.set_switch_state(SwitchId::Mgm0, state);
|
||||
}
|
||||
|
||||
pub fn inject_stuck_bus(&mut self) {
|
||||
self.test_spi_interface().next_mgm_data = MgmLis3RawValues {
|
||||
x: -1,
|
||||
y: -1,
|
||||
z: -1,
|
||||
};
|
||||
}
|
||||
|
||||
/// Drives SPI faults until the SPI fault threshold is exceeded once.
|
||||
pub fn exceed_spi_fault_threshold(&mut self) {
|
||||
self.inject_stuck_bus();
|
||||
for _ in 0..SPI_FAULT_THRESHOLD + 1 {
|
||||
self.handler.periodic_operation();
|
||||
}
|
||||
}
|
||||
|
||||
/// Drives a started power cycle recovery to completion, completing both power-switch
|
||||
/// handshakes.
|
||||
pub fn complete_power_cycle(&mut self) {
|
||||
self.handler.periodic_operation();
|
||||
self.set_switch_state(SwitchState::Off);
|
||||
self.handler.periodic_operation();
|
||||
assert_eq!(self.handler.mode(), DeviceMode::Off);
|
||||
self.handler.periodic_operation();
|
||||
assert_eq!(
|
||||
self.handler.switch_and_mode_helper.target(),
|
||||
Some(DeviceMode::Normal)
|
||||
);
|
||||
self.set_switch_state(SwitchState::On);
|
||||
self.handler.periodic_operation();
|
||||
assert_eq!(self.handler.mode(), DeviceMode::Normal);
|
||||
}
|
||||
|
||||
/// Drives recoveries with a permanently stuck bus until the component is marked faulty.
|
||||
pub fn recover_until_faulty(&mut self) {
|
||||
self.exceed_spi_fault_threshold();
|
||||
for _ in 0..RECOVERY_THRESHOLD {
|
||||
assert_eq!(self.health(), Some(HealthState::NeedsRecovery));
|
||||
self.complete_power_cycle();
|
||||
// The last cycle of the power cycle already polled once.
|
||||
for _ in 0..SPI_FAULT_THRESHOLD {
|
||||
self.handler.periodic_operation();
|
||||
}
|
||||
}
|
||||
assert_eq!(self.health(), Some(HealthState::Faulty));
|
||||
}
|
||||
|
||||
pub fn health(&self) -> Option<HealthState> {
|
||||
self.health_table.health(ComponentId::AcsMgm0.into())
|
||||
}
|
||||
|
||||
pub fn drain_events(&self) -> Vec<mgm::Event> {
|
||||
self.event_rx.try_iter().map(|(_, event)| event).collect()
|
||||
}
|
||||
|
||||
pub fn drain_switch_requests(&self) -> Vec<SwitchStateBinary> {
|
||||
self.switch_rx
|
||||
.try_iter()
|
||||
.map(|req| req.target_state)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn test_spi_interface(&mut self) -> &mut TestSpiInterface {
|
||||
match &mut self.handler.spi_com {
|
||||
SpiCommunication::Dummy(_) | SpiCommunication::Sim(_) => {
|
||||
@@ -894,73 +1073,298 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_spi_fault_above_threshold_marks_component_faulty() {
|
||||
fn test_spi_fault_above_threshold_starts_recovery() {
|
||||
let mut testbench = MgmTestbench::new();
|
||||
testbench.switch_to_normal();
|
||||
// Drain the mode changed event emitted by switch_to_normal().
|
||||
testbench
|
||||
.event_rx
|
||||
.try_recv()
|
||||
.expect("expected mode changed event");
|
||||
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues {
|
||||
x: -1,
|
||||
y: -1,
|
||||
z: -1,
|
||||
};
|
||||
// SPI_FAULT_THRESHOLD is exceeded on the (threshold + 1)-th stuck-bus reading.
|
||||
for _ in 0..SPI_FAULT_THRESHOLD + 1 {
|
||||
testbench.handler.periodic_operation();
|
||||
}
|
||||
assert_eq!(
|
||||
testbench.health_table.health(ComponentId::AcsMgm0.into()),
|
||||
Some(HealthState::Faulty)
|
||||
);
|
||||
testbench.drain_events();
|
||||
testbench.exceed_spi_fault_threshold();
|
||||
assert_eq!(testbench.health(), Some(HealthState::NeedsRecovery));
|
||||
assert!(!testbench.handler.shared_mgm_set.lock().unwrap().valid);
|
||||
let (sender_id, event) = testbench.event_rx.try_recv().expect("expected fault event");
|
||||
assert_eq!(sender_id, ComponentId::AcsMgm0);
|
||||
assert!(matches!(event, mgm::Event::SpiFaultThresholdExceeded));
|
||||
let events = testbench.drain_events();
|
||||
assert!(matches!(
|
||||
events[..],
|
||||
[
|
||||
mgm::Event::SpiFaultThresholdExceeded,
|
||||
mgm::Event::Recovery(RecoveryEvent::Started)
|
||||
]
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_spi_fault_above_threshold_commands_device_off() {
|
||||
fn test_recovery_power_cycles_device() {
|
||||
let mut testbench = MgmTestbench::new();
|
||||
testbench.switch_to_normal();
|
||||
// Drain the switch-on request left over from switch_to_normal().
|
||||
testbench
|
||||
.switch_rx
|
||||
.try_recv()
|
||||
.expect("no switch-on request sent");
|
||||
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues {
|
||||
x: -1,
|
||||
y: -1,
|
||||
z: -1,
|
||||
};
|
||||
testbench.drain_events();
|
||||
testbench.drain_switch_requests();
|
||||
testbench.mode_report_rx.try_iter().for_each(drop);
|
||||
testbench.exceed_spi_fault_threshold();
|
||||
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues::default();
|
||||
let call_count = testbench.test_spi_interface().call_count;
|
||||
|
||||
testbench.complete_power_cycle();
|
||||
|
||||
// The device is only polled again once the power cycle is done.
|
||||
assert_eq!(testbench.test_spi_interface().call_count, call_count + 1);
|
||||
// The power cycle is hidden from the parent.
|
||||
assert!(testbench.mode_report_rx.try_recv().is_err());
|
||||
|
||||
assert_eq!(testbench.health(), Some(HealthState::Healthy));
|
||||
assert_eq!(
|
||||
testbench.drain_switch_requests(),
|
||||
[SwitchStateBinary::Off, SwitchStateBinary::On]
|
||||
);
|
||||
let events = testbench.drain_events();
|
||||
assert!(matches!(
|
||||
events[..],
|
||||
[
|
||||
mgm::Event::SpiFaultThresholdExceeded,
|
||||
mgm::Event::Recovery(RecoveryEvent::Started),
|
||||
mgm::Event::Recovery(RecoveryEvent::Done),
|
||||
]
|
||||
));
|
||||
assert_eq!(testbench.handler.spi_fault_counter.fault_count(), 0);
|
||||
assert!(testbench.handler.shared_mgm_set.lock().unwrap().valid);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_repeated_recovery_marks_component_faulty() {
|
||||
let mut testbench = MgmTestbench::new();
|
||||
testbench.switch_to_normal();
|
||||
testbench.recover_until_faulty();
|
||||
let events = testbench.drain_events();
|
||||
assert!(matches!(
|
||||
events[..],
|
||||
[
|
||||
..,
|
||||
mgm::Event::SpiFaultThresholdExceeded,
|
||||
mgm::Event::Recovery(RecoveryEvent::ThresholdExceeded)
|
||||
]
|
||||
));
|
||||
testbench.drain_switch_requests();
|
||||
|
||||
testbench.handler.periodic_operation();
|
||||
assert_eq!(testbench.drain_switch_requests(), [SwitchStateBinary::Off]);
|
||||
testbench.set_switch_state(SwitchState::Off);
|
||||
testbench.handler.periodic_operation();
|
||||
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
|
||||
assert_eq!(testbench.health(), Some(HealthState::Faulty));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_faulty_device_is_not_polled_while_switching_off() {
|
||||
let mut testbench = MgmTestbench::new();
|
||||
testbench.switch_to_normal();
|
||||
testbench.recover_until_faulty();
|
||||
let call_count = testbench.test_spi_interface().call_count;
|
||||
|
||||
// The switch-off takes a while.
|
||||
for _ in 0..SPI_FAULT_THRESHOLD + 1 {
|
||||
testbench.handler.periodic_operation();
|
||||
}
|
||||
assert_eq!(
|
||||
testbench.health_table.health(ComponentId::AcsMgm0.into()),
|
||||
Some(HealthState::Faulty)
|
||||
);
|
||||
// The Off transition was only started on the last iteration above, so it has not sent
|
||||
// its switch-off request yet: drive one more cycle to let it do so.
|
||||
assert_eq!(testbench.test_spi_interface().call_count, call_count);
|
||||
assert_eq!(testbench.health(), Some(HealthState::Faulty));
|
||||
testbench.set_switch_state(SwitchState::Off);
|
||||
testbench.handler.periodic_operation();
|
||||
let switch_req = testbench
|
||||
.switch_rx
|
||||
.try_recv()
|
||||
.expect("no switch-off request sent after fault");
|
||||
assert_eq!(switch_req.switch_id, SwitchId::Mgm0);
|
||||
assert_eq!(switch_req.target_state, SwitchStateBinary::Off);
|
||||
|
||||
// Simulate the PCDU acting on the switch-off request.
|
||||
testbench
|
||||
.shared_switch_set
|
||||
.lock()
|
||||
.unwrap()
|
||||
.set_switch_state(SwitchId::Mgm0, SwitchState::Off);
|
||||
testbench.handler.periodic_operation();
|
||||
|
||||
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
|
||||
assert_eq!(testbench.health(), Some(HealthState::Faulty));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ground_needs_recovery_power_cycles_device() {
|
||||
let mut testbench = MgmTestbench::new();
|
||||
testbench.switch_to_normal();
|
||||
testbench.drain_events();
|
||||
testbench
|
||||
.tc_tx
|
||||
.send(create_request_tc(
|
||||
MgmSelect::_0,
|
||||
mgm::request::Request::Health(mgm::request::HealthRequest::SetHealth(
|
||||
HealthState::NeedsRecovery,
|
||||
)),
|
||||
))
|
||||
.unwrap();
|
||||
testbench.handler.periodic_operation();
|
||||
assert_eq!(
|
||||
testbench.handler.switch_and_mode_helper.target(),
|
||||
Some(DeviceMode::Off)
|
||||
);
|
||||
testbench.set_switch_state(SwitchState::Off);
|
||||
testbench.handler.periodic_operation();
|
||||
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
|
||||
testbench.handler.periodic_operation();
|
||||
testbench.set_switch_state(SwitchState::On);
|
||||
testbench.handler.periodic_operation();
|
||||
assert_eq!(testbench.handler.mode(), DeviceMode::Normal);
|
||||
assert_eq!(testbench.health(), Some(HealthState::Healthy));
|
||||
let events = testbench.drain_events();
|
||||
assert!(matches!(
|
||||
events[0],
|
||||
mgm::Event::Recovery(RecoveryEvent::Started)
|
||||
));
|
||||
assert!(matches!(
|
||||
events.last(),
|
||||
Some(mgm::Event::Recovery(RecoveryEvent::Done))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_needs_recovery_while_off_sets_healthy() {
|
||||
let mut testbench = MgmTestbench::new();
|
||||
testbench
|
||||
.health_table
|
||||
.set_health(ComponentId::AcsMgm0.into(), HealthState::NeedsRecovery);
|
||||
testbench.handler.periodic_operation();
|
||||
assert_eq!(testbench.health(), Some(HealthState::Healthy));
|
||||
assert!(testbench.drain_switch_requests().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_power_cycle_switch_on_failures_mark_component_faulty() {
|
||||
let mut testbench = MgmTestbench::new();
|
||||
testbench.switch_to_normal();
|
||||
testbench.drain_events();
|
||||
testbench.mode_report_rx.try_iter().for_each(drop);
|
||||
testbench.exceed_spi_fault_threshold();
|
||||
|
||||
// The switch never turns on again. Every failed power cycle costs a recovery attempt.
|
||||
for _ in 0..RECOVERY_THRESHOLD {
|
||||
assert_eq!(testbench.health(), Some(HealthState::NeedsRecovery));
|
||||
testbench.handler.periodic_operation();
|
||||
testbench.set_switch_state(SwitchState::Off);
|
||||
testbench.handler.periodic_operation();
|
||||
testbench.handler.periodic_operation();
|
||||
std::thread::sleep(Duration::from_millis(110));
|
||||
testbench.handler.periodic_operation();
|
||||
}
|
||||
assert_eq!(testbench.health(), Some(HealthState::Faulty));
|
||||
let events = testbench.drain_events();
|
||||
let started = events
|
||||
.iter()
|
||||
.filter(|e| matches!(e, mgm::Event::Recovery(RecoveryEvent::Started)))
|
||||
.count();
|
||||
assert_eq!(started, RECOVERY_THRESHOLD as usize);
|
||||
assert!(matches!(
|
||||
events[..],
|
||||
[
|
||||
..,
|
||||
mgm::Event::Recovery(RecoveryEvent::Failed),
|
||||
mgm::Event::Recovery(RecoveryEvent::ThresholdExceeded)
|
||||
]
|
||||
));
|
||||
// Retries are hidden from the parent.
|
||||
assert!(testbench.mode_report_rx.try_recv().is_err());
|
||||
|
||||
// The faulty device is commanded off, which is reported to the parent.
|
||||
testbench.handler.periodic_operation();
|
||||
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
|
||||
assert!(matches!(
|
||||
testbench.mode_report_rx.try_recv(),
|
||||
Ok(ModeResponse::Mode(DeviceMode::Off))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_power_cycle_switch_off_failures_mark_component_faulty() {
|
||||
let mut testbench = MgmTestbench::new();
|
||||
testbench.switch_to_normal();
|
||||
testbench.drain_events();
|
||||
testbench.mode_report_rx.try_iter().for_each(drop);
|
||||
testbench.exceed_spi_fault_threshold();
|
||||
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues::default();
|
||||
|
||||
// The switch never turns off. Every failed power cycle costs a recovery attempt.
|
||||
for _ in 0..RECOVERY_THRESHOLD {
|
||||
assert_eq!(testbench.health(), Some(HealthState::NeedsRecovery));
|
||||
testbench.handler.periodic_operation();
|
||||
std::thread::sleep(Duration::from_millis(110));
|
||||
testbench.handler.periodic_operation();
|
||||
}
|
||||
assert_eq!(testbench.health(), Some(HealthState::Faulty));
|
||||
assert_eq!(testbench.handler.mode(), DeviceMode::Normal);
|
||||
assert_eq!(
|
||||
testbench.handler.switch_and_mode_helper.target(),
|
||||
Some(DeviceMode::Off)
|
||||
);
|
||||
// The mode never changed, so it was not announced or reported.
|
||||
let events = testbench.drain_events();
|
||||
assert!(
|
||||
!events
|
||||
.iter()
|
||||
.any(|e| matches!(e, mgm::Event::ModeChanged(_)))
|
||||
);
|
||||
assert!(testbench.mode_report_rx.try_recv().is_err());
|
||||
|
||||
testbench.set_switch_state(SwitchState::Off);
|
||||
testbench.handler.periodic_operation();
|
||||
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
|
||||
assert!(matches!(
|
||||
testbench.mode_report_rx.try_recv(),
|
||||
Ok(ModeResponse::Mode(DeviceMode::Off))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_health_command_during_recovery_is_kept() {
|
||||
let mut testbench = MgmTestbench::new();
|
||||
testbench.switch_to_normal();
|
||||
testbench.exceed_spi_fault_threshold();
|
||||
testbench.test_spi_interface().next_mgm_data = MgmLis3RawValues::default();
|
||||
testbench
|
||||
.tc_tx
|
||||
.send(create_request_tc(
|
||||
MgmSelect::_0,
|
||||
mgm::request::Request::Health(mgm::request::HealthRequest::SetHealth(
|
||||
HealthState::ExternalControl,
|
||||
)),
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
// The power cycle is not cancelled, but it does not override the health set by ground.
|
||||
testbench.complete_power_cycle();
|
||||
assert_eq!(testbench.health(), Some(HealthState::ExternalControl));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_read_mode_during_power_cycle_returns_restored_mode() {
|
||||
let mut testbench = MgmTestbench::new();
|
||||
testbench.switch_to_normal();
|
||||
testbench.exceed_spi_fault_threshold();
|
||||
testbench.handler.periodic_operation();
|
||||
testbench.set_switch_state(SwitchState::Off);
|
||||
// Keep the device off until the parent asked for its mode.
|
||||
testbench.handler.recovery_off_duration = Duration::from_secs(60);
|
||||
testbench.handler.periodic_operation();
|
||||
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
|
||||
testbench.mode_report_rx.try_iter().for_each(drop);
|
||||
|
||||
testbench
|
||||
.assembly_mode_request_tx
|
||||
.send(ModeRequest::ReadMode)
|
||||
.unwrap();
|
||||
testbench.handler.periodic_operation();
|
||||
assert!(matches!(
|
||||
testbench.mode_report_rx.try_recv(),
|
||||
Ok(ModeResponse::Mode(DeviceMode::Normal))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mode_command_aborts_recovery() {
|
||||
let mut testbench = MgmTestbench::new();
|
||||
testbench.switch_to_normal();
|
||||
testbench.exceed_spi_fault_threshold();
|
||||
testbench.handler.periodic_operation();
|
||||
testbench.set_switch_state(SwitchState::Off);
|
||||
testbench
|
||||
.assembly_mode_request_tx
|
||||
.send(ModeRequest::SetMode(DeviceMode::Off))
|
||||
.unwrap();
|
||||
testbench.handler.periodic_operation();
|
||||
testbench.handler.periodic_operation();
|
||||
assert_eq!(testbench.handler.mode(), DeviceMode::Off);
|
||||
assert_eq!(testbench.handler.switch_and_mode_helper.target(), None);
|
||||
assert_eq!(testbench.health(), Some(HealthState::Healthy));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,16 +1,21 @@
|
||||
use std::time::Duration;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use types::pcdu::SwitchId;
|
||||
|
||||
use crate::eps::PowerSwitchHelper;
|
||||
|
||||
/// Modes that distinguish a powered-off state from one or more powered-on states, so
|
||||
/// This is a helper trait required to make [SwitchAndModeHelper] generic.
|
||||
///
|
||||
/// It allows distinguish a powered-off state from one or more powered-on states, so
|
||||
/// [`SwitchAndModeHelper`] knows which way to drive the switch for a given target mode.
|
||||
pub trait PowerSwitchedMode: Copy + PartialEq {
|
||||
const OFF: Self;
|
||||
fn requires_power(&self) -> bool;
|
||||
}
|
||||
|
||||
impl PowerSwitchedMode for types::DeviceMode {
|
||||
const OFF: Self = types::DeviceMode::Off;
|
||||
|
||||
fn requires_power(&self) -> bool {
|
||||
*self != types::DeviceMode::Off
|
||||
}
|
||||
@@ -24,13 +29,36 @@ enum SwitchTransitionState {
|
||||
Done,
|
||||
}
|
||||
|
||||
/// Outcome of a single power switch transition.
|
||||
enum SwitchOutcome {
|
||||
Reached(Option<satrs::spacepackets::CcsdsPacketIdAndPsc>),
|
||||
Failed(Option<satrs::spacepackets::CcsdsPacketIdAndPsc>),
|
||||
}
|
||||
|
||||
/// Dedicated states for power cycling a device.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum PowerCycleState<Mode> {
|
||||
Idle,
|
||||
SwitchingOff { restore_mode: Mode },
|
||||
WaitingOff { restore_mode: Mode, since: Instant },
|
||||
SwitchingOn { restore_mode: Mode },
|
||||
}
|
||||
|
||||
/// Outcome of a pending mode transition, once [`SwitchAndModeHelper::handle_mode_transition`]
|
||||
/// has driven it to completion. Carries back whichever TC commanded the transition, if any, so
|
||||
/// the caller can reply to it -- what that reply looks like is handler-specific, so this stays
|
||||
/// out of the helper.
|
||||
pub enum ModeTransitionEvent {
|
||||
pub enum ModeTransitionEvent<Mode> {
|
||||
/// The target mode was reached.
|
||||
Reached(Option<satrs::spacepackets::CcsdsPacketIdAndPsc>),
|
||||
/// The target mode could not be reached.
|
||||
Failed(Option<satrs::spacepackets::CcsdsPacketIdAndPsc>),
|
||||
/// The power cycle completed and the mode before the power cycle was restored.
|
||||
PowerCycleDone,
|
||||
/// Power switching failed during the power cycle. The power cycle is not hidden anymore,
|
||||
/// so [SwitchAndModeHelper::reported_mode] returns the actual mode again. `restore_mode` is
|
||||
/// the mode the power cycle should have restored, which can be used to retry it.
|
||||
PowerCycleFailed { restore_mode: Mode },
|
||||
}
|
||||
|
||||
/// Drives the on/off power-switch commanding state machine (Idle -> PowerSwitching -> Done)
|
||||
@@ -43,6 +71,8 @@ pub struct SwitchAndModeHelper<Mode: PowerSwitchedMode> {
|
||||
mode_helper: satrs_example::ModeHelper<Mode, SwitchTransitionState>,
|
||||
switch_helper: PowerSwitchHelper,
|
||||
switch_id: SwitchId,
|
||||
power_cycle_state: PowerCycleState<Mode>,
|
||||
power_cycle_off_duration: Duration,
|
||||
}
|
||||
|
||||
impl<Mode: PowerSwitchedMode> SwitchAndModeHelper<Mode> {
|
||||
@@ -56,6 +86,8 @@ impl<Mode: PowerSwitchedMode> SwitchAndModeHelper<Mode> {
|
||||
mode_helper: satrs_example::ModeHelper::new(init_mode, timeout),
|
||||
switch_helper,
|
||||
switch_id,
|
||||
power_cycle_state: PowerCycleState::Idle,
|
||||
power_cycle_off_duration: Duration::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,16 +101,109 @@ impl<Mode: PowerSwitchedMode> SwitchAndModeHelper<Mode> {
|
||||
self.mode_helper.target
|
||||
}
|
||||
|
||||
/// Mode which should be reported to other components. A power cycle is hidden from them,
|
||||
/// so this is the mode which is restored after the power cycle while one is active.
|
||||
pub fn reported_mode(&self) -> Mode {
|
||||
match self.power_cycle_state {
|
||||
PowerCycleState::SwitchingOff { restore_mode }
|
||||
| PowerCycleState::WaitingOff { restore_mode, .. }
|
||||
| PowerCycleState::SwitchingOn { restore_mode } => restore_mode,
|
||||
PowerCycleState::Idle => self.mode(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn power_cycle_active(&self) -> bool {
|
||||
self.power_cycle_state != PowerCycleState::Idle
|
||||
}
|
||||
|
||||
/// Starts a new transition, aborting a running power cycle.
|
||||
pub fn start_transition(
|
||||
&mut self,
|
||||
target_mode: Mode,
|
||||
tc_commander: Option<satrs::spacepackets::CcsdsPacketIdAndPsc>,
|
||||
) {
|
||||
self.power_cycle_state = PowerCycleState::Idle;
|
||||
self.start_transition_internal(target_mode, tc_commander);
|
||||
}
|
||||
|
||||
/// Switches the device off, keeps it off for `off_duration` and then switches it to
|
||||
/// `restore_mode`. Reaching the intermediate off mode does not generate an event.
|
||||
pub fn start_power_cycle(&mut self, restore_mode: Mode, off_duration: Duration) {
|
||||
self.power_cycle_state = PowerCycleState::SwitchingOff { restore_mode };
|
||||
self.power_cycle_off_duration = off_duration;
|
||||
self.start_transition_internal(Mode::OFF, None);
|
||||
}
|
||||
|
||||
fn start_transition_internal(
|
||||
&mut self,
|
||||
target_mode: Mode,
|
||||
tc_commander: Option<satrs::spacepackets::CcsdsPacketIdAndPsc>,
|
||||
) {
|
||||
self.mode_helper.tc_commander = tc_commander;
|
||||
self.mode_helper.start(target_mode);
|
||||
}
|
||||
|
||||
pub fn handle_mode_transition(&mut self) -> Option<ModeTransitionEvent> {
|
||||
/// This is the main API that the periodic handler of a device handler should call.
|
||||
///
|
||||
/// It handles the switch commanding and returns relevant events.
|
||||
pub fn handle_mode_transition(&mut self) -> Option<ModeTransitionEvent<Mode>> {
|
||||
// The most probable case: Nothing to do.
|
||||
if self.target().is_none() && !self.power_cycle_active() {
|
||||
return None;
|
||||
}
|
||||
// Handle this as an extra step so the switch transition after this can proceed.
|
||||
self.handle_waiting_for_off_when_power_cycling();
|
||||
// Core logic: Command the switches, check whether target switch state was reached.
|
||||
// Note the ?: if a switch transition is on-going, we might do an early return.
|
||||
let outcome = self.handle_switch_transition()?;
|
||||
// Regular mode transition without power cycling.
|
||||
if self.power_cycle_state == PowerCycleState::Idle {
|
||||
return Some(match outcome {
|
||||
SwitchOutcome::Reached(tc_commander) => ModeTransitionEvent::Reached(tc_commander),
|
||||
SwitchOutcome::Failed(tc_commander) => ModeTransitionEvent::Failed(tc_commander),
|
||||
});
|
||||
}
|
||||
// Power cycling, where a bit more logic is required.
|
||||
// Handle the error case first.
|
||||
if let SwitchOutcome::Failed(_) = outcome {
|
||||
let restore_mode = self.reported_mode();
|
||||
self.power_cycle_state = PowerCycleState::Idle;
|
||||
return Some(ModeTransitionEvent::PowerCycleFailed { restore_mode });
|
||||
}
|
||||
// At this point: The switching was succesfull, so we only match on the
|
||||
// power cycle state.
|
||||
match self.power_cycle_state {
|
||||
// No switching going on for thse cases.
|
||||
PowerCycleState::Idle | PowerCycleState::WaitingOff { .. } => None,
|
||||
PowerCycleState::SwitchingOff { restore_mode } => {
|
||||
self.power_cycle_state = PowerCycleState::WaitingOff {
|
||||
restore_mode,
|
||||
since: Instant::now(),
|
||||
};
|
||||
None
|
||||
}
|
||||
PowerCycleState::SwitchingOn { .. } => {
|
||||
// Power is back and we are done.
|
||||
self.power_cycle_state = PowerCycleState::Idle;
|
||||
Some(ModeTransitionEvent::PowerCycleDone)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_waiting_for_off_when_power_cycling(&mut self) {
|
||||
if let PowerCycleState::WaitingOff {
|
||||
restore_mode,
|
||||
since,
|
||||
} = self.power_cycle_state
|
||||
&& since.elapsed() >= self.power_cycle_off_duration
|
||||
{
|
||||
self.power_cycle_state = PowerCycleState::SwitchingOn { restore_mode };
|
||||
self.start_transition_internal(restore_mode, None);
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_switch_transition(&mut self) -> Option<SwitchOutcome> {
|
||||
let target_mode = self.mode_helper.target?;
|
||||
let switch_target_on = target_mode.requires_power();
|
||||
if self.mode_helper.transition_state == SwitchTransitionState::Idle {
|
||||
@@ -101,12 +226,279 @@ impl<Mode: PowerSwitchedMode> SwitchAndModeHelper<Mode> {
|
||||
log::info!("switch is {}", if switch_target_on { "on" } else { "off" });
|
||||
self.mode_helper.transition_state = SwitchTransitionState::Done;
|
||||
} else if self.mode_helper.timed_out() {
|
||||
return Some(ModeTransitionEvent::Failed(self.mode_helper.finish(false)));
|
||||
return Some(SwitchOutcome::Failed(self.mode_helper.finish(false)));
|
||||
}
|
||||
}
|
||||
if self.mode_helper.transition_state == SwitchTransitionState::Done {
|
||||
return Some(ModeTransitionEvent::Reached(self.mode_helper.finish(true)));
|
||||
return Some(SwitchOutcome::Reached(self.mode_helper.finish(true)));
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{Arc, Mutex, mpsc};
|
||||
|
||||
use arbitrary_int::u11;
|
||||
use satrs::spacepackets::{CcsdsPacketIdAndPsc, SpacePacketHeader};
|
||||
use types::{
|
||||
DeviceMode,
|
||||
pcdu::{SwitchRequest, SwitchState, SwitchStateBinary},
|
||||
};
|
||||
|
||||
use crate::eps::pcdu::{SharedSwitchSet, SwitchMap, SwitchSet};
|
||||
|
||||
use super::*;
|
||||
|
||||
const TIMEOUT: Duration = Duration::from_millis(50);
|
||||
|
||||
struct Testbench {
|
||||
helper: SwitchAndModeHelper<DeviceMode>,
|
||||
switch_rx: mpsc::Receiver<SwitchRequest>,
|
||||
shared_switch_set: SharedSwitchSet,
|
||||
}
|
||||
|
||||
impl Testbench {
|
||||
fn new() -> Self {
|
||||
let (switch_tx, switch_rx) = mpsc::sync_channel(10);
|
||||
let mut switch_map = SwitchMap::new();
|
||||
switch_map.insert(SwitchId::Mgm0, SwitchState::Off);
|
||||
let shared_switch_set: SharedSwitchSet =
|
||||
Arc::new(Mutex::new(SwitchSet::new(switch_map)));
|
||||
Self {
|
||||
helper: SwitchAndModeHelper::new(
|
||||
DeviceMode::Off,
|
||||
TIMEOUT,
|
||||
PowerSwitchHelper::new(switch_tx, shared_switch_set.clone()),
|
||||
SwitchId::Mgm0,
|
||||
),
|
||||
switch_rx,
|
||||
shared_switch_set,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_switch_state(&self, state: SwitchState) {
|
||||
self.shared_switch_set
|
||||
.lock()
|
||||
.unwrap()
|
||||
.set_switch_state(SwitchId::Mgm0, state);
|
||||
}
|
||||
|
||||
fn switch_requests(&self) -> Vec<SwitchStateBinary> {
|
||||
self.switch_rx
|
||||
.try_iter()
|
||||
.map(|req| req.target_state)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Drives a transition to `Normal` to completion.
|
||||
fn switch_to_normal(&mut self) {
|
||||
self.helper.start_transition(DeviceMode::Normal, None);
|
||||
self.set_switch_state(SwitchState::On);
|
||||
assert!(matches!(
|
||||
self.helper.handle_mode_transition(),
|
||||
Some(ModeTransitionEvent::Reached(None))
|
||||
));
|
||||
self.switch_requests();
|
||||
}
|
||||
|
||||
/// Starts a power cycle from `Normal` and drives it until the device is off.
|
||||
fn power_cycle_until_off(&mut self, off_duration: Duration) {
|
||||
self.switch_to_normal();
|
||||
self.helper
|
||||
.start_power_cycle(DeviceMode::Normal, off_duration);
|
||||
assert!(self.helper.handle_mode_transition().is_none());
|
||||
assert_eq!(self.switch_requests(), [SwitchStateBinary::Off]);
|
||||
self.set_switch_state(SwitchState::Off);
|
||||
assert!(self.helper.handle_mode_transition().is_none());
|
||||
assert_eq!(self.helper.mode(), DeviceMode::Off);
|
||||
}
|
||||
}
|
||||
|
||||
fn tc_id() -> CcsdsPacketIdAndPsc {
|
||||
CcsdsPacketIdAndPsc::new_from_ccsds_packet(&SpacePacketHeader::new_from_apid(u11::new(1)))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_transition() {
|
||||
let mut tb = Testbench::new();
|
||||
assert_eq!(tb.helper.mode(), DeviceMode::Off);
|
||||
assert_eq!(tb.helper.target(), None);
|
||||
assert!(tb.helper.handle_mode_transition().is_none());
|
||||
assert!(tb.switch_requests().is_empty());
|
||||
assert!(!tb.helper.power_cycle_active());
|
||||
assert_eq!(tb.helper.reported_mode(), DeviceMode::Off);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_switch_on() {
|
||||
let mut tb = Testbench::new();
|
||||
tb.helper
|
||||
.start_transition(DeviceMode::Normal, Some(tc_id()));
|
||||
assert!(tb.helper.handle_mode_transition().is_none());
|
||||
assert_eq!(tb.switch_requests(), [SwitchStateBinary::On]);
|
||||
assert_eq!(tb.helper.mode(), DeviceMode::Off);
|
||||
assert_eq!(tb.helper.target(), Some(DeviceMode::Normal));
|
||||
|
||||
tb.set_switch_state(SwitchState::On);
|
||||
match tb.helper.handle_mode_transition() {
|
||||
Some(ModeTransitionEvent::Reached(Some(id))) => assert_eq!(id, tc_id()),
|
||||
_ => panic!("expected mode reached event with TC commander"),
|
||||
}
|
||||
assert_eq!(tb.helper.mode(), DeviceMode::Normal);
|
||||
assert_eq!(tb.helper.target(), None);
|
||||
assert!(tb.switch_requests().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_switch_off() {
|
||||
let mut tb = Testbench::new();
|
||||
tb.switch_to_normal();
|
||||
tb.helper.start_transition(DeviceMode::Off, None);
|
||||
assert!(tb.helper.handle_mode_transition().is_none());
|
||||
assert_eq!(tb.switch_requests(), [SwitchStateBinary::Off]);
|
||||
tb.set_switch_state(SwitchState::Off);
|
||||
assert!(matches!(
|
||||
tb.helper.handle_mode_transition(),
|
||||
Some(ModeTransitionEvent::Reached(None))
|
||||
));
|
||||
assert_eq!(tb.helper.mode(), DeviceMode::Off);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_switch_already_in_target_state() {
|
||||
let mut tb = Testbench::new();
|
||||
tb.set_switch_state(SwitchState::On);
|
||||
tb.helper.start_transition(DeviceMode::On, None);
|
||||
assert!(matches!(
|
||||
tb.helper.handle_mode_transition(),
|
||||
Some(ModeTransitionEvent::Reached(None))
|
||||
));
|
||||
// The switch command is still sent.
|
||||
assert_eq!(tb.switch_requests(), [SwitchStateBinary::On]);
|
||||
assert_eq!(tb.helper.mode(), DeviceMode::On);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_switch_timeout() {
|
||||
let mut tb = Testbench::new();
|
||||
tb.helper
|
||||
.start_transition(DeviceMode::Normal, Some(tc_id()));
|
||||
assert!(tb.helper.handle_mode_transition().is_none());
|
||||
std::thread::sleep(TIMEOUT);
|
||||
match tb.helper.handle_mode_transition() {
|
||||
Some(ModeTransitionEvent::Failed(Some(id))) => assert_eq!(id, tc_id()),
|
||||
_ => panic!("expected mode failed event with TC commander"),
|
||||
}
|
||||
assert_eq!(tb.helper.mode(), DeviceMode::Off);
|
||||
assert_eq!(tb.helper.target(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_power_cycle() {
|
||||
let mut tb = Testbench::new();
|
||||
tb.power_cycle_until_off(Duration::ZERO);
|
||||
assert!(tb.helper.power_cycle_active());
|
||||
|
||||
// The off duration elapsed, so switching on starts right away.
|
||||
assert!(tb.helper.handle_mode_transition().is_none());
|
||||
assert_eq!(tb.switch_requests(), [SwitchStateBinary::On]);
|
||||
assert_eq!(tb.helper.target(), Some(DeviceMode::Normal));
|
||||
tb.set_switch_state(SwitchState::On);
|
||||
assert!(matches!(
|
||||
tb.helper.handle_mode_transition(),
|
||||
Some(ModeTransitionEvent::PowerCycleDone)
|
||||
));
|
||||
assert_eq!(tb.helper.mode(), DeviceMode::Normal);
|
||||
assert!(!tb.helper.power_cycle_active());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_power_cycle_reports_restored_mode() {
|
||||
let mut tb = Testbench::new();
|
||||
tb.switch_to_normal();
|
||||
tb.helper
|
||||
.start_power_cycle(DeviceMode::Normal, Duration::from_secs(60));
|
||||
assert_eq!(tb.helper.reported_mode(), DeviceMode::Normal);
|
||||
tb.helper.handle_mode_transition();
|
||||
tb.set_switch_state(SwitchState::Off);
|
||||
tb.helper.handle_mode_transition();
|
||||
assert_eq!(tb.helper.mode(), DeviceMode::Off);
|
||||
assert_eq!(tb.helper.reported_mode(), DeviceMode::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_power_cycle_reports_restored_mode_while_switching_on() {
|
||||
let mut tb = Testbench::new();
|
||||
tb.power_cycle_until_off(Duration::ZERO);
|
||||
assert!(tb.helper.handle_mode_transition().is_none());
|
||||
assert_eq!(tb.helper.target(), Some(DeviceMode::Normal));
|
||||
assert_eq!(tb.helper.mode(), DeviceMode::Off);
|
||||
assert_eq!(tb.helper.reported_mode(), DeviceMode::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_power_cycle_waits_off_duration() {
|
||||
let mut tb = Testbench::new();
|
||||
tb.power_cycle_until_off(Duration::from_secs(60));
|
||||
for _ in 0..3 {
|
||||
assert!(tb.helper.handle_mode_transition().is_none());
|
||||
}
|
||||
assert!(tb.switch_requests().is_empty());
|
||||
assert_eq!(tb.helper.target(), None);
|
||||
assert!(tb.helper.power_cycle_active());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_power_cycle_switch_off_timeout() {
|
||||
let mut tb = Testbench::new();
|
||||
tb.switch_to_normal();
|
||||
tb.helper
|
||||
.start_power_cycle(DeviceMode::Normal, Duration::ZERO);
|
||||
assert!(tb.helper.handle_mode_transition().is_none());
|
||||
std::thread::sleep(TIMEOUT);
|
||||
assert!(matches!(
|
||||
tb.helper.handle_mode_transition(),
|
||||
Some(ModeTransitionEvent::PowerCycleFailed {
|
||||
restore_mode: DeviceMode::Normal
|
||||
})
|
||||
));
|
||||
assert_eq!(tb.helper.mode(), DeviceMode::Normal);
|
||||
assert!(!tb.helper.power_cycle_active());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_power_cycle_switch_on_timeout() {
|
||||
let mut tb = Testbench::new();
|
||||
tb.power_cycle_until_off(Duration::ZERO);
|
||||
assert!(tb.helper.handle_mode_transition().is_none());
|
||||
std::thread::sleep(TIMEOUT);
|
||||
assert!(matches!(
|
||||
tb.helper.handle_mode_transition(),
|
||||
Some(ModeTransitionEvent::PowerCycleFailed {
|
||||
restore_mode: DeviceMode::Normal
|
||||
})
|
||||
));
|
||||
assert_eq!(tb.helper.mode(), DeviceMode::Off);
|
||||
assert!(!tb.helper.power_cycle_active());
|
||||
// The failed power cycle is not hidden anymore.
|
||||
assert_eq!(tb.helper.reported_mode(), DeviceMode::Off);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_transition_aborts_power_cycle() {
|
||||
let mut tb = Testbench::new();
|
||||
tb.power_cycle_until_off(Duration::from_secs(60));
|
||||
tb.helper.start_transition(DeviceMode::On, Some(tc_id()));
|
||||
assert!(!tb.helper.power_cycle_active());
|
||||
assert_eq!(tb.helper.reported_mode(), DeviceMode::Off);
|
||||
tb.set_switch_state(SwitchState::On);
|
||||
// A regular transition event instead of a power cycle event.
|
||||
assert!(matches!(
|
||||
tb.helper.handle_mode_transition(),
|
||||
Some(ModeTransitionEvent::Reached(Some(_)))
|
||||
));
|
||||
assert_eq!(tb.helper.mode(), DeviceMode::On);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user