Pass-N: detect drive fast-fail wedge and abort instead of grinding
The 2026-07-01 overnight rip ground a wedged BU40N for 28 minutes at 0 B/s. Root cause: the handler chain's read_span split errors only into Transport (bus gone) vs Bad (everything else), so the drive's fast-fail wedge sense (ILLEGAL REQUEST 0x05/0x24 — it rejects every CDB in <100ms without attempting recovery) was treated as an ordinary bad sector. The chain hopped to the next section forever, and the existing wedge detector in read_error.rs never saw it because the chain reads through recovery_read directly. Add a pass-level wedge streak: read_span counts consecutive wedge-family (Hardware/IllegalRequest) senses; at WEDGE_ABORT_STREAK (16) it escalates the read to Transport, which every handler already propagates as TransportFault — aborting the whole pass and setting wedged_exit so the caller spin-cycles. The streak is carried across sections via PatchCtx (seeded into and read back from each per-section HandlerCtx) so a wedge is caught even when every bad sub-range is smaller than the streak. Any Good or non-wedge (medium-error) read resets it, so scattered bad sectors on real media never trip it. New fixture test: a wholly-wedged 1000-sector section aborts in <100 reads, not 1000.
This commit is contained in:
+13
-3
@@ -751,6 +751,10 @@ struct PatchCtx<'a, 'o> {
|
|||||||
/// coordinator runs the winners first and lets duds fall back. Reset per
|
/// coordinator runs the winners first and lets duds fall back. Reset per
|
||||||
/// pass (ephemeral, no persistence).
|
/// pass (ephemeral, no persistence).
|
||||||
scoreboard: HandlerScoreboard,
|
scoreboard: HandlerScoreboard,
|
||||||
|
/// Consecutive wedge-family senses across the WHOLE pass. Seeded into each
|
||||||
|
/// per-section `HandlerCtx` and read back after, so a drive fast-fail wedge is
|
||||||
|
/// detected even when every bad sub-range is smaller than the abort streak.
|
||||||
|
wedge_streak: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PatchCtx<'_, '_> {
|
impl PatchCtx<'_, '_> {
|
||||||
@@ -883,7 +887,7 @@ impl PatchCtx<'_, '_> {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let bad_before = bad.total_len();
|
let bad_before = bad.total_len();
|
||||||
let outcome = {
|
let (outcome, wedge_after) = {
|
||||||
// Progress heartbeat: a throttled closure that pushes a fresh
|
// Progress heartbeat: a throttled closure that pushes a fresh
|
||||||
// snapshot to the reporter as recovery happens (called from every
|
// snapshot to the reporter as recovery happens (called from every
|
||||||
// read via `HandlerCtx::progress`), so the bar and speed move DURING
|
// read via `HandlerCtx::progress`), so the bar and speed move DURING
|
||||||
@@ -913,11 +917,16 @@ impl PatchCtx<'_, '_> {
|
|||||||
decrypt_is_aacs: self.decrypt_is_aacs,
|
decrypt_is_aacs: self.decrypt_is_aacs,
|
||||||
tick: Some(&mut tick),
|
tick: Some(&mut tick),
|
||||||
unproductive: 0,
|
unproductive: 0,
|
||||||
|
// Carry the pass-level wedge streak in so a fast-fail wedge is
|
||||||
|
// caught across many small sections, not reset each one.
|
||||||
|
wedge_streak: self.wedge_streak,
|
||||||
};
|
};
|
||||||
run_handlers(&mut ctx, &mut handlers, bad, &mut self.scoreboard, |_bad| {
|
let o = run_handlers(&mut ctx, &mut handlers, bad, &mut self.scoreboard, |_bad| {
|
||||||
now_ptr() + std::time::Duration::from_secs(PER_HANDLER_BUDGET_SECS)
|
now_ptr() + std::time::Duration::from_secs(PER_HANDLER_BUDGET_SECS)
|
||||||
})
|
});
|
||||||
|
(o, ctx.wedge_streak)
|
||||||
};
|
};
|
||||||
|
self.wedge_streak = wedge_after;
|
||||||
|
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
target: "freemkv::disc",
|
target: "freemkv::disc",
|
||||||
@@ -1235,6 +1244,7 @@ impl Disc {
|
|||||||
decrypt_is_aacs,
|
decrypt_is_aacs,
|
||||||
state: PatchLoopState::new(bytes_good_before, total_bytes, initial_batch, work_total),
|
state: PatchLoopState::new(bytes_good_before, total_bytes, initial_batch, work_total),
|
||||||
scoreboard: HandlerScoreboard::default(),
|
scoreboard: HandlerScoreboard::default(),
|
||||||
|
wedge_streak: 0,
|
||||||
};
|
};
|
||||||
ctx.run(&bad_ranges)?;
|
ctx.run(&bad_ranges)?;
|
||||||
ctx.scoreboard.log();
|
ctx.scoreboard.log();
|
||||||
|
|||||||
+134
-2
@@ -33,6 +33,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
|
|||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
use super::patch::{SubRanges, recovery_read};
|
use super::patch::{SubRanges, recovery_read};
|
||||||
|
use super::read_error::SenseFamily;
|
||||||
use crate::sector::SectorSource;
|
use crate::sector::SectorSource;
|
||||||
|
|
||||||
/// One 2048-byte sector.
|
/// One 2048-byte sector.
|
||||||
@@ -55,6 +56,20 @@ const JUMP_AFTER_FAILS: u32 = 2;
|
|||||||
/// "60 s of 0 B/s" stall into a fast hand-off.
|
/// "60 s of 0 B/s" stall into a fast hand-off.
|
||||||
const UNPRODUCTIVE_YIELD: u32 = 4;
|
const UNPRODUCTIVE_YIELD: u32 = 4;
|
||||||
|
|
||||||
|
/// Wedge abort: after this many CONSECUTIVE wedge-family senses (Hardware /
|
||||||
|
/// IllegalRequest — the BU40N firmware's fast-fail state, where it rejects every
|
||||||
|
/// CDB in <100 ms without attempting recovery) the drive is wedged. `read_span`
|
||||||
|
/// escalates the read to `Transport`, which every handler propagates as
|
||||||
|
/// `TransportFault` → the whole pass aborts and the caller spin-cycles the drive
|
||||||
|
/// instead of hammering all remaining sections (which only deepens the wedge).
|
||||||
|
/// Any Good read or non-wedge (medium-error) read resets the streak, so only a
|
||||||
|
/// sustained fast-fail run — never scattered bad sectors on real media — trips
|
||||||
|
/// it. Counted at the PASS level (persisted across sections) so a wedge is caught
|
||||||
|
/// even when every bad sub-range is smaller than the streak. Learned the hard way
|
||||||
|
/// (2026-07-01): the handler chain ground a wedged drive for 28 min at 0 B/s
|
||||||
|
/// because a fast-fail sense was classified as an ordinary bad sector.
|
||||||
|
const WEDGE_ABORT_STREAK: u32 = 16;
|
||||||
|
|
||||||
/// Where a handler left the section after its bounded attempt.
|
/// Where a handler left the section after its bounded attempt.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub(super) enum HandlerOutcome {
|
pub(super) enum HandlerOutcome {
|
||||||
@@ -103,6 +118,11 @@ pub(super) struct HandlerCtx<'a> {
|
|||||||
/// it reaches [`UNPRODUCTIVE_YIELD`] the handler should yield to the next one
|
/// it reaches [`UNPRODUCTIVE_YIELD`] the handler should yield to the next one
|
||||||
/// (see [`HandlerCtx::stalled`]). Reset to 0 before each handler runs.
|
/// (see [`HandlerCtx::stalled`]). Reset to 0 before each handler runs.
|
||||||
pub unproductive: u32,
|
pub unproductive: u32,
|
||||||
|
/// Consecutive wedge-family senses (Hardware / IllegalRequest), updated by
|
||||||
|
/// [`read_span`]. At [`WEDGE_ABORT_STREAK`] the drive is wedged and the read
|
||||||
|
/// escalates to `Transport`. Seeded from and read back into the pass-level
|
||||||
|
/// counter so the streak spans sections; a Good or non-wedge read resets it.
|
||||||
|
pub wedge_streak: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HandlerCtx<'_> {
|
impl HandlerCtx<'_> {
|
||||||
@@ -170,12 +190,37 @@ fn read_span(
|
|||||||
ReadHit::Good
|
ReadHit::Good
|
||||||
}
|
}
|
||||||
Err(e) if e.is_scsi_transport_failure() => ReadHit::Transport,
|
Err(e) if e.is_scsi_transport_failure() => ReadHit::Transport,
|
||||||
Err(_) => ReadHit::Bad,
|
Err(e) => {
|
||||||
|
// Wedge watch: a Hardware / IllegalRequest sense is the drive's
|
||||||
|
// fast-fail signature. Count consecutive ones; a sustained run means
|
||||||
|
// the firmware wedged (rejecting every CDB), so escalate to Transport
|
||||||
|
// and let the pass abort + spin-cycle rather than grind on. A medium
|
||||||
|
// error or any success below resets the streak — real bad sectors
|
||||||
|
// never trip it.
|
||||||
|
let wedge = e
|
||||||
|
.scsi_sense()
|
||||||
|
.map(|s| SenseFamily::from_sense_key(s.sense_key).is_wedge_family())
|
||||||
|
.unwrap_or(false);
|
||||||
|
if wedge {
|
||||||
|
ctx.wedge_streak = ctx.wedge_streak.saturating_add(1);
|
||||||
|
if ctx.wedge_streak >= WEDGE_ABORT_STREAK {
|
||||||
|
ReadHit::Transport
|
||||||
|
} else {
|
||||||
|
ReadHit::Bad
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
ctx.wedge_streak = 0;
|
||||||
|
ReadHit::Bad
|
||||||
|
}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
// Track the dead streak for the early-yield hand-off: a recovering read
|
// Track the dead streak for the early-yield hand-off: a recovering read
|
||||||
// resets it, a fruitless one advances it toward UNPRODUCTIVE_YIELD.
|
// resets it, a fruitless one advances it toward UNPRODUCTIVE_YIELD.
|
||||||
match hit {
|
match hit {
|
||||||
ReadHit::Good => ctx.unproductive = 0,
|
ReadHit::Good => {
|
||||||
|
ctx.unproductive = 0;
|
||||||
|
ctx.wedge_streak = 0;
|
||||||
|
}
|
||||||
_ => ctx.unproductive = ctx.unproductive.saturating_add(1),
|
_ => ctx.unproductive = ctx.unproductive.saturating_add(1),
|
||||||
}
|
}
|
||||||
// Heartbeat after every read (the tick closure throttles to ~250 ms) so the
|
// Heartbeat after every read (the tick closure throttles to ~250 ms) so the
|
||||||
@@ -618,6 +663,10 @@ mod tests {
|
|||||||
/// staying `Send`.
|
/// staying `Send`.
|
||||||
struct FakeDisc {
|
struct FakeDisc {
|
||||||
dead: HashSet<u32>,
|
dead: HashSet<u32>,
|
||||||
|
/// LBAs that return a wedge-family sense (IllegalRequest) — the drive
|
||||||
|
/// fast-fail state, distinct from an ordinary dead sector (which carries
|
||||||
|
/// no sense). Used to exercise wedge detection.
|
||||||
|
wedge: HashSet<u32>,
|
||||||
transport_at: Option<u32>,
|
transport_at: Option<u32>,
|
||||||
clock_nanos: Arc<AtomicU64>,
|
clock_nanos: Arc<AtomicU64>,
|
||||||
per_read: Duration,
|
per_read: Duration,
|
||||||
@@ -645,6 +694,21 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for l in lba..lba + count as u32 {
|
for l in lba..lba + count as u32 {
|
||||||
|
if self.wedge.contains(&l) {
|
||||||
|
// Fast-fail wedge sense: ILLEGAL REQUEST / INVALID FIELD IN
|
||||||
|
// CDB (0x05/0x24), the real BU40N wedge signature. Non-
|
||||||
|
// transport status so it isn't caught as a bus fault, but
|
||||||
|
// carries sense so the wedge classifier sees it.
|
||||||
|
return Err(Error::ScsiError {
|
||||||
|
opcode: crate::scsi::SCSI_READ_10,
|
||||||
|
status: 0x02,
|
||||||
|
sense: Some(crate::scsi::ScsiSense {
|
||||||
|
sense_key: crate::scsi::SENSE_KEY_ILLEGAL_REQUEST,
|
||||||
|
asc: 0x24,
|
||||||
|
ascq: 0x00,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
}
|
||||||
if self.dead.contains(&l) {
|
if self.dead.contains(&l) {
|
||||||
// Non-transport bad-sector error (CHECK CONDITION, 0x02).
|
// Non-transport bad-sector error (CHECK CONDITION, 0x02).
|
||||||
return Err(Error::DiscRead {
|
return Err(Error::DiscRead {
|
||||||
@@ -686,6 +750,7 @@ mod tests {
|
|||||||
let reads = Arc::new(AtomicU64::new(0));
|
let reads = Arc::new(AtomicU64::new(0));
|
||||||
let disc = FakeDisc {
|
let disc = FakeDisc {
|
||||||
dead: dead.iter().copied().collect(),
|
dead: dead.iter().copied().collect(),
|
||||||
|
wedge: HashSet::new(),
|
||||||
transport_at,
|
transport_at,
|
||||||
clock_nanos: clock_nanos.clone(),
|
clock_nanos: clock_nanos.clone(),
|
||||||
per_read,
|
per_read,
|
||||||
@@ -737,6 +802,7 @@ mod tests {
|
|||||||
decrypt_is_aacs: false,
|
decrypt_is_aacs: false,
|
||||||
tick: None,
|
tick: None,
|
||||||
unproductive: 0,
|
unproductive: 0,
|
||||||
|
wedge_streak: 0,
|
||||||
};
|
};
|
||||||
let mut bad = SubRanges::from_section(0, 10 * SECTOR);
|
let mut bad = SubRanges::from_section(0, 10 * SECTOR);
|
||||||
let deadline = (ctx.now)() + Duration::from_secs(10);
|
let deadline = (ctx.now)() + Duration::from_secs(10);
|
||||||
@@ -785,6 +851,7 @@ mod tests {
|
|||||||
decrypt_is_aacs: false,
|
decrypt_is_aacs: false,
|
||||||
tick: None,
|
tick: None,
|
||||||
unproductive: 0,
|
unproductive: 0,
|
||||||
|
wedge_streak: 0,
|
||||||
};
|
};
|
||||||
let mut bad = SubRanges::from_section(0, 40 * SECTOR);
|
let mut bad = SubRanges::from_section(0, 40 * SECTOR);
|
||||||
let deadline = (ctx.now)() + Duration::from_secs(10);
|
let deadline = (ctx.now)() + Duration::from_secs(10);
|
||||||
@@ -822,6 +889,7 @@ mod tests {
|
|||||||
decrypt_is_aacs: false,
|
decrypt_is_aacs: false,
|
||||||
tick: None,
|
tick: None,
|
||||||
unproductive: 0,
|
unproductive: 0,
|
||||||
|
wedge_streak: 0,
|
||||||
};
|
};
|
||||||
let mut bad = SubRanges::from_section(0, 1000 * SECTOR);
|
let mut bad = SubRanges::from_section(0, 1000 * SECTOR);
|
||||||
let deadline = (ctx.now)() + Duration::from_secs(3);
|
let deadline = (ctx.now)() + Duration::from_secs(3);
|
||||||
@@ -857,6 +925,7 @@ mod tests {
|
|||||||
decrypt_is_aacs: false,
|
decrypt_is_aacs: false,
|
||||||
tick: None,
|
tick: None,
|
||||||
unproductive: 0,
|
unproductive: 0,
|
||||||
|
wedge_streak: 0,
|
||||||
};
|
};
|
||||||
let mut bad = SubRanges::from_section(0, 9 * SECTOR);
|
let mut bad = SubRanges::from_section(0, 9 * SECTOR);
|
||||||
let deadline = (ctx.now)() + Duration::from_secs(10);
|
let deadline = (ctx.now)() + Duration::from_secs(10);
|
||||||
@@ -893,6 +962,7 @@ mod tests {
|
|||||||
decrypt_is_aacs: false,
|
decrypt_is_aacs: false,
|
||||||
tick: None,
|
tick: None,
|
||||||
unproductive: 0,
|
unproductive: 0,
|
||||||
|
wedge_streak: 0,
|
||||||
};
|
};
|
||||||
let mut bad = SubRanges::from_section(0, 16 * SECTOR);
|
let mut bad = SubRanges::from_section(0, 16 * SECTOR);
|
||||||
let mut handlers: Vec<Box<dyn SectionHandler>> = vec![
|
let mut handlers: Vec<Box<dyn SectionHandler>> = vec![
|
||||||
@@ -934,6 +1004,7 @@ mod tests {
|
|||||||
decrypt_is_aacs: false,
|
decrypt_is_aacs: false,
|
||||||
tick: None,
|
tick: None,
|
||||||
unproductive: 0,
|
unproductive: 0,
|
||||||
|
wedge_streak: 0,
|
||||||
};
|
};
|
||||||
let mut bad = SubRanges::from_section(0, 64 * SECTOR);
|
let mut bad = SubRanges::from_section(0, 64 * SECTOR);
|
||||||
let mut handlers: Vec<Box<dyn SectionHandler>> = vec![Box::new(Linear {
|
let mut handlers: Vec<Box<dyn SectionHandler>> = vec![Box::new(Linear {
|
||||||
@@ -965,6 +1036,7 @@ mod tests {
|
|||||||
decrypt_is_aacs: false,
|
decrypt_is_aacs: false,
|
||||||
tick: None,
|
tick: None,
|
||||||
unproductive: 0,
|
unproductive: 0,
|
||||||
|
wedge_streak: 0,
|
||||||
};
|
};
|
||||||
// Single-sector batches so the transport LBA is hit directly.
|
// Single-sector batches so the transport LBA is hit directly.
|
||||||
let mut bad = SubRanges::from_section(0, 8 * SECTOR);
|
let mut bad = SubRanges::from_section(0, 8 * SECTOR);
|
||||||
@@ -977,6 +1049,65 @@ mod tests {
|
|||||||
assert_eq!(out, HandlerOutcome::TransportFault);
|
assert_eq!(out, HandlerOutcome::TransportFault);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn wedged_drive_aborts_fast_instead_of_grinding() {
|
||||||
|
// Regression for the 2026-07-01 incident: a fast-fail wedge (drive
|
||||||
|
// returns ILLEGAL REQUEST on every CDB) was classified as an ordinary
|
||||||
|
// bad sector, so the chain ground a dead drive for 28 min at 0 B/s.
|
||||||
|
// Now a sustained run of wedge-family senses escalates to TransportFault
|
||||||
|
// so the pass aborts and the caller spin-cycles. A big section (1000
|
||||||
|
// sectors) that is ENTIRELY wedged must bail after ~WEDGE_ABORT_STREAK
|
||||||
|
// reads, not after reading the whole thing.
|
||||||
|
let (h, disc) = Harness::build(&[], None, Duration::from_millis(1));
|
||||||
|
let mut disc = disc;
|
||||||
|
disc.wedge = (0..1000u32).collect();
|
||||||
|
let mut sink = RecordSink::default();
|
||||||
|
let now = h.now_fn();
|
||||||
|
let mut ctx = HandlerCtx {
|
||||||
|
reader: &mut disc,
|
||||||
|
sink: &mut sink,
|
||||||
|
now: &now,
|
||||||
|
halt: None,
|
||||||
|
decrypt_is_aacs: false,
|
||||||
|
tick: None,
|
||||||
|
unproductive: 0,
|
||||||
|
wedge_streak: 0,
|
||||||
|
};
|
||||||
|
let mut bad = SubRanges::from_section(0, 1000 * SECTOR);
|
||||||
|
// The full tier-0 chain: the wedge streak persists across handlers (only
|
||||||
|
// `unproductive` resets per handler), so it reaches the abort threshold
|
||||||
|
// even though each handler yields early on the dead streak.
|
||||||
|
let mut handlers: Vec<Box<dyn SectionHandler>> = vec![
|
||||||
|
Box::new(Bisect),
|
||||||
|
Box::new(Jump),
|
||||||
|
Box::new(Linear {
|
||||||
|
reverse: true,
|
||||||
|
fast: true,
|
||||||
|
}),
|
||||||
|
Box::new(Linear {
|
||||||
|
reverse: false,
|
||||||
|
fast: true,
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
let mut scoreboard = HandlerScoreboard::default();
|
||||||
|
let out = run_handlers(&mut ctx, &mut handlers, &mut bad, &mut scoreboard, |_| {
|
||||||
|
(h.now_fn())() + Duration::from_secs(60)
|
||||||
|
});
|
||||||
|
assert_eq!(
|
||||||
|
out,
|
||||||
|
HandlerOutcome::TransportFault,
|
||||||
|
"a wholly-wedged section must escalate to TransportFault"
|
||||||
|
);
|
||||||
|
// The whole point: it bailed after a short streak, not after grinding all
|
||||||
|
// 1000 sectors. Generous bound (handlers read in batches) but far below
|
||||||
|
// the section size.
|
||||||
|
assert!(
|
||||||
|
h.read_count() < 100,
|
||||||
|
"wedge must abort fast; did {} reads on a 1000-sector wedged section",
|
||||||
|
h.read_count()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn halt_token_returns_promptly() {
|
fn halt_token_returns_promptly() {
|
||||||
// Halt set before the call: the handler returns Halted on its first
|
// Halt set before the call: the handler returns Halted on its first
|
||||||
@@ -994,6 +1125,7 @@ mod tests {
|
|||||||
decrypt_is_aacs: false,
|
decrypt_is_aacs: false,
|
||||||
tick: None,
|
tick: None,
|
||||||
unproductive: 0,
|
unproductive: 0,
|
||||||
|
wedge_streak: 0,
|
||||||
};
|
};
|
||||||
let mut bad = SubRanges::from_section(0, 100 * SECTOR);
|
let mut bad = SubRanges::from_section(0, 100 * SECTOR);
|
||||||
let deadline = (ctx.now)() + Duration::from_secs(10);
|
let deadline = (ctx.now)() + Duration::from_secs(10);
|
||||||
|
|||||||
Reference in New Issue
Block a user