pq/libfreemkv
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pass-N: detect drive fast-fail wedge and abort instead of grinding

Browse Source 
The 2026-07-01 overnight rip ground a wedged BU40N for 28 minutes at
0 B/s. Root cause: the handler chain's read_span split errors only into
Transport (bus gone) vs Bad (everything else), so the drive's fast-fail
wedge sense (ILLEGAL REQUEST 0x05/0x24 — it rejects every CDB in <100ms
without attempting recovery) was treated as an ordinary bad sector. The
chain hopped to the next section forever, and the existing wedge detector
in read_error.rs never saw it because the chain reads through
recovery_read directly.

Add a pass-level wedge streak: read_span counts consecutive wedge-family
(Hardware/IllegalRequest) senses; at WEDGE_ABORT_STREAK (16) it escalates
the read to Transport, which every handler already propagates as
TransportFault — aborting the whole pass and setting wedged_exit so the
caller spin-cycles. The streak is carried across sections via PatchCtx
(seeded into and read back from each per-section HandlerCtx) so a wedge is
caught even when every bad sub-range is smaller than the streak. Any Good
or non-wedge (medium-error) read resets it, so scattered bad sectors on
real media never trip it. New fixture test: a wholly-wedged 1000-sector
section aborts in <100 reads, not 1000.
This commit is contained in:
Matthew Jackson
2026-07-01 08:02:52 -07:00
parent 8f6a92ccd4
commit fe14a2d5e5
2 changed files with 147 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/disc/patch.rs
+13 -3
View File
@@ -751,6 +751,10 @@ struct PatchCtx<'a, 'o> {
/// coordinator runs the winners first and lets duds fall back. Reset per
/// pass (ephemeral, no persistence).
scoreboard: HandlerScoreboard,
/// Consecutive wedge-family senses across the WHOLE pass. Seeded into each
/// per-section `HandlerCtx` and read back after, so a drive fast-fail wedge is
/// detected even when every bad sub-range is smaller than the abort streak.
wedge_streak: u32,
}
impl PatchCtx<'_, '_> {
@@ -883,7 +887,7 @@ impl PatchCtx<'_, '_> {
};
let bad_before = bad.total_len();
let outcome = {
let (outcome, wedge_after) = {
// Progress heartbeat: a throttled closure that pushes a fresh
// snapshot to the reporter as recovery happens (called from every
// read via `HandlerCtx::progress`), so the bar and speed move DURING
@@ -913,11 +917,16 @@ impl PatchCtx<'_, '_> {
decrypt_is_aacs: self.decrypt_is_aacs,
tick: Some(&mut tick),
unproductive: 0,
// Carry the pass-level wedge streak in so a fast-fail wedge is
// caught across many small sections, not reset each one.
wedge_streak: self.wedge_streak,
};
run_handlers(&mut ctx, &mut handlers, bad, &mut self.scoreboard, |_bad| {
let o = run_handlers(&mut ctx, &mut handlers, bad, &mut self.scoreboard, |_bad| {
now_ptr() + std::time::Duration::from_secs(PER_HANDLER_BUDGET_SECS)
})
});
(o, ctx.wedge_streak)
};
self.wedge_streak = wedge_after;
tracing::info!(
target: "freemkv::disc",
@@ -1235,6 +1244,7 @@ impl Disc {
decrypt_is_aacs,
state: PatchLoopState::new(bytes_good_before, total_bytes, initial_batch, work_total),
scoreboard: HandlerScoreboard::default(),
wedge_streak: 0,
};
ctx.run(&bad_ranges)?;
ctx.scoreboard.log();
src/disc/section_recover.rs
+134 -2
View File
@@ -33,6 +33,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Instant;
use super::patch::{SubRanges, recovery_read};
use super::read_error::SenseFamily;
use crate::sector::SectorSource;
/// One 2048-byte sector.
@@ -55,6 +56,20 @@ const JUMP_AFTER_FAILS: u32 = 2;
/// "60 s of 0 B/s" stall into a fast hand-off.
const UNPRODUCTIVE_YIELD: u32 = 4;
/// Wedge abort: after this many CONSECUTIVE wedge-family senses (Hardware /
/// IllegalRequest — the BU40N firmware's fast-fail state, where it rejects every
/// CDB in <100 ms without attempting recovery) the drive is wedged. `read_span`
/// escalates the read to `Transport`, which every handler propagates as
/// `TransportFault` → the whole pass aborts and the caller spin-cycles the drive
/// instead of hammering all remaining sections (which only deepens the wedge).
/// Any Good read or non-wedge (medium-error) read resets the streak, so only a
/// sustained fast-fail run — never scattered bad sectors on real media — trips
/// it. Counted at the PASS level (persisted across sections) so a wedge is caught
/// even when every bad sub-range is smaller than the streak. Learned the hard way
/// (2026-07-01): the handler chain ground a wedged drive for 28 min at 0 B/s
/// because a fast-fail sense was classified as an ordinary bad sector.
const WEDGE_ABORT_STREAK: u32 = 16;
/// Where a handler left the section after its bounded attempt.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum HandlerOutcome {
@@ -103,6 +118,11 @@ pub(super) struct HandlerCtx<'a> {
/// it reaches [`UNPRODUCTIVE_YIELD`] the handler should yield to the next one
/// (see [`HandlerCtx::stalled`]). Reset to 0 before each handler runs.
pub unproductive: u32,
/// Consecutive wedge-family senses (Hardware / IllegalRequest), updated by
/// [`read_span`]. At [`WEDGE_ABORT_STREAK`] the drive is wedged and the read
/// escalates to `Transport`. Seeded from and read back into the pass-level
/// counter so the streak spans sections; a Good or non-wedge read resets it.
pub wedge_streak: u32,
}
impl HandlerCtx<'_> {
@@ -170,12 +190,37 @@ fn read_span(
ReadHit::Good
}
Err(e) if e.is_scsi_transport_failure() => ReadHit::Transport,
Err(_) => ReadHit::Bad,
Err(e) => {
// Wedge watch: a Hardware / IllegalRequest sense is the drive's
// fast-fail signature. Count consecutive ones; a sustained run means
// the firmware wedged (rejecting every CDB), so escalate to Transport
// and let the pass abort + spin-cycle rather than grind on. A medium
// error or any success below resets the streak — real bad sectors
// never trip it.
let wedge = e
.scsi_sense()
.map(|s| SenseFamily::from_sense_key(s.sense_key).is_wedge_family())
.unwrap_or(false);
if wedge {
ctx.wedge_streak = ctx.wedge_streak.saturating_add(1);
if ctx.wedge_streak >= WEDGE_ABORT_STREAK {
ReadHit::Transport
} else {
ReadHit::Bad
}
} else {
ctx.wedge_streak = 0;
ReadHit::Bad
}
}
};
// Track the dead streak for the early-yield hand-off: a recovering read
// resets it, a fruitless one advances it toward UNPRODUCTIVE_YIELD.
match hit {
ReadHit::Good => ctx.unproductive = 0,
ReadHit::Good => {
ctx.unproductive = 0;
ctx.wedge_streak = 0;
}
_ => ctx.unproductive = ctx.unproductive.saturating_add(1),
}
// Heartbeat after every read (the tick closure throttles to ~250 ms) so the
@@ -618,6 +663,10 @@ mod tests {
/// staying `Send`.
struct FakeDisc {
dead: HashSet<u32>,
/// LBAs that return a wedge-family sense (IllegalRequest) — the drive
/// fast-fail state, distinct from an ordinary dead sector (which carries
/// no sense). Used to exercise wedge detection.
wedge: HashSet<u32>,
transport_at: Option<u32>,
clock_nanos: Arc<AtomicU64>,
per_read: Duration,
@@ -645,6 +694,21 @@ mod tests {
}
}
for l in lba..lba + count as u32 {
if self.wedge.contains(&l) {
// Fast-fail wedge sense: ILLEGAL REQUEST / INVALID FIELD IN
// CDB (0x05/0x24), the real BU40N wedge signature. Non-
// transport status so it isn't caught as a bus fault, but
// carries sense so the wedge classifier sees it.
return Err(Error::ScsiError {
opcode: crate::scsi::SCSI_READ_10,
status: 0x02,
sense: Some(crate::scsi::ScsiSense {
sense_key: crate::scsi::SENSE_KEY_ILLEGAL_REQUEST,
asc: 0x24,
ascq: 0x00,
}),
});
}
if self.dead.contains(&l) {
// Non-transport bad-sector error (CHECK CONDITION, 0x02).
return Err(Error::DiscRead {
@@ -686,6 +750,7 @@ mod tests {
let reads = Arc::new(AtomicU64::new(0));
let disc = FakeDisc {
dead: dead.iter().copied().collect(),
wedge: HashSet::new(),
transport_at,
clock_nanos: clock_nanos.clone(),
per_read,
@@ -737,6 +802,7 @@ mod tests {
decrypt_is_aacs: false,
tick: None,
unproductive: 0,
wedge_streak: 0,
};
let mut bad = SubRanges::from_section(0, 10 * SECTOR);
let deadline = (ctx.now)() + Duration::from_secs(10);
@@ -785,6 +851,7 @@ mod tests {
decrypt_is_aacs: false,
tick: None,
unproductive: 0,
wedge_streak: 0,
};
let mut bad = SubRanges::from_section(0, 40 * SECTOR);
let deadline = (ctx.now)() + Duration::from_secs(10);
@@ -822,6 +889,7 @@ mod tests {
decrypt_is_aacs: false,
tick: None,
unproductive: 0,
wedge_streak: 0,
};
let mut bad = SubRanges::from_section(0, 1000 * SECTOR);
let deadline = (ctx.now)() + Duration::from_secs(3);
@@ -857,6 +925,7 @@ mod tests {
decrypt_is_aacs: false,
tick: None,
unproductive: 0,
wedge_streak: 0,
};
let mut bad = SubRanges::from_section(0, 9 * SECTOR);
let deadline = (ctx.now)() + Duration::from_secs(10);
@@ -893,6 +962,7 @@ mod tests {
decrypt_is_aacs: false,
tick: None,
unproductive: 0,
wedge_streak: 0,
};
let mut bad = SubRanges::from_section(0, 16 * SECTOR);
let mut handlers: Vec<Box<dyn SectionHandler>> = vec![
@@ -934,6 +1004,7 @@ mod tests {
decrypt_is_aacs: false,
tick: None,
unproductive: 0,
wedge_streak: 0,
};
let mut bad = SubRanges::from_section(0, 64 * SECTOR);
let mut handlers: Vec<Box<dyn SectionHandler>> = vec![Box::new(Linear {
@@ -965,6 +1036,7 @@ mod tests {
decrypt_is_aacs: false,
tick: None,
unproductive: 0,
wedge_streak: 0,
};
// Single-sector batches so the transport LBA is hit directly.
let mut bad = SubRanges::from_section(0, 8 * SECTOR);
@@ -977,6 +1049,65 @@ mod tests {
assert_eq!(out, HandlerOutcome::TransportFault);
}
#[test]
fn wedged_drive_aborts_fast_instead_of_grinding() {
// Regression for the 2026-07-01 incident: a fast-fail wedge (drive
// returns ILLEGAL REQUEST on every CDB) was classified as an ordinary
// bad sector, so the chain ground a dead drive for 28 min at 0 B/s.
// Now a sustained run of wedge-family senses escalates to TransportFault
// so the pass aborts and the caller spin-cycles. A big section (1000
// sectors) that is ENTIRELY wedged must bail after ~WEDGE_ABORT_STREAK
// reads, not after reading the whole thing.
let (h, disc) = Harness::build(&[], None, Duration::from_millis(1));
let mut disc = disc;
disc.wedge = (0..1000u32).collect();
let mut sink = RecordSink::default();
let now = h.now_fn();
let mut ctx = HandlerCtx {
reader: &mut disc,
sink: &mut sink,
now: &now,
halt: None,
decrypt_is_aacs: false,
tick: None,
unproductive: 0,
wedge_streak: 0,
};
let mut bad = SubRanges::from_section(0, 1000 * SECTOR);
// The full tier-0 chain: the wedge streak persists across handlers (only
// `unproductive` resets per handler), so it reaches the abort threshold
// even though each handler yields early on the dead streak.
let mut handlers: Vec<Box<dyn SectionHandler>> = vec![
Box::new(Bisect),
Box::new(Jump),
Box::new(Linear {
reverse: true,
fast: true,
}),
Box::new(Linear {
reverse: false,
fast: true,
}),
];
let mut scoreboard = HandlerScoreboard::default();
let out = run_handlers(&mut ctx, &mut handlers, &mut bad, &mut scoreboard, |_| {
(h.now_fn())() + Duration::from_secs(60)
});
assert_eq!(
out,
HandlerOutcome::TransportFault,
"a wholly-wedged section must escalate to TransportFault"
);
// The whole point: it bailed after a short streak, not after grinding all
// 1000 sectors. Generous bound (handlers read in batches) but far below
// the section size.
assert!(
h.read_count() < 100,
"wedge must abort fast; did {} reads on a 1000-sector wedged section",
h.read_count()
);
}
#[test]
fn halt_token_returns_promptly() {
// Halt set before the call: the handler returns Halted on its first
@@ -994,6 +1125,7 @@ mod tests {
decrypt_is_aacs: false,
tick: None,
unproductive: 0,
wedge_streak: 0,
};
let mut bad = SubRanges::from_section(0, 100 * SECTOR);
let deadline = (ctx.now)() + Duration::from_secs(10);