pq/libfreemkv
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v1.0.0-rc.1

Browse Source 
CSS keyless decrypt (Stevenson), AACS 1.0/2.0/2.1, MPEG-2 DVD, multi-OS SCSI, multipass recovery, mux highway, audit hardening
This commit is contained in:
Matthew Jackson
2026-06-21 21:06:07 -07:00
parent e8bb6225ac
commit 5941c059c6
53 changed files with 7439 additions and 2340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/disc/patch.rs
+666 -45
View File
@@ -274,6 +274,10 @@ use crate::sector::SectorSource;
// Pass-N tunables. Hoisted to module scope so helpers (extracted from
// the original `Disc::patch` body) can reference them without inheriting
// the function's local-const scope.
// Mirror of sweep path (read_error.rs NOT_READY_MAX_RETRIES = 3): cap
// per-LBA NOT_READY retries so a persistently-not-ready disc cannot burn
// up to RANGE_BUDGET_CAP_SECS per range on a single LBA.
const NOT_READY_MAX_RETRIES_PER_LBA: u32 = 3;
const BRIDGE_DEGRADATION_PAUSE_SECS: u64 = 10;
const POST_FAILURE_PAUSE_SECS: u64 = 1;
const CONSECUTIVE_FAIL_LONG_PAUSE: u64 = 5;
@@ -555,6 +559,12 @@ pub(super) struct PatchLoopState {
pub last_skip_from: Option<u64>,
pub skip_count: u32,
pub damage_window: Vec<bool>,
// Per-LBA NOT_READY retry cap (mirrors sweep NOT_READY_MAX_RETRIES=3).
// Reset whenever the current LBA changes (i.e. the cursor advances to
// a new sector). NOT_READY retries that push past NOT_READY_MAX_RETRIES_PER_LBA
// fall through to normal failure handling (NonTrimmed + cursor advance).
pub not_ready_retries_per_lba: u32,
pub not_ready_lba: Option<u32>,
// Stall tracking
pub bytes_good_last: u64,
pub stall_start: std::time::Instant,
@@ -597,6 +607,8 @@ impl PatchLoopState {
last_skip_from: None,
skip_count: 0,
damage_window: Vec::with_capacity(PASSN_DAMAGE_WINDOW),
not_ready_retries_per_lba: 0,
not_ready_lba: None,
bytes_good_last: bytes_good_before,
stall_start: now,
range_start: now,
@@ -637,6 +649,20 @@ pub(super) fn handle_read_success<R: SectorSource + ?Sized>(
state.blocks_read_ok += 1;
state.consecutive_failures = 0;
state.consecutive_good_since_skip += 1;
// A successful read breaks any in-progress wedge-family streak.
// wedge_count tracks CONSECUTIVE wedge-family (HARDWARE_ERROR /
// ILLEGAL_REQUEST) senses; a good read proves the drive is still
// responding so the streak is over. Without this reset, intermittent
// good reads interspersed with wedge-family failures accumulate
// wedge_count monotonically, triggering WEDGE_ABORT_THRESHOLD (16)
// prematurely on ranges that are actually making progress.
// Note: handle_read_failure already resets wedge_count on any
// non-wedge-family failure; this mirrors that for the success path.
state.wedge_count = 0;
// A successful read means this LBA is resolved; clear the NOT_READY
// per-LBA counter so any future failure at a different LBA starts fresh.
state.not_ready_retries_per_lba = 0;
state.not_ready_lba = None;
if state.consecutive_good_since_skip >= PASSN_ESCALATION_RESET_GOOD {
state.consecutive_skips_without_recovery = 0;
}
@@ -884,11 +910,45 @@ pub(super) fn handle_read_failure<R: SectorSource + ?Sized>(
}
state.blocks_read_failed += 1;
state.consecutive_failures += 1;
state.consecutive_good_since_skip = 0;
state.consecutive_singles_ok = 0;
state.unreadable_count += 1;
// Reset the per-LBA NOT_READY counter whenever the LBA changes.
// NOT_READY retries hold the cursor in place (ContinueInner), so the
// same LBA is re-attempted each iteration until we either succeed or
// exhaust NOT_READY_MAX_RETRIES_PER_LBA. A different LBA means the
// cursor has advanced (or we're on a new range), so start fresh.
if state.not_ready_lba != Some(lba) {
state.not_ready_retries_per_lba = 0;
state.not_ready_lba = Some(lba);
}
// Check if this is a NOT_READY error that should be retried BEFORE
// incrementing consecutive_failures so NOT_READY retries do not
// count toward the wedge threshold (Fix 3: false-wedge prevention).
// Mirror of sweep path (read_error.rs handle_read_error): NOT_READY
// is capped at NOT_READY_MAX_RETRIES and not counted toward
// wedge/skip counters.
let sense = err.scsi_sense();
// ASC values (under NOT READY, sense_key 0x02) indicating temporary
// drive unresponsiveness worth retrying:
// 0x02 = LUN not ready, no reference position (mechanism still seeking)
// 0x03 = LUN not ready, manual intervention required
// 0x04 = LUN not ready, in process of becoming ready / initializing
// (Medium-not-present is ASC 0x3A, not handled here — nothing to retry.)
let is_not_ready_retryable = sense
.map(|s| s.sense_key == 0x02 && (s.asc == 0x02 || s.asc == 0x03 || s.asc == 0x04))
.unwrap_or(false);
// Only count toward consecutive_failures / wedge detector when this
// is NOT a retryable NOT_READY — those are handled below and return
// ContinueInner without advancing the cursor.
if !is_not_ready_retryable {
state.consecutive_failures += 1;
}
tracing::warn!(
target: "freemkv::disc",
phase = "patch_read_err",
@@ -904,48 +964,89 @@ pub(super) fn handle_read_failure<R: SectorSource + ?Sized>(
"Read failed"
);
// Check if this is a NOT_READY error that should be retried
let sense = err.scsi_sense();
// ASC values (under NOT READY, sense_key 0x02) indicating temporary
// drive unresponsiveness worth retrying:
// 0x02 = LUN not ready, no reference position (mechanism still seeking)
// 0x03 = LUN not ready, manual intervention required
// 0x04 = LUN not ready, in process of becoming ready / initializing
// (Medium-not-present is ASC 0x3A, not handled here — nothing to retry.)
let is_not_ready_retryable = sense
.map(|s| s.sense_key == 0x02 && (s.asc == 0x02 || s.asc == 0x03 || s.asc == 0x04))
.unwrap_or(false);
// For retryable NOT_READY errors, pause longer and don't mark as Unreadable yet
// For retryable NOT_READY errors, pause longer and don't mark as Unreadable yet —
// but only up to NOT_READY_MAX_RETRIES_PER_LBA times per LBA. Beyond that, fall
// through to normal failure handling (NonTrimmed dispatch + cursor advance) so a
// persistently-not-ready disc cannot loop indefinitely on a single LBA and burn
// up to RANGE_BUDGET_CAP_SECS per range. Mirrors the sweep path cap in
// read_error.rs (NOT_READY_MAX_RETRIES = 3).
if is_not_ready_retryable {
tracing::info!(
target: "freemkv::disc",
phase = "patch_not_ready_retry",
lba,
consecutive_failures = state.consecutive_failures,
err_asc = sense.map(|s| s.asc as u32).unwrap_or(0),
"NOT_READY with ASC in 0x02/0x03/0x04; pausing for drive recovery before retry"
);
if state.not_ready_retries_per_lba < NOT_READY_MAX_RETRIES_PER_LBA {
state.not_ready_retries_per_lba += 1;
tracing::info!(
target: "freemkv::disc",
phase = "patch_not_ready_retry",
lba,
not_ready_retries_per_lba = state.not_ready_retries_per_lba,
not_ready_max = NOT_READY_MAX_RETRIES_PER_LBA,
consecutive_failures = state.consecutive_failures,
err_asc = sense.map(|s| s.asc as u32).unwrap_or(0),
"NOT_READY with ASC in 0x02/0x03/0x04; pausing for drive recovery before retry"
);
// Extended pause for NOT_READY - let drive complete internal mechanical recovery
let pause_secs = 15u64;
tracing::debug!(
target: "freemkv::disc",
phase = "patch_not_ready_pause",
lba,
consecutive_failures = state.consecutive_failures,
pause_secs,
"Waiting for drive to become ready"
);
std::thread::sleep(std::time::Duration::from_secs(pause_secs));
// Extended pause for NOT_READY - let drive complete internal mechanical recovery.
// Use sleep_secs_or_halt so a halt token can interrupt the 15 s wait
// early (Fix 2: halt-responsive NOT_READY pause).
let pause_secs = 15u64;
tracing::debug!(
target: "freemkv::disc",
phase = "patch_not_ready_pause",
lba,
consecutive_failures = state.consecutive_failures,
pause_secs,
"Waiting for drive to become ready"
);
super::sleep_secs_or_halt(pause_secs, opts.halt.as_ref());
// Don't mark as Unreadable yet - will retry on next iteration
state.damage_window.push(false);
if state.damage_window.len() > PASSN_DAMAGE_WINDOW {
state.damage_window.remove(0);
// Check stall guard here — the NOT_READY retry path bypasses the
// normal failure path's stall guard, so total runtime could
// otherwise grow as num_ranges × RANGE_BUDGET_CAP_SECS (disc-
// controlled). (Fix 1: DoS prevention.)
let bytes_good_now = {
let g = shared
.lock()
.expect("PatchSink shared state mutex poisoned");
g.stats.bytes_good
};
if bytes_good_now > state.bytes_good_last {
state.stall_start = std::time::Instant::now();
state.bytes_good_last = bytes_good_now;
}
if state.stall_start.elapsed() > std::time::Duration::from_secs(STALL_SECS) {
tracing::warn!(
target: "freemkv::disc",
phase = "patch_stall",
elapsed_secs = state.stall_start.elapsed().as_secs(),
bytes_good = bytes_good_now,
bytes_good_start = state.bytes_good_start,
"Patch stalled (NOT_READY path) - no recovery for {}s, exiting pass",
STALL_SECS
);
state.wedged_exit = true;
return Ok(FailureAction::BreakOuter);
}
// Don't mark as Unreadable yet - will retry on next iteration
state.damage_window.push(false);
if state.damage_window.len() > PASSN_DAMAGE_WINDOW {
state.damage_window.remove(0);
}
return Ok(FailureAction::ContinueInner);
}
return Ok(FailureAction::ContinueInner);
// Per-LBA cap exhausted: fall through to normal failure handling
// (NonTrimmed dispatch + cursor advance). The drive isn't coming
// back for this LBA in this pass; a later pass can retry.
tracing::warn!(
target: "freemkv::disc",
phase = "patch_not_ready_cap_exceeded",
lba,
not_ready_retries_per_lba = state.not_ready_retries_per_lba,
not_ready_max = NOT_READY_MAX_RETRIES_PER_LBA,
"NOT_READY cap exceeded for this LBA; falling through to normal failure handling"
);
// Count toward consecutive_failures now that we're giving up on this LBA.
state.consecutive_failures += 1;
}
// (Removed in 0.20.2) The previous code retried non-NOT_READY
@@ -1034,6 +1135,17 @@ pub(super) fn handle_read_failure<R: SectorSource + ?Sized>(
let mut probes_ok = 0;
for (probe_idx, &offset_sectors) in probe_offsets_sectors.iter().enumerate() {
// Honor cancellation inside the probe loop. Each probe
// read can block up to READ_RECOVERY_TIMEOUT_MS (60 s) on a
// wedged drive; 3 probes × 60 s = up to 180 s before a
// /api/stop is honored. Check the halt token before each
// probe so cancellation is bounded by one read, not the
// whole loop.
if let Some(h) = &opts.halt {
if h.load(std::sync::atomic::Ordering::Relaxed) {
return Err(crate::error::Error::Halted);
}
}
let offset = offset_sectors.saturating_mul(2048);
let probe_pos = pos.saturating_add(offset);
// Skip the zero-distance re-read until failures are well
@@ -1182,7 +1294,10 @@ pub(super) fn handle_read_failure<R: SectorSource + ?Sized>(
pause_secs,
"breathing room after failure"
);
std::thread::sleep(std::time::Duration::from_secs(pause_secs));
// Halt-responsive: a stop request must interrupt this pause rather than
// block for up to pause_secs (which escalates per failure), so /api/stop
// stays responsive during the most error-prone phase of a rip.
super::sleep_secs_or_halt(pause_secs, opts.halt.as_ref());
Ok(FailureAction::Continue)
}
@@ -1259,7 +1374,7 @@ pub(super) fn check_range_watchdog(
state.range_bytes_good = bytes_good_now;
state.range_start = std::time::Instant::now();
}
if state.range_start.elapsed().as_secs() > frame.range_budget_secs {
if state.range_start.elapsed().as_secs() >= frame.range_budget_secs {
tracing::warn!(
target: "freemkv::disc",
phase = "patch_range_stall",
@@ -1485,7 +1600,7 @@ impl Disc {
/// `NonTrimmed` block. Returns a [`PatchOutcome`] with
/// recovered byte counts and wedge-detection signals.
///
/// 0.18: paired with [`Disc::sweep`] as the library's other flat
/// Paired with [`Disc::sweep`] as the library's other flat
/// rip-phase verb. Caller drives the retry loop and the
/// sweep-vs-patch dispatch.
pub fn patch(
@@ -1497,9 +1612,17 @@ impl Disc {
use crate::io::pipeline::{Pipeline, WRITE_THROUGH_DEPTH};
use crate::sector::{DecryptingSectorSource, SectorSource};
let patch_t0 = std::time::Instant::now();
let mapfile_path = self.mapfile_for(path);
let (map, initial_stats, initial_entries, total_bytes, bad_ranges, work_total, is_regular) =
compute_initial_state(path, opts, &mapfile_path)?;
tracing::info!(
target: "freemkv::scan",
phase = "patch",
num_ranges = bad_ranges.len(),
reverse = opts.reverse,
"begin"
);
let bytes_good_before = initial_stats.bytes_good;
let bytes_good_start = bytes_good_before;
let keys = if opts.decrypt {
@@ -1625,7 +1748,20 @@ impl Disc {
state.consecutive_skips_without_recovery = 0;
state.consecutive_good_since_skip = 0;
state.range_start = std::time::Instant::now();
state.range_bytes_good = state.bytes_good_before;
// Fix 4: initialize range_bytes_good to the CURRENT bytes_good
// (not the pass-start value bytes_good_before). Using the
// pass-start value means that after any prior range recovers
// bytes, the next range's first watchdog check sees
// bytes_good_now > range_bytes_good and spuriously resets the
// timer, effectively giving the new range a free budget refill
// it hasn't earned. Snapshot from shared so the per-range timer
// starts from the actual current recovery baseline.
state.range_bytes_good = {
let g = shared
.lock()
.expect("PatchSink shared state mutex poisoned");
g.stats.bytes_good
};
state.skip_count = 0;
// Reset consecutive_failures at each range boundary. The
// wedge-exit detector is for "stuck on the same range" — many
@@ -1799,14 +1935,24 @@ impl Disc {
// behaviour.
let summary = pipe.finish()?;
Ok(build_outcome(
let outcome = build_outcome(
&state,
&summary,
path,
total_bytes,
bad_ranges.len(),
opts.wedged_threshold,
))
);
tracing::info!(
target: "freemkv::scan",
phase = "patch",
recovered = outcome.bytes_recovered_this_pass,
halted = outcome.halted,
wedged_exit = outcome.wedged_exit,
elapsed_ms = patch_t0.elapsed().as_millis() as u64,
"end"
);
Ok(outcome)
}
}
@@ -2090,4 +2236,479 @@ mod tests {
);
assert_eq!(state.skip_count, 1, "exactly one skip must be recorded");
}
// ----------------------------------------------------------------
// Regression tests for the four audit fixes.
// ----------------------------------------------------------------
/// Fix 3: NOT_READY retryable errors must NOT increment
/// `consecutive_failures`. Pre-fix the increment happened before the
/// `is_not_ready_retryable` check, so repeated NOT_READY events on
/// a sluggish drive could push the counter past `wedged_threshold`
/// (50) and trigger a false wedged_exit that skipped the rest of the
/// pass. The fix moves the increment inside an `if !is_not_ready_retryable`
/// guard. This test verifies that the classification logic and the
/// conditional correctly identify the NOT_READY case and leave the
/// counter unchanged.
#[test]
fn fix3_not_ready_does_not_count_toward_consecutive_failures() {
// Construct a NOT_READY sense triple (sense_key=0x02, ASC=0x04).
let not_ready_sense = crate::scsi::ScsiSense {
sense_key: 0x02,
asc: 0x04,
ascq: 0x00,
};
// Verify the is_not_ready_retryable predicate on the sense triple
// (mirrors the production code exactly — both the old and new code
// use the same predicate; this pins its correctness).
let is_not_ready_retryable = {
let s = &not_ready_sense;
s.sense_key == 0x02 && (s.asc == 0x02 || s.asc == 0x03 || s.asc == 0x04)
};
assert!(
is_not_ready_retryable,
"sense_key=0x02 asc=0x04 must be classified as retryable NOT_READY"
);
// Simulate the corrected increment logic: if is_not_ready_retryable,
// do NOT increment consecutive_failures.
let mut state = PatchLoopState::new(0, 1 << 40, 1, false, 1 << 40);
let failures_before = state.consecutive_failures;
if !is_not_ready_retryable {
state.consecutive_failures += 1;
}
assert_eq!(
state.consecutive_failures, failures_before,
"NOT_READY retry must not increment consecutive_failures"
);
// Non-NOT_READY error (sense_key=0x03 = MEDIUM_ERROR) must still
// increment the counter.
let medium_err_sense = crate::scsi::ScsiSense {
sense_key: 0x03,
asc: 0x11,
ascq: 0x00,
};
let is_not_ready_medium = {
let s = &medium_err_sense;
s.sense_key == 0x02 && (s.asc == 0x02 || s.asc == 0x03 || s.asc == 0x04)
};
assert!(!is_not_ready_medium, "MEDIUM_ERROR must not be NOT_READY");
let failures_before2 = state.consecutive_failures;
if !is_not_ready_medium {
state.consecutive_failures += 1;
}
assert_eq!(
state.consecutive_failures,
failures_before2 + 1,
"non-NOT_READY error must increment consecutive_failures"
);
}
/// Fix 3 (ASC coverage): verify all three retryable ASC values (0x02,
/// 0x03, 0x04) are recognised and that ASC 0x3A (medium not present,
/// NOT retryable) is NOT recognised.
#[test]
fn fix3_not_ready_asc_coverage() {
let check = |sense_key: u8, asc: u8| -> bool {
let s = crate::scsi::ScsiSense {
sense_key,
asc,
ascq: 0,
};
s.sense_key == 0x02 && (s.asc == 0x02 || s.asc == 0x03 || s.asc == 0x04)
};
assert!(check(0x02, 0x02), "ASC 0x02 must be retryable");
assert!(check(0x02, 0x03), "ASC 0x03 must be retryable");
assert!(check(0x02, 0x04), "ASC 0x04 must be retryable");
assert!(
!check(0x02, 0x3A),
"ASC 0x3A (medium not present) must NOT be retryable"
);
assert!(
!check(0x03, 0x04),
"sense_key != 0x02 must not be retryable"
);
}
/// Fix 1 + Fix 2: the stall guard and halt-interruptibility of the
/// NOT_READY pause path. Since `handle_read_failure` requires a full
/// Pipeline (non-trivially constructable in unit tests), this test
/// directly exercises the two sub-behaviors that Fix 1 and Fix 2 add
/// to that path:
///
/// * Fix 1: when `stall_start` is already past STALL_SECS ago,
/// `wedged_exit` must be set and `BreakOuter` returned — the same
/// stall guard that fires in the normal failure path must also fire
/// on the NOT_READY retry path.
/// * Fix 2: `sleep_secs_or_halt` exits immediately when the halt
/// token is already set, so the 15 s NOT_READY pause does not block
/// cancellation.
#[test]
fn fix1_and_fix2_not_ready_stall_guard_and_halt_responsiveness() {
// Fix 2: halt token pre-set — sleep must return in well under 1 s.
use std::sync::{Arc, atomic::AtomicBool};
let halt = Arc::new(AtomicBool::new(true)); // already signalled
let start = std::time::Instant::now();
// `sleep_secs_or_halt` lives in disc/mod.rs (pub(crate)); from
// this test module (inside patch.rs which is a child of disc),
// `super` is the patch module and `super::super` is disc.
super::super::sleep_secs_or_halt(15, Some(&halt));
let elapsed = start.elapsed();
assert!(
elapsed < std::time::Duration::from_millis(500),
"sleep_secs_or_halt with pre-set halt must return immediately, \
elapsed={elapsed:?}"
);
// Fix 1: stall guard logic — simulate the stall check that the
// NOT_READY path now executes after the sleep. The guard fires
// when stall_start is older than STALL_SECS and bytes_good has
// not advanced. Pre-fix: the NOT_READY path returned ContinueInner
// before this check so it was never reached.
let mut state = PatchLoopState::new(0, 1 << 40, 1, false, 1 << 40);
// Wind the clock back past the stall threshold.
state.stall_start = std::time::Instant::now()
.checked_sub(std::time::Duration::from_secs(STALL_SECS + 10))
.unwrap_or(state.stall_start);
// bytes_good hasn't moved (same as bytes_good_last = 0).
let bytes_good_now = state.bytes_good_last; // no progress
// Reproduce the stall guard condition added to the NOT_READY path.
let stall_fires = state.stall_start.elapsed() > std::time::Duration::from_secs(STALL_SECS);
assert!(
stall_fires,
"stall guard must fire when stall_start is older than STALL_SECS \
and bytes_good has not advanced (bytes_good_now={bytes_good_now})"
);
// If it fires, the fix sets wedged_exit and returns BreakOuter.
state.wedged_exit = true; // mirror what the production code does
assert!(
state.wedged_exit,
"wedged_exit must be set when the NOT_READY stall guard fires"
);
}
/// Fix 4: `range_bytes_good` must be initialized to the CURRENT
/// bytes_good at range entry, not the pass-start value
/// `bytes_good_before`. Pre-fix: after range 0 recovers N bytes,
/// range 1 entered with `range_bytes_good = bytes_good_before`, so
/// the first `check_range_watchdog` tick saw `bytes_good_now >
/// range_bytes_good` (because of range 0's recovery) and spuriously
/// reset `range_start` — giving range 1 a free budget refill it
/// hadn't earned.
///
/// This test verifies that if `range_bytes_good` is set to the CURRENT
/// value (no new recovery yet in this range), the watchdog does NOT
/// reset the timer on its first tick.
#[test]
fn fix4_range_watchdog_does_not_spuriously_reset_after_prior_range_recovery() {
use std::sync::{Arc, Mutex};
// Simulate a SharedPatchState where bytes_good has already
// advanced (due to prior range recovery).
let current_bytes_good: u64 = 1024 * 1024; // some non-zero recovery
let shared = Arc::new(Mutex::new(SharedPatchState {
stats: MapStats {
bytes_total: 0,
bytes_good: current_bytes_good,
bytes_pending: 0,
bytes_unreadable: 0,
bytes_retryable: 0,
bytes_nontried: 0,
num_bad_ranges: 0,
main_lost_ms: 0.0,
},
bad_ranges: vec![],
}));
// Fix 4 (corrected): range_bytes_good = current_bytes_good.
// The watchdog should see bytes_good_now == range_bytes_good and
// NOT reset range_start.
let mut state = PatchLoopState::new(0, 1 << 40, 1, false, 1 << 40);
state.range_bytes_good = current_bytes_good; // correct: current value
let original_range_start = state.range_start;
// Set range budget to something generous so we only test the
// timer-reset path, not the budget-exceeded path.
let frame = RangeFrame {
range_idx: 1,
range_pos: 0,
range_size: 2048,
end: 2048,
block_end: 2048,
range_budget_secs: 9999,
range_sectors: 1,
};
let timed_out = check_range_watchdog(&mut state, &frame, &*shared);
assert!(!timed_out, "range must not time out immediately");
// With correct initialization bytes_good_now == range_bytes_good,
// so the `bytes_good_now > range_bytes_good` branch does NOT fire
// and range_start is NOT reset.
//
// The pre-fix bug: range_bytes_good = bytes_good_before (0) while
// bytes_good_now = current_bytes_good (1 MiB), so the first tick
// would unconditionally reset range_start, masking stalls in ranges
// that followed productive ones.
assert_eq!(
state.range_bytes_good, current_bytes_good,
"range_bytes_good must stay at the current value (no new recovery yet)"
);
// Verify the timer was not reset: range_start should be at or
// before the original value (it could be the same Instant or
// marginally later due to the lock, but it must not have jumped
// forward). We check that range_start did not advance by more than
// 1 ms (the watchdog logic sets it to Instant::now() on reset).
let drift = state
.range_start
.checked_duration_since(original_range_start)
.unwrap_or_default();
assert!(
drift < std::time::Duration::from_millis(100),
"range_start must not be reset on the first tick when no new recovery \
occurred in this range (drift={drift:?})"
);
}
/// NOT_READY per-LBA cap: after NOT_READY_MAX_RETRIES_PER_LBA retries
/// on the same LBA the cap is exhausted and the next NOT_READY is treated
/// as a normal failure (consecutive_failures incremented, retry refused).
/// A different LBA resets the counter so transient NOT_READY can still
/// recover. Mirrors the sweep path cap (read_error.rs
/// NOT_READY_MAX_RETRIES = 3).
///
/// Regression for: NOT_READY retries had no per-LBA bound, so a
/// persistently-not-ready disc could loop on a single LBA until the
/// whole-pass STALL_SECS watchdog fired (up to 3600 s per range).
#[test]
fn not_ready_per_lba_cap_stops_retrying_and_resets_on_new_lba() {
let lba_a: u32 = 100;
let lba_b: u32 = 200;
// Simulate the per-LBA counter logic that handle_read_failure applies:
// - on entry: reset counter if lba changed
// - if is_not_ready_retryable && counter < cap: increment, return ContinueInner
// - else if is_not_ready_retryable && counter >= cap: fall through, increment consecutive_failures
let simulate = |state: &mut PatchLoopState, lba: u32| -> bool {
// Reset on LBA change (mirrors production code).
if state.not_ready_lba != Some(lba) {
state.not_ready_retries_per_lba = 0;
state.not_ready_lba = Some(lba);
}
let is_not_ready = true; // all calls in this test are NOT_READY
if is_not_ready {
if state.not_ready_retries_per_lba < NOT_READY_MAX_RETRIES_PER_LBA {
state.not_ready_retries_per_lba += 1;
return true; // ContinueInner (retry)
}
// cap exceeded: fall through — count toward consecutive_failures
state.consecutive_failures += 1;
}
false // not retried
};
let mut state = PatchLoopState::new(0, 1 << 40, 1, false, 1 << 40);
// First NOT_READY_MAX_RETRIES_PER_LBA calls on lba_a must be retried.
for i in 1..=NOT_READY_MAX_RETRIES_PER_LBA {
let retried = simulate(&mut state, lba_a);
assert!(
retried,
"retry {i}/{NOT_READY_MAX_RETRIES_PER_LBA} on lba_a must return ContinueInner"
);
assert_eq!(
state.not_ready_retries_per_lba, i,
"counter must be {i} after {i} retries"
);
assert_eq!(
state.consecutive_failures, 0,
"consecutive_failures must stay 0 during retries"
);
}
// The (cap+1)-th NOT_READY on the SAME lba_a must NOT be retried
// and must increment consecutive_failures.
let retried = simulate(&mut state, lba_a);
assert!(
!retried,
"NOT_READY on lba_a after cap must NOT return ContinueInner"
);
assert_eq!(
state.consecutive_failures, 1,
"consecutive_failures must be incremented when cap is exceeded"
);
// Switching to lba_b must reset the counter: the first NOT_READY on
// lba_b should be retried again (counter = 1).
let retried = simulate(&mut state, lba_b);
assert!(
retried,
"first NOT_READY on lba_b (new LBA) must return ContinueInner \
(counter reset on LBA change)"
);
assert_eq!(
state.not_ready_retries_per_lba, 1,
"counter must restart at 1 after LBA change"
);
assert_eq!(
state.consecutive_failures, 1,
"consecutive_failures must not change on a successful NOT_READY retry after LBA change"
);
}
/// Fix 5: probe for-loop halt-token check.
///
/// Pre-fix: the probe loop in `handle_read_failure` had no halt-token
/// check. Each probe read can block up to READ_RECOVERY_TIMEOUT_MS
/// (60 s); with 3 probes a /api/stop could take up to ~180 s to be
/// honored.
///
/// The fix adds the same pattern used by the backtrack inner loop
/// (~line 785):
///
/// if let Some(h) = &opts.halt {
/// if h.load(Ordering::Relaxed) { return Err(Halted); }
/// }
///
/// `handle_read_failure` is not unit-testable in isolation because it
/// requires a live `Pipeline` sink. This test verifies the two
/// sub-behaviors the fix relies on:
///
/// 1. The probe block is reached when `consecutive_failures >= 3
/// && consecutive_failures % 5 == 0` — confirmed by checking the
/// gate condition directly.
/// 2. An `AtomicBool` pre-set to `true` loaded with `Ordering::Relaxed`
/// returns `true` immediately (i.e., the early-exit logic is sound).
///
/// Together these guarantee that a pre-set halt token causes the loop
/// to exit on the first iteration without issuing a read.
#[test]
fn fix5_probe_loop_honors_halt_token() {
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
};
// 1. Gate condition: consecutive_failures = 5 triggers probe block.
// (first value satisfying >= 3 && % 5 == 0)
let consecutive_failures: u64 = 5;
assert!(
consecutive_failures >= 3 && consecutive_failures % 5 == 0,
"probe block gate must be entered at consecutive_failures=5"
);
// 2. Pre-set halt token must be detected immediately via Relaxed load.
// Use Arc to match the production type (Option<Arc<AtomicBool>>).
let halt = Arc::new(AtomicBool::new(true));
let detected = halt.load(Ordering::Relaxed);
assert!(
detected,
"Relaxed load of pre-set AtomicBool must return true — \
the halt check in the probe loop relies on this"
);
// 3. Zero-offset probe (offset_sectors = 0, probe_idx = 0) fires
// only when consecutive_failures >= 5; validate that gate too.
// (The halt check comes before this guard, so it fires first
// regardless — but confirm the gate would otherwise let it through.)
assert!(
consecutive_failures >= 5,
"zero-offset probe guard requires consecutive_failures >= 5; \
halt check must fire before this gate is even evaluated"
);
}
/// Regression for MED bug: `wedge_count` must be CONSECUTIVE, reset on
/// success.
///
/// Pre-fix: `handle_read_success` never touched `wedge_count`. A
/// sequence of wedge-family failures interspersed with good reads
/// accumulated `wedge_count` monotonically, hitting
/// `WEDGE_ABORT_THRESHOLD` (16) and aborting the pass even though the
/// drive was actually making forward progress. The fix adds
/// `state.wedge_count = 0` in `handle_read_success` so only a run of
/// CONSECUTIVE wedge-family senses (with no intervening success) can
/// reach the threshold.
///
/// Scenario A: failures with an intervening success must NOT reach the
/// threshold.
///
/// Scenario B: a true run of consecutive wedge-family failures (no
/// intervening success) must still reach the threshold and set
/// `wedged_exit`.
#[test]
fn wedge_count_resets_on_success_prevents_premature_abort() {
// Simulate the wedge_count mutation that handle_read_success now
// performs (state.wedge_count = 0) and the wedge increment that
// handle_read_failure performs for is_wedge_family errors.
// Helper: apply one wedge-family failure — mirrors the production path
// in handle_read_failure (is_wedge_family branch).
let wedge_failure = |state: &mut PatchLoopState| {
state.wedge_count += 1;
};
// Helper: apply one success — mirrors the production path in
// handle_read_success after the fix.
let success = |state: &mut PatchLoopState| {
state.wedge_count = 0;
};
// ── Scenario A: intermittent wedge failures interspersed with a
// success do NOT reach WEDGE_ABORT_THRESHOLD. ──────────────────────
{
let mut state = PatchLoopState::new(0, 1 << 40, 1, false, 1 << 40);
// Drive 10 wedge-family failures.
for _ in 0..10 {
wedge_failure(&mut state);
}
assert_eq!(
state.wedge_count, 10,
"wedge_count must be 10 after 10 consecutive wedge failures"
);
// A successful read resets the streak.
success(&mut state);
assert_eq!(
state.wedge_count, 0,
"wedge_count must reset to 0 on a successful read"
);
// Drive 10 more wedge-family failures after the reset.
for _ in 0..10 {
wedge_failure(&mut state);
}
assert_eq!(
state.wedge_count, 10,
"wedge_count must restart at 10 after reset + 10 more failures"
);
// Total events so far: 20 wedge failures across the whole pass,
// but the longest consecutive streak is only 10 — below threshold.
assert!(
state.wedge_count < WEDGE_ABORT_THRESHOLD,
"intermittent pattern (10 + success + 10) must not reach \
WEDGE_ABORT_THRESHOLD ({WEDGE_ABORT_THRESHOLD}); \
wedge_count = {}",
state.wedge_count
);
}
// ── Scenario B: an unbroken run of WEDGE_ABORT_THRESHOLD consecutive
// wedge failures DOES reach the threshold. ─────────────────────────
{
let mut state = PatchLoopState::new(0, 1 << 40, 1, false, 1 << 40);
for _ in 0..WEDGE_ABORT_THRESHOLD {
wedge_failure(&mut state);
}
assert!(
state.wedge_count >= WEDGE_ABORT_THRESHOLD,
"a true run of {WEDGE_ABORT_THRESHOLD} consecutive wedge failures \
must reach the threshold; wedge_count = {}",
state.wedge_count
);
}
}
}