pq/libfreemkv
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

unified read-error handler + pass N size-aware skip

Browse Source 
New disc/read_error.rs as the single entry point all read failures flow
through. Handler classifies the error, updates the in-flight context
(damage window, retry budgets, jump multiplier), and returns a
ReadAction the caller dispatches on. Pass 1 (sweep) refactored to use
it; ~340 lines of nested if/else collapsed into ~120 lines of action
dispatch. Adding a new error class = one match arm. Logging is in one
place. Bisect inner failures don't poison the damage window. Jump
multiplier capped at 64 (max 1 GB jump for batch=32 — observed prior
unbounded behavior produce a single 56 GB jump on a wedged drive).

Pass N (patch) damage_skip is now size-aware: each skip is capped at
range_remaining/4 rather than the absolute MB-scale escalation. The
old logic could leap over a 100-sector bad range that hides a 50-sector
good middle; size-aware convergence finds the good middles instead.

Tests in tests/pass_n_size_aware_skip.rs exercise the size-aware skip
against synthetic patterns (25-bad/50-good/25-bad and three good
middles in a row) and prove ≥98% of good middles are recovered.
Existing test test_disc_copy_marks_failed_ecc_blocks_as_nontrimmed
updated to reflect that MEDIUM_ERROR now triggers single-sector
bisect (which the BlockSizeFailingReader succeeds at).
This commit is contained in:
MattJackson
2026-05-07 09:08:43 -07:00
parent 3f6bf33de5
commit 97e1a4cad3
5 changed files with 886 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/disc/mod.rs
+161 -287
View File
@@ -12,6 +12,7 @@ mod bluray;
mod dvd;
mod encrypt;
pub mod mapfile;
pub mod read_error;
use crate::drive::{Drive, extract_scsi_context};
use crate::error::{Error, Result};
@@ -1428,19 +1429,15 @@ impl Disc {
let mut read_ok_count: u64 = 0;
let mut read_err_count: u64 = 0;
let mut last_log_iter: u64 = 0;
let mut not_ready_retries: u32 = 0;
const NOT_READY_MAX_RETRIES: u32 = 3;
let mut bridge_degradation_count: u32 = 0;
const BRIDGE_DEGRADATION_MAX: u32 = 5;
const BRIDGE_DEGRADATION_COOLDOWN_SECS: u64 = 10;
const PASS1_DAMAGE_WINDOW: usize = 16;
const PASS1_DAMAGE_THRESHOLD_PCT: usize = 12;
const PASS1_JUMP_SECTORS_FACTOR: u64 = 256;
const PASS1_ESCALATION_RESET_GOOD: u64 = PASS1_DAMAGE_WINDOW as u64;
let mut damage_window: Vec<bool> = Vec::with_capacity(PASS1_DAMAGE_WINDOW);
let mut jump_multiplier: u64 = 1;
let mut consecutive_good: u64 = 0;
// ALL read state lives in one place. The single error-handling
// entry point (`read_error::handle_read_error`) owns the
// counters, retry budgets, and damage-window updates.
let mut read_ctx = read_error::ReadCtx::for_sweep(batch);
// Speed control derives from damage-zone state. We track the
// transition locally so we only call set_speed on edges, not
// every iteration.
let mut in_damage_zone = false;
const DAMAGE_ZONE_EXIT_THRESHOLD: u64 = 16;
tracing::trace!(
target: "freemkv::disc",
phase = "copy_start",
@@ -1451,10 +1448,16 @@ impl Disc {
);
'outer: loop {
// Every pass retries every non-Finished range. Includes
// Unreadable so each pass gets its own shot at sectors prior
// passes gave up on — drive state may have changed (cooled
// down, bridge stabilized, etc.). Mapfile is binary in
// intent: Finished or not-yet-good.
let regions_to_do = map.ranges_with(&[
mapfile::SectorStatus::NonTried,
mapfile::SectorStatus::NonTrimmed,
mapfile::SectorStatus::NonScraped,
mapfile::SectorStatus::Unreadable,
]);
tracing::trace!(
target: "freemkv::disc",
@@ -1500,17 +1503,16 @@ impl Disc {
recovery,
);
let mut did_skip_ahead = false;
if read_result.is_ok() {
match read_result {
Ok(_) => {
// === SUCCESS PATH ===
read_ok_count += 1;
damage_window.push(true);
if damage_window.len() > PASS1_DAMAGE_WINDOW {
damage_window.remove(0);
}
consecutive_good += 1;
if consecutive_good >= PASS1_ESCALATION_RESET_GOOD {
jump_multiplier = 1;
read_ctx.on_success();
// Damage-zone exit: after enough consecutive good
// reads, restore max speed and reset jump multiplier.
if read_ctx.consecutive_good >= DAMAGE_ZONE_EXIT_THRESHOLD {
read_ctx.jump_multiplier = 1;
if in_damage_zone {
in_damage_zone = false;
reader.set_speed(0xFFFF);
@@ -1522,7 +1524,7 @@ impl Disc {
);
}
}
bridge_degradation_count = 0;
read_ctx.bridge_degradation_count = 0;
if opts.decrypt {
crate::decrypt::decrypt_sectors(
@@ -1538,264 +1540,102 @@ impl Disc {
map.record(pos, block_bytes, mapfile::SectorStatus::Finished)
.map_err(|e| Error::IoError { source: e })?;
bytes_done = bytes_done.saturating_add(block_bytes);
} else if !opts.skip_on_error {
let (status, sense) = read_result
.as_ref()
.err()
.map(extract_scsi_context)
.unwrap_or((0, None));
pos += block_bytes;
}
Err(err) if !opts.skip_on_error => {
// Caller asked us not to skip. Surface the error verbatim.
let (status, sense) = extract_scsi_context(&err);
return Err(Error::DiscRead {
sector: block_lba as u64,
status: Some(status),
sense,
});
} else {
let err = match read_result {
Ok(_) => unreachable!(),
Err(e) => e,
};
}
Err(err) => {
// === ERROR PATH — single source of truth ===
// ALL errors flow through handle_read_error. New
// error class = one new arm in that function.
// Logging, counter updates, retry budgets all live
// there. The dispatch below is purely the I/O side
// of each action.
read_err_count += 1;
consecutive_good = 0;
let action = read_error::handle_read_error(&err, &mut read_ctx);
// Transport failure means the USB bridge crashed/wedged. Instead of aborting,
// drop to single-sector reads with extended timeout and retry a few times.
// This allows recovery without requiring manual intervention.
if err.is_scsi_transport_failure() {
tracing::warn!(
target: "freemkv::disc",
phase = "transport_failure",
lba = block_lba,
error = %err,
"transport failure (bridge crash); dropping to single-sector read with extended timeout"
);
// Give the bridge time to recover from crash state
std::thread::sleep(std::time::Duration::from_secs(5));
// Try reading as a single sector instead of a batch - this often succeeds
// where bulk reads fail because the bridge can handle smaller transactions
let mut single_buf = [0u8; 2048];
match reader.read_sectors(block_lba, 1, &mut single_buf[..], true) {
Ok(_) => {
tracing::info!(
target: "freemkv::disc",
phase = "transport_recovery_success",
lba = block_lba,
"Single-sector read succeeded after transport failure"
);
// Write the recovered sector and continue
file.seek(SeekFrom::Start(pos))
.map_err(|e| Error::IoError { source: e })?;
file.write_all(&single_buf)
.map_err(|e| Error::IoError { source: e })?;
// Mark remaining sectors in the batch as NonTrimmed (we only recovered 1 sector)
let zero = vec![0u8; block_bytes as usize];
file.seek(SeekFrom::Start(pos))
.map_err(|e| Error::IoError { source: e })?;
file.write_all(&zero)
.map_err(|e| Error::IoError { source: e })?;
map.record(pos, block_bytes, mapfile::SectorStatus::NonTrimmed)
.map_err(|e| Error::IoError { source: e })?;
bytes_done = bytes_done.saturating_add(block_bytes);
damage_window.push(false);
if damage_window.len() > PASS1_DAMAGE_WINDOW {
damage_window.remove(0);
match action {
read_error::ReadAction::Retry { pause_secs } => {
if pause_secs > 0 {
std::thread::sleep(std::time::Duration::from_secs(pause_secs));
}
did_skip_ahead = false;
pos += block_bytes;
continue;
// Don't advance pos — same LBA next iteration.
}
Err(e2) => {
tracing::warn!(
target: "freemkv::disc",
phase = "transport_recovery_failed",
lba = block_lba,
single_sector_error = %e2,
"Single-sector read also failed after transport failure"
);
// If even single sector fails, mark the whole batch as unreadable and continue
let zero = vec![0u8; block_bytes as usize];
file.seek(SeekFrom::Start(pos))
.map_err(|e| Error::IoError { source: e })?;
file.write_all(&zero)
.map_err(|e| Error::IoError { source: e })?;
map.record(pos, block_bytes, mapfile::SectorStatus::Unreadable)
.map_err(|e| Error::IoError { source: e })?;
bytes_done = bytes_done.saturating_add(block_bytes);
damage_window.push(false);
if damage_window.len() > PASS1_DAMAGE_WINDOW {
damage_window.remove(0);
}
did_skip_ahead = false;
pos += block_bytes;
continue;
}
}
}
if err.is_bridge_degradation() {
if bridge_degradation_count < BRIDGE_DEGRADATION_MAX {
bridge_degradation_count += 1;
tracing::warn!(
target: "freemkv::disc",
phase = "bridge_degradation",
lba = block_lba,
degradation_count = bridge_degradation_count,
error = %err,
"bridge degradation (firmware stress); cooling down 15s"
);
// Extended cooldown for bridge recovery - increased from 10s to 15s
std::thread::sleep(std::time::Duration::from_secs(15));
// Reduce batch size after degradation to be gentler on the bridge
let reduced_batch = (batch as u64 / 2).max(8);
tracing::debug!(
target: "freemkv::disc",
phase = "batch_reduction",
from_batch = batch,
to_batch = reduced_batch,
"Reducing batch size after bridge degradation"
);
// Continue with the same sector to retry the read
continue;
}
tracing::warn!(
target: "freemkv::disc",
phase = "bridge_degradation_exhausted",
lba = block_lba,
error = %err,
"bridge degradation retries exhausted; treating as bad sector"
);
}
let sense = err.scsi_sense();
let sense_key = sense.map(|s| s.sense_key).unwrap_or(0);
let asc = sense.map(|s| s.asc).unwrap_or(0);
let ascq = sense.map(|s| s.ascq).unwrap_or(0);
if sense_key == crate::scsi::SENSE_KEY_NOT_READY
&& not_ready_retries < NOT_READY_MAX_RETRIES
{
not_ready_retries += 1;
tracing::warn!(
target: "freemkv::disc",
phase = "not_ready_pause",
lba = block_lba,
sense_key,
asc,
ascq,
retry = not_ready_retries,
"NOT READY; pausing 3s then retrying"
);
std::thread::sleep(std::time::Duration::from_secs(3));
continue;
}
not_ready_retries = 0;
// For marginal errors (MEDIUM_ERROR, ABORTED_COMMAND), try smaller reads first
// before giving up on the whole batch. This prevents bridge crashes from
// forcing us to skip large sections of disc.
let should_try_smaller_reads = match sense_key {
crate::scsi::SENSE_KEY_MEDIUM_ERROR => true,
crate::scsi::SENSE_KEY_ABORTED_COMMAND => true,
_ => false,
};
if should_try_smaller_reads && block_count > 1 {
tracing::info!(
target: "freemkv::disc",
phase = "retry_with_smaller_read",
lba = block_lba,
original_sectors = block_count,
sense_key,
asc,
ascq,
"Marginal error detected; retrying with smaller reads"
);
// Try reading the batch as individual sectors - this is gentler on the bridge
let mut all_failed = true;
read_error::ReadAction::Bisect => {
// Re-issue the failed batch as single-sector reads.
// ctx.bisecting=true so the inner failures don't
// recursively request another bisect.
read_ctx.bisecting = true;
let saved_batch = read_ctx.batch;
read_ctx.batch = 1;
for sector_offset in 0..block_count {
if let Some(ref h) = opts.halt {
if h.load(std::sync::atomic::Ordering::Relaxed) {
halt_requested = true;
read_ctx.bisecting = false;
read_ctx.batch = saved_batch;
break 'outer;
}
}
let sector_lba = block_lba + (sector_offset as u32);
let mut sector_buf = [0u8; 2048];
match reader.read_sectors(sector_lba, 1, &mut sector_buf[..], true) {
Ok(_) => {
all_failed = false;
// Write recovered sector
let write_pos = pos + (sector_offset as u64 * 2048);
match reader.read_sectors(
sector_lba,
1,
&mut sector_buf[..],
true,
) {
Ok(_) => {
read_ctx.on_success();
file.seek(SeekFrom::Start(write_pos))
.map_err(|e| Error::IoError { source: e })?;
file.write_all(&sector_buf)
.map_err(|e| Error::IoError { source: e })?;
// Mark this sector as finished in mapfile
map.record(write_pos, 2048, mapfile::SectorStatus::Finished)
map.record(
write_pos,
2048,
mapfile::SectorStatus::Finished,
)
.map_err(|e| Error::IoError { source: e })?;
}
Err(_) => {
// Mark failed sector as NonTrimmed
let write_pos = pos + (sector_offset as u64 * 2048);
Err(inner_err) => {
// Inner failure goes through the same handler — it'll
// see bisecting=true and won't recurse. We only honour
// the SkipBlock action here (the bisect by definition
// can't return another Bisect, and JumpAhead inside a
// single-sector retry doesn't make sense).
let _ = read_error::handle_read_error(
&inner_err,
&mut read_ctx,
);
let zero = [0u8; 2048];
file.seek(SeekFrom::Start(write_pos))
.map_err(|e| Error::IoError { source: e })?;
let zero = vec![0u8; 2048];
file.write_all(&zero)
.map_err(|e| Error::IoError { source: e })?;
map.record(write_pos, 2048, mapfile::SectorStatus::NonTrimmed)
map.record(
write_pos,
2048,
mapfile::SectorStatus::NonTrimmed,
)
.map_err(|e| Error::IoError { source: e })?;
}
}
}
if !all_failed {
tracing::info!(
target: "freemkv::disc",
phase = "smaller_read_partial_success",
lba = block_lba,
recovered_sectors = block_count - 1, // We count failures as NonTrimmed above
total_sectors = block_count,
"Partial recovery from smaller reads"
);
}
read_ctx.bisecting = false;
read_ctx.batch = saved_batch;
bytes_done = bytes_done.saturating_add(block_bytes);
damage_window.push(false);
if damage_window.len() > PASS1_DAMAGE_WINDOW {
damage_window.remove(0);
}
did_skip_ahead = false;
pos += block_bytes;
continue;
}
tracing::warn!(
target: "freemkv::disc",
phase = "skip_ecc_block",
lba = block_lba,
sectors = block_count,
sense_key,
asc,
ascq,
error = %err,
"ECC block failed; marking NonTrimmed"
);
read_error::ReadAction::SkipBlock { pause_secs } => {
let zero = vec![0u8; block_bytes as usize];
file.seek(SeekFrom::Start(pos))
.map_err(|e| Error::IoError { source: e })?;
@@ -1804,21 +1644,26 @@ impl Disc {
map.record(pos, block_bytes, mapfile::SectorStatus::NonTrimmed)
.map_err(|e| Error::IoError { source: e })?;
bytes_done = bytes_done.saturating_add(block_bytes);
damage_window.push(false);
if damage_window.len() > PASS1_DAMAGE_WINDOW {
damage_window.remove(0);
if pause_secs > 0 {
std::thread::sleep(std::time::Duration::from_secs(pause_secs));
}
pos += block_bytes;
}
read_error::ReadAction::JumpAhead {
sectors,
pause_secs,
} => {
// Mark the failed batch + the gap up to jump_pos NonTrimmed.
let zero_batch = vec![0u8; block_bytes as usize];
file.seek(SeekFrom::Start(pos))
.map_err(|e| Error::IoError { source: e })?;
file.write_all(&zero_batch)
.map_err(|e| Error::IoError { source: e })?;
map.record(pos, block_bytes, mapfile::SectorStatus::NonTrimmed)
.map_err(|e| Error::IoError { source: e })?;
bytes_done = bytes_done.saturating_add(block_bytes);
let bad_count = damage_window.iter().filter(|&&b| !b).count();
if damage_window.len() >= PASS1_DAMAGE_WINDOW
&& bad_count * 100 / damage_window.len() >= PASS1_DAMAGE_THRESHOLD_PCT
{
let jump_sectors =
PASS1_JUMP_SECTORS_FACTOR * batch as u64 * jump_multiplier;
let jump_lba = ((pos / 2048) + jump_sectors) as u32;
let region_end_lba = (region_end / 2048) as u32;
if jump_lba < region_end_lba {
// Damage-zone enter: drop to minimum read speed.
if !in_damage_zone {
in_damage_zone = true;
reader.set_speed(0x0000);
@@ -1829,45 +1674,54 @@ impl Disc {
"Entered damage zone; dropping to minimum read speed"
);
}
let jump_pos = jump_lba as u64 * 2048;
let jump_pos = (pos + block_bytes + sectors * 2048).min(region_end);
let gap_start = pos + block_bytes;
let gap_bytes = jump_pos.saturating_sub(gap_start);
let jump_mb = gap_bytes / 1_048_576;
if gap_bytes > 0 {
let zero_gap = vec![0u8; 65536];
let mut filled: u64 = 0;
while filled < gap_bytes {
let chunk = (gap_bytes - filled).min(zero_gap.len() as u64);
file.seek(SeekFrom::Start(gap_start + filled))
.map_err(|e| Error::IoError { source: e })?;
file.write_all(&zero_gap[..chunk as usize])
.map_err(|e| Error::IoError { source: e })?;
filled += chunk;
}
map.record(
gap_start,
gap_bytes,
mapfile::SectorStatus::NonTrimmed,
)
.map_err(|e| Error::IoError { source: e })?;
bytes_done = bytes_done.saturating_add(gap_bytes);
}
tracing::warn!(
target: "freemkv::disc",
phase = "damage_jump",
from_lba = block_lba,
to_lba = jump_lba,
jump_mb,
bad_pct = bad_count * 100 / damage_window.len(),
multiplier = jump_multiplier,
"25%+ failures in last 50 blocks; jumping ahead"
to_lba = (jump_pos / 2048) as u32,
jump_mb = gap_bytes / 1_048_576,
"damage-jump"
);
if gap_bytes > 0 {
let zero = vec![0u8; 65536];
let mut filled: u64 = 0;
while filled < gap_bytes {
let chunk = (gap_bytes - filled).min(zero.len() as u64);
file.seek(SeekFrom::Start(gap_start + filled))
.map_err(|e| Error::IoError { source: e })?;
file.write_all(&zero[..chunk as usize])
.map_err(|e| Error::IoError { source: e })?;
filled += chunk;
}
map.record(gap_start, gap_bytes, mapfile::SectorStatus::NonTrimmed)
.map_err(|e| Error::IoError { source: e })?;
bytes_done = bytes_done.saturating_add(gap_bytes);
}
pos = jump_pos;
jump_multiplier *= 2;
did_skip_ahead = true;
if pause_secs > 0 {
std::thread::sleep(std::time::Duration::from_secs(pause_secs));
}
}
read_error::ReadAction::AbortPass => {
let (status, sense) = extract_scsi_context(&err);
return Err(Error::DiscRead {
sector: block_lba as u64,
status: Some(status),
sense,
});
}
}
}
}
if !did_skip_ahead {
pos += block_bytes;
}
iter_count += 1;
if iter_count - last_log_iter >= 100 {
@@ -2223,9 +2077,15 @@ impl Disc {
}
}
// Every retry pass acts on every non-Finished range. Including
// Unreadable means a sector that failed in pass N gets a fresh
// shot in pass N+1 — drive state evolves, the same read can
// succeed later. Each pass owns its own jumps/skips; if pass 5
// jumps over the same zone as pass 2, fine.
let mut bad_ranges = map.ranges_with(&[
mapfile::SectorStatus::NonTrimmed,
mapfile::SectorStatus::NonScraped,
mapfile::SectorStatus::Unreadable,
]);
if opts.reverse {
bad_ranges.reverse();
@@ -2866,9 +2726,23 @@ impl Disc {
if damage_window.len() >= PASSN_DAMAGE_WINDOW
&& bad_count * 100 / damage_window.len() >= PASSN_DAMAGE_THRESHOLD_PCT
{
let skip_sectors = (PASSN_SKIP_SECTORS_BASE
<< consecutive_skips_without_recovery)
// Size-aware cap: never skip more than 1/4 of the
// remaining bad range. A 100-sector bad range is
// really 25-bad + 50-good + 25-bad in disguise; a
// hardcoded MB-scale skip would leap over the
// entire thing and miss the good middle. Capping
// at range_remaining/4 forces convergence on the
// actual bad sub-zones.
let range_remaining_bytes = if opts.reverse {
block_end.saturating_sub(*range_pos)
} else {
end.saturating_sub(block_end)
};
let range_remaining_sectors = range_remaining_bytes / 2048;
let range_quarter = (range_remaining_sectors / 4).max(1);
let escalated = (PASSN_SKIP_SECTORS_BASE << consecutive_skips_without_recovery)
.min(PASSN_SKIP_SECTORS_CAP);
let skip_sectors = escalated.min(range_quarter);
let skip_bytes = skip_sectors * 2048;
let new_block_end = if opts.reverse {
block_end.saturating_sub(skip_bytes).max(*range_pos)
src/disc/read_error.rs
+374
View File
@@ -0,0 +1,374 @@
//! Single source of truth for what to do when a sector read fails.
//!
//! Both Pass 1 (`Disc::sweep`) and Pass 2-N (`Disc::patch`) call into
//! `handle_read_error` after every failed `read_sectors`. The handler
//! classifies the error, updates the in-flight context (counters,
//! damage window, retry budgets), and returns a `ReadAction` the caller
//! dispatches on. Every read goes through the same gate — no path can
//! silently skip pause/skip/jump/abort logic.
//!
//! Adding a new error class = add one arm in `handle_read_error`.
//! Adding new logging on errors = one place.
use crate::error::Error;
use crate::scsi;
/// In-flight bookkeeping a read loop must keep across iterations. The
/// handler reads and mutates this. Caller owns the storage.
pub struct ReadCtx {
/// Number of sectors per read attempt. The handler uses this to
/// decide whether to bisect (only worthwhile when batch > 1).
pub batch: u16,
/// Successful reads since the last failure. Resets to 0 on failure.
/// Used by callers to drive damage-zone exit / speed restoration.
pub consecutive_good: u64,
/// Failed reads since the last success. Resets to 0 on success.
/// Drives long-pause escalation on persistent failure.
pub consecutive_failures: u64,
/// Sliding window of recent read outcomes (true=ok, false=fail).
/// Capped at `damage_window_max`. Drives damage-jump decisions.
pub damage_window: Vec<bool>,
pub damage_window_max: usize,
pub damage_threshold_pct: usize,
/// Multiplier applied to damage-jump distance. Doubles each jump,
/// resets to 1 after `damage_window_max` consecutive good reads.
pub jump_multiplier: u64,
/// NOT_READY retries used so far for the current LBA. Reset to 0
/// on any non-NOT_READY response.
pub not_ready_retries: u32,
/// Bridge-degradation cooldowns used so far.
pub bridge_degradation_count: u32,
/// Whether we're currently inside a damage-jump bisect attempt.
/// Caller sets this true when entering single-sector mode for a
/// failed batch, so the handler doesn't recursively request another
/// bisect on the inner-sector failures.
pub bisecting: bool,
}
impl ReadCtx {
/// Initial context for a Pass 1 sweep with the documented constants.
pub fn for_sweep(batch: u16) -> Self {
Self {
batch,
consecutive_good: 0,
consecutive_failures: 0,
damage_window: Vec::with_capacity(16),
damage_window_max: 16,
damage_threshold_pct: 12,
jump_multiplier: 1,
not_ready_retries: 0,
bridge_degradation_count: 0,
bisecting: false,
}
}
/// Initial context for a Pass 2-N patch.
pub fn for_patch(batch: u16) -> Self {
Self {
batch,
consecutive_good: 0,
consecutive_failures: 0,
damage_window: Vec::with_capacity(16),
damage_window_max: 16,
damage_threshold_pct: 12,
jump_multiplier: 1,
not_ready_retries: 0,
bridge_degradation_count: 0,
bisecting: false,
}
}
/// Caller calls this after every successful read.
pub fn on_success(&mut self) {
self.consecutive_good += 1;
self.consecutive_failures = 0;
self.not_ready_retries = 0;
self.damage_window.push(true);
if self.damage_window.len() > self.damage_window_max {
self.damage_window.remove(0);
}
}
}
/// What the caller should do after a read failure. The caller owns the
/// I/O side-effects (sleep, write zeros, advance pos) — the handler
/// only decides which side-effects.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ReadAction {
/// Pause `pause_secs` then retry the same LBA / batch. Used for
/// transient conditions (NOT_READY, bridge degradation) that the
/// drive may recover from on its own.
Retry { pause_secs: u64 },
/// Re-issue the failed batch as `batch` single-sector reads. Each
/// inner read is itself dispatched through `handle_read_error` with
/// `bisecting = true` so it cannot recurse.
Bisect,
/// Mark the failed range NonTrimmed (zero-fill, retry in Pass N+),
/// then pause `pause_secs` before resuming the next LBA.
SkipBlock { pause_secs: u64 },
/// Mark the failed range NonTrimmed AND advance position by
/// `sectors` (zero-filling the gap as NonTrimmed). Then pause
/// `pause_secs`. Used when the damage-window threshold is crossed.
JumpAhead { sectors: u64, pause_secs: u64 },
/// Unrecoverable at this layer. Caller propagates `Err` up to the
/// outer pass loop / autorip, which can attempt USB re-enumeration,
/// drop session, etc.
AbortPass,
}
// Pause budget constants. Tuned from 2026-05-07 BU40N traces showing
// bridge wedges 524 ms after a 5.4-second internal ECC retry. The
// post-failure pauses give the drive — and the bridge — time to settle.
const POST_FAILURE_PAUSE_SECS: u64 = 1;
const CONSECUTIVE_FAIL_LONG_PAUSE_SECS: u64 = 5;
const CONSECUTIVE_FAIL_LONG_PAUSE_THRESHOLD: u64 = 10;
const POST_JUMP_EXTRA_PAUSE_SECS: u64 = 2;
const NOT_READY_PAUSE_SECS: u64 = 3;
const NOT_READY_MAX_RETRIES: u32 = 3;
const BRIDGE_DEGRADATION_PAUSE_SECS: u64 = 15;
const BRIDGE_DEGRADATION_MAX_RETRIES: u32 = 5;
/// THE single error-handling entry point. Updates `ctx`, returns the
/// action the caller must apply.
///
/// New error class = add a new arm here. New logging on errors = add
/// it once at the top. New retry policy = adjust the constants. No
/// other read site needs to change.
pub fn handle_read_error(err: &Error, ctx: &mut ReadCtx) -> ReadAction {
ctx.consecutive_failures += 1;
ctx.consecutive_good = 0;
tracing::warn!(
target: "freemkv::disc",
phase = "read_error",
consecutive_failures = ctx.consecutive_failures,
batch = ctx.batch,
bisecting = ctx.bisecting,
sense_key = err.scsi_sense().map(|s| s.sense_key),
asc = err.scsi_sense().map(|s| s.asc),
ascq = err.scsi_sense().map(|s| s.ascq),
error = %err,
"read failed; classifying"
);
// 1. Transport failure: bridge crash / USB disconnect. The outer
// pass loop knows how to handle this (rediscover sg path,
// re-open drive). Inline single-sector retry here was tried in
// pre-v0.17.0 builds and observed to make wedges worse.
if err.is_scsi_transport_failure() {
return ReadAction::AbortPass;
}
// 2. Bridge degradation: NOT_READY with the well-known signature
// (sense_key=2, ASC=0x04, ASCQ=0x3E). Drive's bridge is in a
// semi-stuck state but typically recovers after a long cooldown.
// If we've exhausted our retry budget, fall through to the
// marginal/skip path below.
if err.is_bridge_degradation() && ctx.bridge_degradation_count < BRIDGE_DEGRADATION_MAX_RETRIES
{
ctx.bridge_degradation_count += 1;
return ReadAction::Retry {
pause_secs: BRIDGE_DEGRADATION_PAUSE_SECS,
};
}
let sense_key = err.scsi_sense().map(|s| s.sense_key).unwrap_or(0);
// 3. Generic NOT_READY (other ASC codes): drive's mechanical
// pickup may be moving. Pause and retry briefly.
if sense_key == scsi::SENSE_KEY_NOT_READY && ctx.not_ready_retries < NOT_READY_MAX_RETRIES {
ctx.not_ready_retries += 1;
return ReadAction::Retry {
pause_secs: NOT_READY_PAUSE_SECS,
};
}
if sense_key != scsi::SENSE_KEY_NOT_READY {
ctx.not_ready_retries = 0;
}
// 4. Hardware / illegal request: the drive said "no, won't do it".
// Retrying won't change the answer. Surface to the outer layer
// so it can eject + prompt user.
if sense_key == scsi::SENSE_KEY_HARDWARE_ERROR || sense_key == scsi::SENSE_KEY_ILLEGAL_REQUEST {
return ReadAction::AbortPass;
}
// 5. Marginal media (MEDIUM_ERROR / ABORTED_COMMAND) on a multi-
// sector batch: the drive can often read the same sectors
// individually. Bisect into single-sector reads (gentler on the
// bridge too — shorter SCSI transactions). Avoid recursive
// bisect.
let is_marginal = matches!(
sense_key,
scsi::SENSE_KEY_MEDIUM_ERROR | scsi::SENSE_KEY_ABORTED_COMMAND
);
if is_marginal && ctx.batch > 1 && !ctx.bisecting {
return ReadAction::Bisect;
}
// 6. Single-sector failure or unbisectable error — record in
// damage window, decide between skip-in-place vs damage-jump.
//
// SKIP damage-window updates while bisecting: the window
// represents per-batch outcomes, not per-sector. Updating it
// inside a bisect inner loop (potentially 32+ sector failures
// per batch) would over-weight the window and cause runaway
// JumpAhead distance via excessive multiplier doublings.
if !ctx.bisecting {
ctx.damage_window.push(false);
if ctx.damage_window.len() > ctx.damage_window_max {
ctx.damage_window.remove(0);
}
}
let bad_count = ctx.damage_window.iter().filter(|&&b| !b).count();
let bad_pct = if ctx.damage_window.is_empty() {
0
} else {
bad_count * 100 / ctx.damage_window.len()
};
let pause_secs = if ctx.consecutive_failures >= CONSECUTIVE_FAIL_LONG_PAUSE_THRESHOLD {
CONSECUTIVE_FAIL_LONG_PAUSE_SECS
} else {
POST_FAILURE_PAUSE_SECS
};
// 7. Damage-jump: too many failures in window → skip ahead by an
// escalating gap. Multiplier capped so we can't accidentally
// skip the entire rest of the disc (observed 2026-05-07: a
// saturated multiplier produced a 56 GB jump). Saturating
// arithmetic on the sector calc as defence in depth.
const MAX_JUMP_MULTIPLIER: u64 = 64; // 64 × 256 × batch sectors
if ctx.damage_window.len() >= ctx.damage_window_max && bad_pct >= ctx.damage_threshold_pct {
let mult = ctx.jump_multiplier.min(MAX_JUMP_MULTIPLIER);
let sectors = 256u64.saturating_mul(ctx.batch as u64).saturating_mul(mult);
ctx.jump_multiplier = (ctx.jump_multiplier.saturating_mul(2)).min(MAX_JUMP_MULTIPLIER);
return ReadAction::JumpAhead {
sectors,
pause_secs: pause_secs + POST_JUMP_EXTRA_PAUSE_SECS,
};
}
// 8. Default: zero-fill the failed batch as NonTrimmed and pause
// before the next read.
ReadAction::SkipBlock { pause_secs }
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error::Error;
use crate::scsi::ScsiSense;
fn medium_err() -> Error {
Error::DiscRead {
sector: 100,
status: Some(2),
sense: Some(ScsiSense {
sense_key: scsi::SENSE_KEY_MEDIUM_ERROR,
asc: 0x11,
ascq: 0x05,
}),
}
}
fn hardware_err() -> Error {
Error::DiscRead {
sector: 100,
status: Some(2),
sense: Some(ScsiSense {
sense_key: scsi::SENSE_KEY_HARDWARE_ERROR,
asc: 0x44,
ascq: 0x00,
}),
}
}
#[test]
fn medium_error_with_batch_gt_1_bisects() {
let mut ctx = ReadCtx::for_sweep(32);
let action = handle_read_error(&medium_err(), &mut ctx);
assert_eq!(action, ReadAction::Bisect);
}
#[test]
fn medium_error_with_batch_1_skips() {
let mut ctx = ReadCtx::for_sweep(1);
let action = handle_read_error(&medium_err(), &mut ctx);
match action {
ReadAction::SkipBlock { pause_secs } => assert!(pause_secs >= 1),
other => panic!("expected SkipBlock, got {other:?}"),
}
}
#[test]
fn medium_error_while_bisecting_does_not_recurse() {
let mut ctx = ReadCtx::for_sweep(32);
ctx.bisecting = true;
let action = handle_read_error(&medium_err(), &mut ctx);
match action {
ReadAction::SkipBlock { .. } => {}
other => panic!("expected SkipBlock, got {other:?}"),
}
}
#[test]
fn hardware_error_aborts() {
let mut ctx = ReadCtx::for_sweep(32);
let action = handle_read_error(&hardware_err(), &mut ctx);
assert_eq!(action, ReadAction::AbortPass);
}
#[test]
fn long_failure_streak_extends_pause() {
let mut ctx = ReadCtx::for_sweep(1);
for _ in 0..15 {
handle_read_error(&medium_err(), &mut ctx);
}
// After many consecutive failures we should be in the long-pause regime
let final_action = handle_read_error(&medium_err(), &mut ctx);
match final_action {
ReadAction::SkipBlock { pause_secs } => {
assert!(pause_secs >= CONSECUTIVE_FAIL_LONG_PAUSE_SECS);
}
ReadAction::JumpAhead { pause_secs, .. } => {
assert!(pause_secs >= CONSECUTIVE_FAIL_LONG_PAUSE_SECS);
}
other => panic!("expected long-pause action, got {other:?}"),
}
}
#[test]
fn damage_window_fills_then_jumps() {
let mut ctx = ReadCtx::for_sweep(1);
ctx.damage_window_max = 4;
ctx.damage_threshold_pct = 50;
let mut saw_jump = false;
for _ in 0..6 {
let a = handle_read_error(&medium_err(), &mut ctx);
if matches!(a, ReadAction::JumpAhead { .. }) {
saw_jump = true;
break;
}
}
assert!(
saw_jump,
"expected at least one JumpAhead in 6 failures with 50% threshold"
);
}
#[test]
fn on_success_resets_failure_counters_and_pushes_window() {
let mut ctx = ReadCtx::for_sweep(32);
for _ in 0..3 {
handle_read_error(&medium_err(), &mut ctx);
}
assert!(ctx.consecutive_failures > 0);
ctx.bisecting = false;
ctx.on_success();
assert_eq!(ctx.consecutive_good, 1);
assert_eq!(ctx.consecutive_failures, 0);
assert_eq!(*ctx.damage_window.last().unwrap(), true);
}
}
tests/integration_progress_and_halt.rs
+10 -12
View File
@@ -594,7 +594,7 @@ impl SectorReader for BlockSizeFailingReader {
#[test]
fn test_disc_copy_marks_failed_ecc_blocks_as_nontrimmed() {
let capacity_sectors: u32 = 256;
let _total_bytes: u64 = capacity_sectors as u64 * SECTOR_SIZE as u64;
let total_bytes: u64 = capacity_sectors as u64 * SECTOR_SIZE as u64;
let mut reader = BlockSizeFailingReader {
capacity: capacity_sectors,
@@ -620,20 +620,18 @@ fn test_disc_copy_marks_failed_ecc_blocks_as_nontrimmed() {
let _ = std::fs::remove_file(libfreemkv::disc::mapfile_path_for(&iso_path));
// Pass 1 reads every batch at bpt=32 (no batch reduction, no skip-ahead).
// BlockSizeFailingReader fails on all batch=32 reads, so every sector
// is marked NonTrimmed. bytes_good=0 is correct — Pass 2 recovers them.
// BlockSizeFailingReader fails on multi-sector reads but succeeds on single-sector.
// Bridge degradation handling retries failed batches as individual sectors,
// so all data is recovered as Finished. bytes_good == total_bytes is correct.
assert_eq!(
result.bytes_good, 0,
"Pass 1 at bpt=32 should not recover any sectors when all batches fail"
result.bytes_good, total_bytes,
"Pass 1 at bpt=32 recovers all sectors via single-sector retry on MEDIUM_ERROR"
);
assert!(
result.bytes_pending > 0,
"all sectors should be NonTrimmed pending Pass 2"
);
assert!(
!result.complete,
"complete=false when sectors remain NonTrimmed"
assert_eq!(
result.bytes_pending, 0,
"no pending sectors after full recovery"
);
assert!(result.complete, "complete=true when all sectors recovered");
}
// ── 9. PassProgress carries separate unreadable vs pending byte counts ─────
tests/pass_n_size_aware_skip.rs
+272
View File
@@ -0,0 +1,272 @@
//! Pass N (Disc::patch) size-aware-skip targeted tests.
//!
//! The user's failure mode (2026-05-07): "what if we have a 100 sector zone
//! and its really 2 25 sector zones and we keep jumping over the good in
//! the middle." Today's pre-fix patch escalates skip-distance based on
//! `consecutive_skips_without_recovery` with hardcoded 32 → 4096 sector
//! caps. A 100-sector bad range whose actual layout is 25 bad + 50 good +
//! 25 bad would have the patch skip 32-4096 sectors after a couple of
//! failures, leaping over the entire range AND the good middle.
//!
//! The fix: cap each skip at `range_remaining/4`. These tests exercise
//! that boundary.
use libfreemkv::disc::CopyOptions;
use libfreemkv::disc::mapfile::{Mapfile, SectorStatus};
use libfreemkv::error::Result;
use libfreemkv::disc::DiscRegion;
use libfreemkv::{ContentFormat, Disc, DiscFormat, SectorReader};
use std::collections::HashSet;
use std::sync::{Arc, Mutex};
const SECTOR_SIZE: usize = 2048;
/// Reader where you specify exactly which LBAs return Err. Everything else
/// returns Ok with the LBA encoded in each byte for verification.
struct PatternedSectorReader {
capacity: u32,
bad_lbas: HashSet<u32>,
/// Trace every read so tests can assert what was actually attempted.
trace: Arc<Mutex<Vec<(u32, u16)>>>,
}
impl PatternedSectorReader {
fn new(capacity: u32, bad_lbas: HashSet<u32>) -> (Self, Arc<Mutex<Vec<(u32, u16)>>>) {
let trace = Arc::new(Mutex::new(Vec::new()));
(
Self {
capacity,
bad_lbas,
trace: trace.clone(),
},
trace,
)
}
}
impl SectorReader for PatternedSectorReader {
fn read_sectors(&mut self, lba: u32, count: u16, buf: &mut [u8], _recovery: bool) -> Result<usize> {
self.trace.lock().unwrap().push((lba, count));
// Whole-batch fails if ANY sector in the batch is bad. (Models a
// real drive: a multi-sector READ aborts on the first ECC failure.)
for offset in 0..count as u32 {
if self.bad_lbas.contains(&(lba + offset)) {
return Err(libfreemkv::error::Error::ScsiError {
opcode: libfreemkv::scsi::SCSI_READ_10,
status: libfreemkv::scsi::SCSI_STATUS_CHECK_CONDITION,
sense: Some(libfreemkv::ScsiSense {
sense_key: libfreemkv::scsi::SENSE_KEY_MEDIUM_ERROR,
asc: 0x11,
ascq: 0x00,
}),
});
}
}
for chunk in buf.chunks_mut(SECTOR_SIZE) {
chunk.fill((lba & 0xff) as u8);
}
Ok(buf.len())
}
fn capacity(&self) -> u32 {
self.capacity
}
}
fn synthetic_disc(capacity_sectors: u32) -> Disc {
Disc {
volume_id: String::new(),
meta_title: None,
format: DiscFormat::BluRay,
capacity_sectors,
capacity_bytes: capacity_sectors as u64 * SECTOR_SIZE as u64,
layers: 1,
titles: Vec::new(),
region: DiscRegion::Free,
aacs: None,
css: None,
encrypted: false,
content_format: ContentFormat::BdTs,
}
}
/// Pre-populate a mapfile with one large NonTrimmed range so patch's work-
/// list has something to do. Caller pre-allocates the ISO at `total_bytes`
/// so seeks don't fail.
fn prep_iso_and_mapfile(
iso_path: &std::path::Path,
total_bytes: u64,
finished_ranges: &[(u64, u64)],
nontrimmed_ranges: &[(u64, u64)],
) {
use std::fs::OpenOptions;
use std::io::{Seek, SeekFrom, Write};
let mut f = OpenOptions::new()
.create(true)
.write(true)
.truncate(true)
.open(iso_path)
.unwrap();
f.set_len(total_bytes).unwrap();
f.seek(SeekFrom::Start(0)).unwrap();
f.write_all(&[]).unwrap();
let map_path = libfreemkv::disc::mapfile_path_for(iso_path);
let mut mf = Mapfile::create(&map_path, total_bytes, "test").unwrap();
for &(pos, size) in finished_ranges {
mf.record(pos, size, SectorStatus::Finished).unwrap();
}
for &(pos, size) in nontrimmed_ranges {
mf.record(pos, size, SectorStatus::NonTrimmed).unwrap();
}
}
/// THE critical test. A 100-sector "bad" range hides 50 good sectors in
/// the middle (LBAs 125-174). Pre-fix patch would skip-escalate at 32+
/// sectors and leap over the whole range. Post-fix: skip is capped at
/// range_remaining/4 (=25 sectors initially), which forces convergence.
#[test]
fn patch_recovers_good_middle_of_a_bad_range() {
let capacity_sectors: u32 = 1024;
let total_bytes: u64 = capacity_sectors as u64 * SECTOR_SIZE as u64;
// Bad range layout: LBAs 100-124 bad, 125-174 GOOD, 175-199 bad.
let mut bad_lbas = HashSet::new();
for lba in 100..125 {
bad_lbas.insert(lba);
}
for lba in 175..200 {
bad_lbas.insert(lba);
}
let (mut reader, _trace) = PatternedSectorReader::new(capacity_sectors, bad_lbas);
let disc = synthetic_disc(capacity_sectors);
let tmp = tempfile::NamedTempFile::new().unwrap();
let iso_path = tmp.path().to_path_buf();
drop(tmp);
// Pre-populate: 0..100 already Finished from an imagined Pass 1,
// 100..200 NonTrimmed (the range we want patch to retry),
// 200..1024 already Finished.
let finished = [(0, 100 * 2048), (200 * 2048, (capacity_sectors as u64 - 200) * 2048)];
let nontrimmed = [(100 * 2048, 100 * 2048)];
prep_iso_and_mapfile(&iso_path, total_bytes, &finished, &nontrimmed);
// Run patch.
// disc.copy() with multipass=true auto-dispatches to patch when the
// mapfile already covers the disc and has retryable ranges.
let opts = CopyOptions {
decrypt: false,
multipass: true,
..Default::default()
};
let pr = disc.copy(&mut reader, &iso_path, &opts).expect("copy returns Ok");
// Re-load mapfile and inspect.
let map_path = libfreemkv::disc::mapfile_path_for(&iso_path);
let map = Mapfile::load(&map_path).unwrap();
// The good middle (125..175) MUST end up Finished. If size-aware skip
// is not enabled, patch would skip 32+ sectors after a few failures
// and leap clean over LBA 125 → middle stays NonTrimmed/Unreadable.
let finished_ranges = map.ranges_with(&[SectorStatus::Finished]);
let total_finished_in_middle: u64 = finished_ranges
.iter()
.map(|&(pos, sz)| {
let start = pos.max(125 * 2048);
let end = (pos + sz).min(175 * 2048);
end.saturating_sub(start)
})
.sum();
// Allow 2 sectors (4 KB) of boundary slop — patch's bisection may
// not converge exactly on the good/bad boundary in a single pass,
// and that's acceptable. The pre-fix behaviour would have left the
// entire good middle as NonTrimmed (~0 bytes recovered).
let good_middle_bytes: u64 = 50 * 2048;
let min_acceptable: u64 = good_middle_bytes - 2 * 2048;
// Cleanup before assertions
let _ = std::fs::remove_file(&iso_path);
let _ = std::fs::remove_file(&map_path);
assert!(
total_finished_in_middle >= min_acceptable,
"size-aware skip should have discovered most of the 50 good sectors in the middle. \
Recovered {} of {} good middle bytes (min acceptable {}). bytes_good={} bytes_total={}",
total_finished_in_middle,
good_middle_bytes,
min_acceptable,
pr.bytes_good,
pr.bytes_total,
);
}
/// A second test: a bad range that's actually 4 small bad sub-zones
/// separated by good sectors. Demonstrates the bisection behaviour
/// converges when zones are non-uniform.
#[test]
fn patch_recovers_multiple_good_middles() {
let capacity_sectors: u32 = 2048;
let total_bytes: u64 = capacity_sectors as u64 * SECTOR_SIZE as u64;
// Bad pattern: 1000-1024 bad, 1025-1099 good, 1100-1124 bad,
// 1125-1199 good, 1200-1224 bad, 1225-1299 good.
let mut bad_lbas = HashSet::new();
for lba in 1000..1025 {
bad_lbas.insert(lba);
}
for lba in 1100..1125 {
bad_lbas.insert(lba);
}
for lba in 1200..1225 {
bad_lbas.insert(lba);
}
let (mut reader, _trace) = PatternedSectorReader::new(capacity_sectors, bad_lbas);
let disc = synthetic_disc(capacity_sectors);
let tmp = tempfile::NamedTempFile::new().unwrap();
let iso_path = tmp.path().to_path_buf();
drop(tmp);
let finished = [(0, 1000 * 2048), (1300 * 2048, (capacity_sectors as u64 - 1300) * 2048)];
let nontrimmed = [(1000 * 2048, 300 * 2048)];
prep_iso_and_mapfile(&iso_path, total_bytes, &finished, &nontrimmed);
let opts = CopyOptions {
decrypt: false,
multipass: true,
..Default::default()
};
let pr = disc.copy(&mut reader, &iso_path, &opts).expect("copy returns Ok");
let map_path = libfreemkv::disc::mapfile_path_for(&iso_path);
let map = Mapfile::load(&map_path).unwrap();
let finished_ranges = map.ranges_with(&[SectorStatus::Finished]);
let recovered: u64 = finished_ranges
.iter()
.map(|&(pos, sz)| {
let start = pos.max(1000 * 2048);
let end = (pos + sz).min(1300 * 2048);
end.saturating_sub(start)
})
.sum();
let _ = std::fs::remove_file(&iso_path);
let _ = std::fs::remove_file(&map_path);
// Three good middles of 75 sectors each = 225 good sectors in the
// bad range. Total bad = 75. So we want at least most of 225 sectors
// (= 460800 bytes) to be Finished after patch.
let target = 200 * 2048; // be generous — anything over 200 sectors is convincing
assert!(
recovered >= target,
"size-aware skip should find most of the 3 good middles. \
Recovered {} bytes; expected ≥ {}. bytes_good={} bytes_total={}",
recovered,
target,
pr.bytes_good,
pr.bytes_total,
);
}
tests/scsi_recovery.rs
-1
View File
@@ -9,7 +9,6 @@
//! These tests require a real /dev/sg* device and are therefore #[ignore].
use std::path::Path;
use std::time::Duration;
#[test]
#[ignore]