v0.13.45: multipass adaptive probe, NOT_READY retry, progress bytes_bad_total

- Adaptive probe algorithm in Disc::copy skip_on_error mode: after 4
  consecutive errors, probe 1 sector at 256x batch (8 MB) ahead. If
  good, zero-fill gap, mark NonTrimmed, jump. Clears bad zones in
  seconds instead of hours.
- NOT_READY sense key (0x02) now retries up to 3x with 3s pause before
  marking NonTrimmed. BU40N returns NOT READY for bad sectors, not
  MEDIUM ERROR.
- PassProgress struct gains bytes_bad_total field for consumer-side
  bad/retryable byte counts.
- Mapfile header version string fix: no longer duplicates 'libfreemkv v'
  prefix on each write.
- Structured sense_key/asc/ascq logging in copy error path.
This commit is contained in:
MattJackson
2026-04-29 19:41:34 -07:00
parent 9e3d0f1383
commit a6f1bd19bc
4 changed files with 93 additions and 105 deletions
+1 -1
View File
@@ -318,7 +318,7 @@ impl Mapfile {
let mut w = std::io::BufWriter::new(file); let mut w = std::io::BufWriter::new(file);
writeln!( writeln!(
w, w,
"# Rescue Logfile. Created by libfreemkv v{}", "# Rescue Logfile. Created by {}",
self.version self.version
)?; )?;
writeln!(w, "# Current pos / status / pass / pass_time")?; writeln!(w, "# Current pos / status / pass / pass_time")?;
+79 -90
View File
@@ -1249,7 +1249,7 @@ impl Disc {
let _ = std::fs::remove_file(&mapfile_path); let _ = std::fs::remove_file(&mapfile_path);
} }
let mut map = let mut map =
mapfile::Mapfile::open_or_create(&mapfile_path, total_bytes, env!("CARGO_PKG_VERSION")) mapfile::Mapfile::open_or_create(&mapfile_path, total_bytes, concat!("libfreemkv v", env!("CARGO_PKG_VERSION")))
.map_err(|e| Error::IoError { source: e })?; .map_err(|e| Error::IoError { source: e })?;
// ISO file: if resuming and mapfile has Finished ranges, open existing; // ISO file: if resuming and mapfile has Finished ranges, open existing;
@@ -1281,15 +1281,15 @@ impl Disc {
let mut buf = vec![0u8; batch as usize * 2048]; let mut buf = vec![0u8; batch as usize * 2048];
let mut bytes_done = 0u64; let mut bytes_done = 0u64;
let mut halt_requested = false; let mut halt_requested = false;
let mut current_batch = batch;
let mut consecutive_ok_since_error: u64 = 0;
let mut consecutive_errors: u64 = 0;
let mut skip_power: u32 = 0;
let copy_t0 = std::time::Instant::now(); let copy_t0 = std::time::Instant::now();
let mut iter_count: u64 = 0; let mut iter_count: u64 = 0;
let mut read_ok_count: u64 = 0; let mut read_ok_count: u64 = 0;
let mut read_err_count: u64 = 0; let mut read_err_count: u64 = 0;
let mut last_log_iter: u64 = 0; let mut last_log_iter: u64 = 0;
let mut not_ready_retries: u32 = 0;
const NOT_READY_MAX_RETRIES: u32 = 3;
let mut error_count_in_zone: u64 = 0;
let mut probe_buf = vec![0u8; 2048];
tracing::trace!( tracing::trace!(
target: "freemkv::disc", target: "freemkv::disc",
phase = "copy_start", phase = "copy_start",
@@ -1337,7 +1337,7 @@ impl Disc {
} }
} }
let block_bytes = (region_end - pos).min(current_batch as u64 * 2048); let block_bytes = (region_end - pos).min(batch as u64 * 2048);
let block_lba = (pos / 2048) as u32; let block_lba = (pos / 2048) as u32;
let block_count = (block_bytes / 2048) as u16; let block_count = (block_bytes / 2048) as u16;
let recovery = !opts.skip_on_error; let recovery = !opts.skip_on_error;
@@ -1349,31 +1349,11 @@ impl Disc {
recovery, recovery,
); );
let mut did_skip_ahead = false;
if read_result.is_ok() { if read_result.is_ok() {
read_ok_count += 1; read_ok_count += 1;
consecutive_ok_since_error += 1; error_count_in_zone = 0;
consecutive_errors = 0;
skip_power = 0;
if current_batch < batch
&& consecutive_ok_since_error >= COPY_BATCH_RESTORE_STREAK
{
let next_batch = (current_batch * 2).min(batch);
tracing::info!(
target: "freemkv::disc",
phase = "batch_restore",
prev_batch = current_batch,
batch = next_batch,
lba = block_lba,
streak = consecutive_ok_since_error,
"graduated batch restore"
);
current_batch = next_batch;
if (current_batch as usize * 2048) > buf.len() {
buf.resize(current_batch as usize * 2048, 0);
}
consecutive_ok_since_error = 0;
}
if opts.decrypt { if opts.decrypt {
crate::decrypt::decrypt_sectors( crate::decrypt::decrypt_sectors(
@@ -1403,19 +1383,6 @@ impl Disc {
} else { } else {
let err = read_result.err().unwrap(); let err = read_result.err().unwrap();
read_err_count += 1; read_err_count += 1;
consecutive_ok_since_error = 0;
consecutive_errors += 1;
if current_batch > 1 {
current_batch = 1;
tracing::warn!(
target: "freemkv::disc",
phase = "batch_reduce",
lba = block_lba,
prev_batch = block_count,
"dropping to single-sector reads after error"
);
}
if err.is_scsi_transport_failure() { if err.is_scsi_transport_failure() {
tracing::warn!( tracing::warn!(
@@ -1428,18 +1395,36 @@ impl Disc {
return Err(err); return Err(err);
} }
if !err.is_marginal_read() let sense = err.scsi_sense();
&& err.scsi_sense().is_none_or(|s| !s.is_medium_error()) let sense_key = sense.map(|s| s.sense_key).unwrap_or(0);
{ let asc = sense.map(|s| s.asc).unwrap_or(0);
return Err(err); let ascq = sense.map(|s| s.ascq).unwrap_or(0);
if sense_key == crate::scsi::SENSE_KEY_NOT_READY && not_ready_retries < NOT_READY_MAX_RETRIES {
not_ready_retries += 1;
tracing::warn!(
target: "freemkv::disc",
phase = "not_ready_pause",
lba = block_lba,
sense_key,
asc,
ascq,
retry = not_ready_retries,
"NOT READY; pausing 3s then retrying"
);
std::thread::sleep(std::time::Duration::from_secs(3));
continue;
} }
not_ready_retries = 0;
tracing::warn!( tracing::warn!(
target: "freemkv::disc", target: "freemkv::disc",
phase = "skip_ecc_block", phase = "skip_ecc_block",
lba = block_lba, lba = block_lba,
sectors = block_count, sectors = block_count,
consecutive_errors, sense_key,
asc,
ascq,
error = %err, error = %err,
"ECC block failed; marking NonTrimmed" "ECC block failed; marking NonTrimmed"
); );
@@ -1452,46 +1437,55 @@ impl Disc {
.map_err(|e| Error::IoError { source: e })?; .map_err(|e| Error::IoError { source: e })?;
bytes_done = bytes_done.saturating_add(block_bytes); bytes_done = bytes_done.saturating_add(block_bytes);
let pause_ms = opts.error_pause_ms.unwrap_or(COPY_ERROR_PAUSE_MS); error_count_in_zone += 1;
if pause_ms > 0 { if error_count_in_zone >= 4 {
std::thread::sleep(std::time::Duration::from_millis(pause_ms)); let probe_offset_sectors = 256 * batch as u64;
} let probe_lba = ((pos / 2048) + probe_offset_sectors) as u32;
let probe_end_lba = (region_end / 2048) as u32;
if consecutive_errors >= COPY_SKIP_THRESHOLD { if probe_lba < probe_end_lba {
let skip_sectors = match reader.read_sectors(probe_lba, 1, &mut probe_buf, false) {
(COPY_SKIP_BASE_SECTORS << skip_power).min(COPY_SKIP_MAX_SECTORS); Ok(_) => {
let available = region_end.saturating_sub(pos + block_bytes); let jump_pos = probe_lba as u64 * 2048;
let skip_bytes = (skip_sectors as u64 * 2048).min(available); let gap_start = pos + block_bytes;
if skip_bytes > 0 { let gap_bytes = jump_pos.saturating_sub(gap_start);
let skip_lba = ((pos + block_bytes) / 2048) as u32; let jump_mb = gap_bytes / 1_048_576;
tracing::warn!( if gap_bytes > 0 {
target: "freemkv::disc", tracing::warn!(
phase = "skip_ahead", target: "freemkv::disc",
from_lba = skip_lba, phase = "probe_jump",
skip_sectors, from_lba = block_lba,
skip_power, to_lba = probe_lba,
consecutive_errors, jump_mb,
"skipping ahead through bad zone" errors = error_count_in_zone,
); "probe found good media; jumping ahead"
let skip_zero = vec![0u8; skip_bytes as usize]; );
file.seek(SeekFrom::Start(pos + block_bytes)) let zero = vec![0u8; 65536];
.map_err(|e| Error::IoError { source: e })?; let mut filled: u64 = 0;
file.write_all(&skip_zero) while filled < gap_bytes {
.map_err(|e| Error::IoError { source: e })?; let chunk = (gap_bytes - filled).min(zero.len() as u64);
map.record( file.seek(SeekFrom::Start(gap_start + filled))
pos + block_bytes, .map_err(|e| Error::IoError { source: e })?;
skip_bytes, file.write_all(&zero[..chunk as usize])
mapfile::SectorStatus::NonTrimmed, .map_err(|e| Error::IoError { source: e })?;
) filled += chunk;
.map_err(|e| Error::IoError { source: e })?; }
bytes_done = bytes_done.saturating_add(skip_bytes); map.record(gap_start, gap_bytes, mapfile::SectorStatus::NonTrimmed)
pos += skip_bytes; .map_err(|e| Error::IoError { source: e })?;
skip_power = skip_power.saturating_add(1); bytes_done = bytes_done.saturating_add(gap_bytes);
}
pos = jump_pos;
did_skip_ahead = true;
error_count_in_zone = 0;
}
Err(_) => {}
}
} }
} }
} }
pos += block_bytes; if !did_skip_ahead {
pos += block_bytes;
}
iter_count += 1; iter_count += 1;
if iter_count - last_log_iter >= 100 { if iter_count - last_log_iter >= 100 {
@@ -1519,6 +1513,7 @@ impl Disc {
work_done: pos, work_done: pos,
work_total: total_bytes, work_total: total_bytes,
bytes_good_total: stats.bytes_good, bytes_good_total: stats.bytes_good,
bytes_bad_total: stats.bytes_unreadable + stats.bytes_retryable,
bytes_total_disc: total_bytes, bytes_total_disc: total_bytes,
}); });
} }
@@ -1560,7 +1555,6 @@ pub struct CopyOptions<'a> {
pub skip_on_error: bool, pub skip_on_error: bool,
pub progress: Option<&'a dyn crate::progress::Progress>, pub progress: Option<&'a dyn crate::progress::Progress>,
pub halt: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>, pub halt: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
pub error_pause_ms: Option<u64>,
} }
/// Result of `Disc::copy`. `complete=true` means every byte reached a terminal /// Result of `Disc::copy`. `complete=true` means every byte reached a terminal
@@ -1827,6 +1821,7 @@ impl Disc {
work_done, work_done,
work_total, work_total,
bytes_good_total: s.bytes_good, bytes_good_total: s.bytes_good,
bytes_bad_total: s.bytes_unreadable + s.bytes_retryable,
bytes_total_disc: total_bytes, bytes_total_disc: total_bytes,
}); });
} }
@@ -1866,12 +1861,6 @@ const MAX_BATCH_SECTORS: u16 = 510;
const DEFAULT_BATCH_SECTORS: u16 = 60; const DEFAULT_BATCH_SECTORS: u16 = 60;
const MIN_BATCH_SECTORS: u16 = 3; const MIN_BATCH_SECTORS: u16 = 3;
const COPY_ERROR_PAUSE_MS: u64 = 2000;
const COPY_SKIP_THRESHOLD: u64 = 8;
const COPY_SKIP_BASE_SECTORS: u32 = 32;
const COPY_SKIP_MAX_SECTORS: u32 = 8192;
const COPY_BATCH_RESTORE_STREAK: u64 = 200;
/// Coarse damage tier for a finished or in-progress rip. Maps the /// Coarse damage tier for a finished or in-progress rip. Maps the
/// observable signals (bad sector count + lost wallclock playback time) /// observable signals (bad sector count + lost wallclock playback time)
/// onto a small discrete classification so UIs can render a colored badge /// onto a small discrete classification so UIs can render a colored badge
+3
View File
@@ -47,6 +47,9 @@ pub struct PassProgress {
/// Cumulative bytes confirmed clean (`Finished` mapfile state) across /// Cumulative bytes confirmed clean (`Finished` mapfile state) across
/// every pass run on this rip. Doesn't change across pass boundaries. /// every pass run on this rip. Doesn't change across pass boundaries.
pub bytes_good_total: u64, pub bytes_good_total: u64,
/// Cumulative bytes marked bad (`NonTrimmed` + `NonScraped` +
/// `Unreadable`) across every pass run on this rip.
pub bytes_bad_total: u64,
/// Total disc capacity in bytes. Constant. /// Total disc capacity in bytes. Constant.
pub bytes_total_disc: u64, pub bytes_total_disc: u64,
} }
+10 -14
View File
@@ -439,7 +439,7 @@ fn test_disc_copy_completes_full_disc_with_failing_reader() {
let opts = CopyOptions { let opts = CopyOptions {
decrypt: false, decrypt: false,
skip_on_error: true, skip_on_error: true,
error_pause_ms: Some(0),
..Default::default() ..Default::default()
}; };
@@ -519,7 +519,7 @@ fn test_disc_copy_halts_promptly_on_failing_reader() {
let opts = CopyOptions { let opts = CopyOptions {
decrypt: false, decrypt: false,
skip_on_error: true, skip_on_error: true,
error_pause_ms: Some(0),
halt: Some(halt), halt: Some(halt),
..Default::default() ..Default::default()
}; };
@@ -607,7 +607,7 @@ fn test_disc_copy_marks_failed_ecc_blocks_as_nontrimmed() {
let opts = CopyOptions { let opts = CopyOptions {
decrypt: false, decrypt: false,
skip_on_error: true, skip_on_error: true,
error_pause_ms: Some(0),
..Default::default() ..Default::default()
}; };
@@ -618,20 +618,16 @@ fn test_disc_copy_marks_failed_ecc_blocks_as_nontrimmed() {
let _ = std::fs::remove_file(&iso_path); let _ = std::fs::remove_file(&iso_path);
let _ = std::fs::remove_file(libfreemkv::disc::mapfile_path_for(&iso_path)); let _ = std::fs::remove_file(libfreemkv::disc::mapfile_path_for(&iso_path));
// With graduated batch restore, Pass 1 drops to batch=1 after the // Pass 1 reads every batch at bpt=32 (no batch reduction, no skip-ahead).
// first batch=32 failure, reads individually (count=1 succeeds for // BlockSizeFailingReader fails on all batch=32 reads, so every sector
// BlockSizeFailingReader), and recovers those sectors. After 200 OK // is marked NonTrimmed. bytes_good=0 is correct — Pass 2 recovers them.
// at batch=1, it tries batch=2 which fails again. Net result: most assert_eq!(
// sectors are recovered (Finished), only the batch>1 failures produce result.bytes_good, 0,
// NonTrimmed blocks. "Pass 1 at bpt=32 should not recover any sectors when all batches fail"
assert!(
result.bytes_good > 0,
"Pass 1 should recover batch=1-readable sectors. Got bytes_good={}",
result.bytes_good
); );
assert!( assert!(
result.bytes_pending > 0, result.bytes_pending > 0,
"batch>1 failures should produce NonTrimmed pending Pass 2" "all sectors should be NonTrimmed pending Pass 2"
); );
assert!( assert!(
!result.complete, !result.complete,