pq/libfreemkv
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

0.31.0: hardening and correctness pass across mux, codec, AACS/CSS, UDF/MPLS/CLPI, recovery, drive/SCSI, labels, and I/O

Browse Source 
Library-wide review-and-fix pass: tightened AACS keydb/handshake/variant
handling and trailing-partial-unit policy, corrected MPLS mark offset and
added UDF allocation bounds, hardened the mux/codec framing and M2TS paths,
guarded SCSI READ CAPACITY short transfers and unified error mapping, added
overflow guards on untrusted disc input, and made prefetch shutdown
deterministic. Release profile now builds with thin LTO + single codegen unit.
This commit is contained in:
Matthew Jackson
2026-06-07 17:37:38 -07:00
parent 5b6ea8f5c4
commit 061f68594a
128 changed files with 11838 additions and 3831 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/disc/bluray.rs
+67 -21
View File
@@ -30,6 +30,12 @@ impl Disc {
titles
}
/// Parse one MPLS playlist into a [`DiscTitle`].
///
/// Sums PlayItem durations; returns `None` if the playlist is under
/// 30 seconds (skips menu / clip-info stub playlists) or fails to
/// parse. Physical sector extents are pulled from the UDF allocation
/// descriptors of each referenced `.m2ts` (deduplicated by clip_id).
pub(super) fn parse_playlist(
reader: &mut dyn SectorSource,
udf_fs: &udf::UdfFs,
@@ -55,27 +61,41 @@ impl Disc {
let mut extents = Vec::new();
let mut total_size: u64 = 0;
let mut clips = Vec::with_capacity(parsed.play_items.len());
// BD playlists legally reference the same .m2ts clip_id from
// multiple PlayItems (multi-angle, seamless splits, looped
// segments). The physical extents and packet count must be
// counted ONCE per unique clip — mux reads extents in order, so
// a duplicate would mux the A/V twice and inflate size_bytes.
// Per-PlayItem Clip entries (differing in/out times) still get
// recorded.
let mut seen_clips: std::collections::HashSet<String> = std::collections::HashSet::new();
for play_item in &parsed.play_items {
let clip_dur = play_item.out_time.saturating_sub(play_item.in_time) as f64 / 45000.0;
let mut pkt_count: u32 = 0;
let first_ref = seen_clips.insert(play_item.clip_id.clone());
let clpi_path = format!("/BDMV/CLIPINF/{}.clpi", play_item.clip_id);
if let Ok(clpi_data) = udf_fs.read_file(reader, &clpi_path) {
if let Ok(clip_info) = clpi::parse(&clpi_data) {
pkt_count = clip_info.source_packet_count;
total_size += pkt_count as u64 * 192;
// Get m2ts file extents from UDF allocation descriptors.
// Dual-layer discs split files across layers — UDF knows the real layout.
let m2ts_path = format!("/BDMV/STREAM/{}.m2ts", play_item.clip_id);
if let Ok(file_exts) = udf_fs.file_extents(reader, &m2ts_path) {
for (lba, sectors) in file_exts {
if sectors > 0 && lba > 0 {
extents.push(Extent {
start_lba: lba,
sector_count: sectors,
});
// Only fetch/push the physical extents and add to the
// total size the first time this clip_id is seen.
if first_ref {
total_size += pkt_count as u64 * 192;
// Get m2ts file extents from UDF allocation descriptors.
// Dual-layer discs split files across layers — UDF knows the real layout.
let m2ts_path = format!("/BDMV/STREAM/{}.m2ts", play_item.clip_id);
if let Ok(file_exts) = udf_fs.file_extents(reader, &m2ts_path) {
for (lba, sectors) in file_exts {
if sectors > 0 && lba > 0 {
extents.push(Extent {
start_lba: lba,
sector_count: sectors,
});
}
}
}
}
@@ -170,23 +190,49 @@ impl Disc {
})
.collect();
// Convert marks to chapters (mark_type 0 or 1 = chapter entry, 2 = link)
let first_in_time = parsed.play_items.first().map(|pi| pi.in_time).unwrap_or(0);
// Convert marks to chapters. mark_type == 1 is an entry-mark
// (chapter); type 2 is a link point and type 0 is reserved, so
// neither is a chapter.
//
// Each mark's timestamp is in the timebase of the PlayItem it
// references (play_item_ref). The chapter's position on the
// muxed timeline is the summed duration of every preceding
// PlayItem plus the mark's offset within its own PlayItem. Using
// play_items[0].in_time for every mark would misplace chapters in
// multi-PlayItem playlists.
let chapters: Vec<Chapter> = parsed
.marks
.iter()
.filter(|m| m.mark_type <= 1)
.enumerate()
.map(|(i, m)| {
let time_secs = (m.timestamp as f64 - first_in_time as f64) / 45000.0;
Chapter {
.filter(|m| m.mark_type == 1)
.filter_map(|m| {
let pi_idx = m.play_item_ref as usize;
let pi = parsed.play_items.get(pi_idx)?;
let preceding: f64 = parsed.play_items[..pi_idx]
.iter()
.map(|p| p.out_time.saturating_sub(p.in_time) as f64 / 45000.0)
.sum();
let within = (m.timestamp as f64 - pi.in_time as f64) / 45000.0;
let time_secs = preceding + within;
Some(Chapter {
time_secs: if time_secs < 0.0 { 0.0 } else { time_secs },
name: format!("Chapter {}", i + 1),
}
name: String::new(), // filled with the ordinal below
})
})
.enumerate()
.map(|(i, mut ch)| {
ch.name = super::chapter_name(i);
ch
})
.collect();
let playlist_num = filename.trim_end_matches(".mpls").trim_end_matches(".MPLS");
// Strip the .mpls suffix case-insensitively before parsing the
// numeric playlist id (the dir scan accepts any-case .mpls).
let playlist_num = filename
.get(..filename.len().saturating_sub(5))
.filter(|_| {
filename.len() >= 5 && filename[filename.len() - 5..].eq_ignore_ascii_case(".mpls")
})
.unwrap_or(filename);
let playlist_id = playlist_num.parse::<u16>().unwrap_or(0);
Some(DiscTitle {
src/disc/dvd.rs
+21 -4
View File
@@ -34,15 +34,27 @@ impl Disc {
label: String::new(),
});
// Map DvdAudioAttr to Stream::Audio
// Map DvdAudioAttr to Stream::Audio. The PID is derived from the
// stream's REAL on-wire private_stream_1 sub-stream id (assigned
// by per-codec ordinal in the IFO scan) via the same
// `dvd_audio_pid` table the demuxer's `PsPacket::dvd_pid` uses,
// so a mixed-codec title (AC-3 + DTS + LPCM) routes correctly
// instead of colliding on 0xBD00. Streams carried as a regular
// MPEG-audio PES (MP1/MP2, no sub-id) fall back to a distinct
// 0xBD00+ordinal PID — disjoint from the 0xBD80+ canonical audio
// space — though they are not routed via `dvd_pid` today.
let audio_streams: Vec<Stream> = ts
.audio_streams
.iter()
.enumerate()
.map(|(i, a)| {
let codec = a.codec;
let pid = a
.sub_stream_id
.and_then(crate::mux::ps::dvd_audio_pid)
.unwrap_or(0xBD00 + i as u16);
Stream::Audio(AudioStream {
pid: 0xBD00 + i as u16, // DVD private stream 1 sub-IDs
pid,
codec,
channels: AudioChannels::from_count(a.channels),
language: a.language.clone(),
@@ -88,8 +100,13 @@ impl Disc {
.iter()
.enumerate()
.map(|(i, s)| {
// VobSub sub-stream ids run 0x20..=0x3F; PID = sub-id
// (identity), shared with the demuxer via
// `dvd_subtitle_pid`.
let sub_id = 0x20u8.saturating_add(i.min(0x1F) as u8);
let pid = crate::mux::ps::dvd_subtitle_pid(sub_id).unwrap_or(sub_id as u16);
Stream::Subtitle(SubtitleStream {
pid: 0x20 + i as u16, // DVD sub-stream IDs 0x20-0x3F
pid,
codec: Codec::DvdSub,
language: s.language.clone(),
forced: false,
@@ -109,7 +126,7 @@ impl Disc {
.enumerate()
.map(|(i, &t)| Chapter {
time_secs: t,
name: format!("Chapter {}", i + 1),
name: chapter_name(i),
})
.collect();
src/disc/encrypt.rs
+45 -33
View File
@@ -88,7 +88,7 @@ impl Disc {
}
let mut vid = [0u8; 16];
vid.copy_from_slice(&buf[4..20]);
tracing::warn!(
tracing::debug!(
target: "freemkv::disc",
phase = "oem_vid_ok",
"OEM VID retrieved"
@@ -119,10 +119,10 @@ impl Disc {
///
/// Returns `(handshake, error)`:
/// * `(Some(_), None)` — VID acquired
/// * `(None, Some(_))` — specific failure mode (see
/// `AacsHostCertRejected` / `AacsRawReadUnsupported` /
/// `AacsVidUnavailable` / `DriveProfileMissing` /
/// `VidCdbUnavailable` variants in `error.rs`)
/// * `(None, Some(_))` — specific failure mode; only
/// `AacsHostCertRejected` and `AacsVidUnavailable` are returned
/// here (the OEM-path `DriveProfileMissing` / `VidCdbUnavailable`
/// errors are caught internally and fall through to cert auth)
/// * `(None, None)` — handshake not attempted (no keydb;
/// resolution will proceed with VID=zero and rely on path 1
/// disc-hash → VUK lookup)
@@ -131,7 +131,7 @@ impl Disc {
opts: &ScanOptions,
) -> (Option<HandshakeResult>, Option<Error>) {
let unlocked = session.is_unlocked();
tracing::warn!(
tracing::debug!(
target: "freemkv::disc",
phase = "handshake_entry",
unlocked,
@@ -199,25 +199,24 @@ impl Disc {
};
let host_cert_count = host_certs.len();
tracing::warn!(
tracing::debug!(
target: "freemkv::disc",
phase = "handshake_start",
host_cert_count,
"handshake starting"
);
// v0.25.7 wedge fix. Pre-0.25.7 this loop fired up to 16 AACS
// authenticate attempts back-to-back with no pause. Each attempt
// is 5-10 SCSI REPORT_KEY/SEND_KEY exchanges. On a disc whose
// host cert isn't in our KEYDB (or one the drive rejects),
// that's 80-160 SCSI commands hammered at the drive in a
// few hundred milliseconds — and the BU40N (and most consumer
// optical drives) responds by entering a fast-fail firmware
// wedge state where every subsequent CDB returns
// ILLEGAL_REQUEST/INVALID_FIELD_IN_CDB (sense 05/24) until
// power-cycled. Hit live on rip1 2026-05-20 during a MOVIE
// UHD scan: KEYDB miss → 16 cert attempts in a tight loop →
// wedge → forced host reboot + drive disconnect to recover.
// Cert-attempt wedge guard. An earlier version fired up to 16
// AACS authenticate attempts back-to-back with no pause. Each
// attempt is 5-10 SCSI REPORT_KEY/SEND_KEY exchanges. On a disc
// whose host cert isn't in the KEYDB (or one the drive rejects),
// that's 80-160 SCSI commands hammered at the drive in a few
// hundred milliseconds — and consumer optical drives can respond
// by entering a fast-fail firmware wedge state where every
// subsequent CDB returns ILLEGAL_REQUEST/INVALID_FIELD_IN_CDB
// (sense 05/24) until power-cycled. Observed live on a UHD scan:
// KEYDB miss → many cert attempts in a tight loop → wedge →
// forced power cycle to recover.
//
// Defense-in-depth: cap attempts, sleep between, and bail
// early on the drive's wedge sense so any later regression
@@ -262,20 +261,31 @@ impl Disc {
);
}
Err(e) => {
let code = e.code();
last_err_code = Some(code);
// Drive wedge senses (any with high byte 0x05 =
// ILLEGAL_REQUEST). The drive isn't merely
// rejecting our cert — it's saying "I won't talk
// to you anymore." Trying more certs makes the
// wedge worse. Bail out immediately.
let sense_key = ((code >> 8) & 0xFF) as u8;
if sense_key == 0x05 {
last_err_code = Some(e.code());
// Log the real SCSI sense triple, not `e.code()` —
// `code()` collapses every ScsiError to the flat
// E_SCSI_ERROR constant and carries no sense key,
// so it has no diagnostic value for auth-failure
// routing.
let sense = e.scsi_sense();
// Drive wedge senses (ILLEGAL_REQUEST, sense key
// 0x05). The drive isn't merely rejecting our
// cert — it's signalling it won't talk to us
// anymore. Trying more certs makes the wedge worse,
// so bail out immediately. NOTE: this must read the
// sense key off the structured ScsiSense, NOT off
// `e.code()`; `code()` is a flat constant for every
// ScsiError so the old `(code >> 8) & 0xFF` guard
// never matched and was dead code (the very wedge
// this defense exists to prevent could recur).
if sense.map(|s| s.is_illegal_request()).unwrap_or(false) {
tracing::warn!(
target: "freemkv::disc",
phase = "handshake_wedge_detected",
cert_index = idx,
error_code = code,
sense_key = sense.map(|s| s.sense_key),
asc = sense.map(|s| s.asc),
ascq = sense.map(|s| s.ascq),
"drive returned ILLEGAL_REQUEST during auth; bailing out to avoid wedge"
);
return (None, Some(Error::AacsHostCertRejected));
@@ -339,13 +349,15 @@ impl Disc {
.or_else(|_| udf_fs.read_file(reader, "/AACS/MKB_RW.inf"))
.ok()
.unwrap_or_default();
// Trim to the real record length. truncate is a no-op when n >=
// len and correctly empties the vec when n == 0 (zeroed/corrupt
// MKB), so it never leaves the full ~128 MiB zero-pad on
// AacsState.mkb.
let n = aacs::mkb_content_len(&mkb_bytes);
if n > 0 && n < mkb_bytes.len() {
mkb_bytes.truncate(n);
}
mkb_bytes.truncate(n);
let mkb_ver = aacs::mkb_version(&mkb_bytes);
tracing::warn!(
tracing::debug!(
target: "freemkv::disc",
phase = "scan_aacs_vid_only",
disc_hash = %aacs::disc_hash_hex(&dh),
src/disc/mapfile.rs
+222 -14
View File
@@ -6,7 +6,7 @@
//!
//! Format:
//! ```text
//! # Rescue Logfile. Created by libfreemkv v0.11.21
//! # Rescue Logfile. Created by libfreemkv vX.Y.Z
//! # Current pos / status / pass / pass_time (ddrescue state machine — we only populate pos)
//! 0x000000000 ? 1 0
//! # pos size status
@@ -50,6 +50,8 @@ pub enum SectorStatus {
}
impl SectorStatus {
/// The single ddrescue status character for this status
/// (`?`/`*`/`/`/`-`/`+`).
pub fn to_char(self) -> char {
match self {
Self::NonTried => '?',
@@ -59,6 +61,8 @@ impl SectorStatus {
Self::Finished => '+',
}
}
/// Parse a ddrescue status character into a `SectorStatus`. Returns
/// `None` for any character that is not one of `?*/-+`.
pub fn from_char(c: char) -> Option<Self> {
Some(match c {
'?' => Self::NonTried,
@@ -102,8 +106,8 @@ pub struct MapStats {
/// retry" UI bucket; `bytes_pending` over-counts because it folds
/// in `bytes_nontried`.
pub bytes_retryable: u64,
/// Number of unreadable ranges (for UI display). Computed from
/// `ranges_with(&[Unreadable])`.
/// Number of distinct `Unreadable` ranges (for UI display).
/// Computed by `compute_stats` (counts coalesced `-` entries).
pub num_bad_ranges: u32,
/// Largest gap among unreadable ranges in milliseconds. Computed as
/// largest range size / bytes_per_sec * 1000. Set by caller (autorip)
@@ -231,6 +235,16 @@ impl Mapfile {
}
let pos = parse_hex(fields[0])?;
let size = parse_hex(fields[1])?;
// Reject an entry whose pos+size overflows u64 up front. The
// downstream overlap/coalesce/next_with code adds pos+size
// freely; a crafted/corrupt line like
// `0xfffffffffffffff0 0x20 +` would otherwise panic (debug)
// or wrap to a tiny range (release), corrupting stats and
// resume logic.
if pos.checked_add(size).is_none() {
let e: io::Error = crate::error::Error::MapfileInvalid { kind: "range" }.into();
return Err(e);
}
let status = fields[2]
.chars()
.next()
@@ -247,7 +261,31 @@ impl Mapfile {
entries.push(MapEntry { pos, size, status });
}
entries.sort_by_key(|e| e.pos);
let total_size = entries.last().map(|e| e.pos + e.size).unwrap_or(0);
// Reject overlapping ranges. A well-formed ddrescue mapfile is a
// disjoint partition; overlaps (from a corrupt/hand-edited file)
// would make compute_stats double-count, so bytes_good /
// bytes_unreadable / bytes_pending could exceed bytes_total and
// inflate resume / abort-on-loss decisions and >100% progress.
for pair in entries.windows(2) {
let prev_end = pair[0].pos.saturating_add(pair[0].size);
if prev_end > pair[1].pos {
let e: io::Error = crate::error::Error::MapfileInvalid { kind: "overlap" }.into();
return Err(e);
}
}
let total_size = entries
.last()
.map(|e| e.pos.saturating_add(e.size))
.unwrap_or(0);
// Enforce the keys-XOR-vid invariant that set_unit_keys()
// guarantees: a corrupt/hand-edited file carrying both comment
// types would otherwise load with vid=Some AND non-empty
// unit_keys, violating the invariant downstream code relies on.
// Unit keys win, matching the setter (it clears vid when keys
// are present).
if !unit_keys.is_empty() {
vid = None;
}
let stats = Self::compute_stats(&entries, total_size);
Ok(Self {
path: path.to_path_buf(),
@@ -265,7 +303,26 @@ impl Mapfile {
/// Load if the file exists, otherwise create a fresh mapfile.
pub fn open_or_create(path: &Path, total_size: u64, version: &str) -> io::Result<Self> {
match Self::load(path) {
Ok(mf) => Ok(mf),
Ok(mf) => {
// load() derives total_size from the last entry's
// pos+size; if that disagrees with the caller's
// expected disc size (different disc, edited/partial
// file, trimmed trailing region) the downstream
// resume/progress math keys off the wrong basis. Surface
// it so an operator can spot a mismatched mapfile rather
// than failing the resume outright.
if mf.total_size != total_size {
tracing::warn!(
target: "freemkv::disc",
phase = "mapfile_total_size_mismatch",
loaded_total = mf.total_size,
supplied_total = total_size,
path = %path.display(),
"loaded mapfile coverage differs from supplied disc size"
);
}
Ok(mf)
}
Err(e) if e.kind() == io::ErrorKind::NotFound => {
Self::create(path, total_size, version)
}
@@ -280,11 +337,18 @@ impl Mapfile {
if size == 0 {
return Ok(());
}
let end = pos.saturating_add(size);
// Mirror load()'s overflow contract: reject a range that would
// wrap u64 rather than storing a saturated entry narrower than
// its size, which load() would then reject on the next resume
// (making the mapfile unreadable).
let Some(end) = pos.checked_add(size) else {
let e: io::Error = crate::error::Error::MapfileInvalid { kind: "range" }.into();
return Err(e);
};
let mut new_entries = Vec::with_capacity(self.entries.len() + 2);
for e in self.entries.drain(..) {
let e_end = e.pos + e.size;
let e_end = e.pos.saturating_add(e.size);
if e_end <= pos || e.pos >= end {
// entirely before or after — keep
new_entries.push(e);
@@ -313,8 +377,8 @@ impl Mapfile {
let mut merged: Vec<MapEntry> = Vec::with_capacity(new_entries.len());
for e in new_entries {
if let Some(last) = merged.last_mut() {
if last.pos + last.size == e.pos && last.status == e.status {
last.size += e.size;
if last.pos.saturating_add(last.size) == e.pos && last.status == e.status {
last.size = last.size.saturating_add(e.size);
continue;
}
}
@@ -383,10 +447,13 @@ impl Mapfile {
&self.unit_keys
}
/// All map entries, sorted ascending by `pos` and (after load)
/// guaranteed disjoint and non-overflowing.
pub fn entries(&self) -> &[MapEntry] {
&self.entries
}
/// Total image size in bytes, i.e. the end byte of the last entry.
pub fn total_size(&self) -> u64 {
self.total_size
}
@@ -397,7 +464,7 @@ impl Mapfile {
if e.status != status {
continue;
}
let e_end = e.pos + e.size;
let e_end = e.pos.saturating_add(e.size);
if e_end <= from {
continue;
}
@@ -416,6 +483,8 @@ impl Mapfile {
.collect()
}
/// Snapshot of the incrementally-maintained summary statistics.
/// O(1) — returns the cached `MapStats`.
pub fn stats(&self) -> MapStats {
self.stats
}
@@ -428,7 +497,10 @@ impl Mapfile {
for e in entries {
match e.status {
SectorStatus::Finished => s.bytes_good += e.size,
SectorStatus::Unreadable => s.bytes_unreadable += e.size,
SectorStatus::Unreadable => {
s.bytes_unreadable += e.size;
s.num_bad_ranges += 1;
}
SectorStatus::NonTried => {
s.bytes_pending += e.size;
s.bytes_nontried += e.size;
@@ -514,16 +586,36 @@ impl Drop for Mapfile {
/// error, so a corrupt header never fails a mapfile load.
fn parse_vid_hex(s: &str) -> Option<[u8; 16]> {
let s = s.strip_prefix("0x").unwrap_or(s);
if s.len() != 32 {
// Parse on bytes, not on the &str: slicing a &str by byte index
// (`&s[i*2..i*2+2]`) panics when the cut lands inside a multi-byte
// UTF-8 char. A hand-edited/corrupt `# freemkv-vid:` comment of
// exactly 32 bytes containing a multi-byte char would otherwise
// kill the whole load. ASCII hex is one byte per char, so anything
// non-ASCII is simply rejected here as malformed.
let bytes = s.as_bytes();
if bytes.len() != 32 {
return None;
}
let mut out = [0u8; 16];
for (i, b) in out.iter_mut().enumerate() {
*b = u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).ok()?;
let hi = hex_nibble(bytes[i * 2])?;
let lo = hex_nibble(bytes[i * 2 + 1])?;
*b = (hi << 4) | lo;
}
Some(out)
}
/// Map a single ASCII hex digit byte to its 0-15 value. Returns `None`
/// for any non-hex byte (including any non-ASCII / multi-byte lead byte).
fn hex_nibble(c: u8) -> Option<u8> {
match c {
b'0'..=b'9' => Some(c - b'0'),
b'a'..=b'f' => Some(c - b'a' + 10),
b'A'..=b'F' => Some(c - b'A' + 10),
_ => None,
}
}
/// Parse a `# freemkv-uk:` value `<cps>:<32hex>` into `(cps_unit, key)`. Returns
/// `None` on any malformation so a corrupt line is ignored, never fatal.
fn parse_uk_line(s: &str) -> Option<(u32, [u8; 16])> {
@@ -557,7 +649,9 @@ mod tests {
tag,
n
);
std::env::temp_dir().join(name)
let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("target/test-scratch");
let _ = std::fs::create_dir_all(&dir);
dir.join(name)
}
#[test]
@@ -758,6 +852,54 @@ mod tests {
let _ = std::fs::remove_file(&p2);
}
#[test]
fn load_rejects_entry_whose_range_overflows_u64() {
let p = tmpfile("load_overflow");
let _ = std::fs::remove_file(&p);
// pos near u64::MAX with a nonzero size overflows pos+size.
let body = format!("0x{:x} 0x10 +\n", u64::MAX - 4);
std::fs::write(&p, body).unwrap();
let kind = match Mapfile::load(&p) {
Ok(_) => panic!("overflowing entry must be rejected"),
Err(e) => e.kind(),
};
assert_eq!(kind, io::ErrorKind::InvalidData);
let _ = std::fs::remove_file(&p);
}
#[test]
fn record_rejects_range_overflowing_u64() {
let p = tmpfile("record_overflow");
let _ = std::fs::remove_file(&p);
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
let err = mf
.record(u64::MAX - 4, 16, SectorStatus::Finished)
.expect_err("overflowing record must be rejected");
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
let _ = std::fs::remove_file(&p);
}
#[test]
fn load_enforces_keys_xor_vid_on_malformed_file() {
let p = tmpfile("load_keys_xor_vid");
let _ = std::fs::remove_file(&p);
// Hand-craft a file carrying BOTH a vid comment and a uk comment
// (which write_to_disk would never emit together). load() must
// resolve to keys-only, matching set_unit_keys()'s invariant.
let body = "# freemkv-vid:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n\
# freemkv-uk: 0:11111111111111111111111111111111\n\
0x0 0x200 +\n";
std::fs::write(&p, body).unwrap();
let loaded = Mapfile::load(&p).unwrap();
assert_eq!(
loaded.vid(),
None,
"load() must clear vid when unit keys are present"
);
assert_eq!(loaded.unit_keys(), &[(0u32, [0x11u8; 16])]);
let _ = std::fs::remove_file(&p);
}
#[test]
fn vid_round_trips_and_data_lines_unaffected() {
let p = tmpfile("vid_round_trips");
@@ -831,6 +973,72 @@ mod tests {
let _ = std::fs::remove_file(&resaved);
}
#[test]
fn parse_vid_hex_does_not_panic_on_multibyte_32_byte_input() {
// A 32-BYTE comment containing a multi-byte char would make the
// old `&s[i*2..i*2+2]` slice fall inside a char boundary and
// panic. Must return None instead.
let s = "中".to_string() + &"a".repeat(29); // 3 + 29 = 32 bytes
assert_eq!(s.len(), 32);
assert_eq!(parse_vid_hex(&s), None);
// A valid 32-char ASCII hex string still parses.
assert_eq!(
parse_vid_hex("00112233445566778899aabbccddeeff"),
Some([
0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd,
0xee, 0xff,
])
);
}
#[test]
fn load_rejects_overflowing_pos_plus_size() {
let p = tmpfile("load_rejects_overflow");
let _ = std::fs::remove_file(&p);
std::fs::write(
&p,
"# Rescue Logfile. Created by test\n\
0x000000000 ? 1 0\n\
0xfffffffffffffff0 0x20 +\n",
)
.unwrap();
assert!(
Mapfile::load(&p).is_err(),
"a pos+size that overflows u64 must be rejected, not wrap"
);
let _ = std::fs::remove_file(&p);
}
#[test]
fn load_rejects_overlapping_ranges() {
let p = tmpfile("load_rejects_overlap");
let _ = std::fs::remove_file(&p);
std::fs::write(
&p,
"# Rescue Logfile. Created by test\n\
0x000000000 ? 1 0\n\
0x000000000 0x00000100 +\n\
0x000000080 0x00000100 -\n",
)
.unwrap();
assert!(
Mapfile::load(&p).is_err(),
"overlapping ranges must be rejected so stats can't double-count"
);
let _ = std::fs::remove_file(&p);
}
#[test]
fn num_bad_ranges_counts_unreadable_entries() {
let p = tmpfile("num_bad_ranges");
let _ = std::fs::remove_file(&p);
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
mf.record(100, 50, SectorStatus::Unreadable).unwrap();
mf.record(300, 50, SectorStatus::Unreadable).unwrap();
assert_eq!(mf.stats().num_bad_ranges, 2);
let _ = std::fs::remove_file(&p);
}
#[test]
fn stats_consistent_after_split_record() {
let p = tmpfile("stats_consistent_after_split");
src/disc/mod.rs
+153 -99
View File
@@ -364,10 +364,19 @@ pub enum ColorSpace {
pub struct Chapter {
/// Chapter start time in seconds
pub time_secs: f64,
/// Chapter name (e.g. "Chapter 1", "Chapter 2")
/// Chapter name — a bare 1-based index ("1", "2", …). The library
/// emits no localized prose; consuming apps prepend any "Chapter "
/// prefix in the user's language.
pub name: String,
}
/// Default chapter name for the 0-based chapter index `i`: the bare
/// 1-based ordinal as a string. Keeps chapter labelling language-neutral
/// (apps localize) and gives BD and DVD a single source of truth.
pub(crate) fn chapter_name(i: usize) -> String {
(i + 1).to_string()
}
/// A contiguous range of sectors on disc.
#[derive(Debug, Clone, Copy)]
pub struct Extent {
@@ -454,24 +463,24 @@ pub fn bytes_bad_in_title(title: &DiscTitle, bad_ranges: &[(u64, u64)]) -> u64 {
if bad_ranges.is_empty() || title.extents.is_empty() {
return 0;
}
let t_start = title.extents.first().map(|e| (e.start_lba as u64) * 2048);
let t_end = title
.extents
.last()
.map(|e| ((e.start_lba as u64) + (e.sector_count as u64)) * 2048);
let (Some(ts), Some(te)) = (t_start, t_end) else {
return 0;
};
bad_ranges
.iter()
.map(|(pos, size)| {
// Overlap each bad range against every extent individually. A single
// bounding box (first extent start → last extent end) would count
// bad sectors in inter-extent gaps (other titles' data, BDMV
// metadata) as bad bytes in this title, over-counting lost_ms for
// titles with non-contiguous clips.
let mut total: u64 = 0;
for ext in &title.extents {
let es = (ext.start_lba as u64) * 2048;
let ee = ((ext.start_lba as u64) + (ext.sector_count as u64)) * 2048;
for (pos, size) in bad_ranges {
let r_start = *pos;
let r_end = *pos + *size;
let overlap_start = r_start.max(ts);
let overlap_end = r_end.min(te);
overlap_end.saturating_sub(overlap_start)
})
.sum()
let r_end = pos.saturating_add(*size);
let overlap_start = r_start.max(es);
let overlap_end = r_end.min(ee);
total = total.saturating_add(overlap_end.saturating_sub(overlap_start));
}
}
total
}
// ─── Display helpers ────────────────────────────────────────────────────────
@@ -535,7 +544,10 @@ impl Codec {
0x81 => Codec::Ac3,
0x84 | 0xA1 => Codec::Ac3Plus,
0x80 => Codec::Lpcm,
0xA2 => Codec::DtsHdHr,
// 0x86 (primary) / 0xA2 (secondary) are the DTS-HD MA
// lossless pair, parallel to 0x81/0xA1 for AC-3. 0xA2 is
// lossless MA, not lossy HR.
0xA2 => Codec::DtsHdMa,
0x90 | 0x91 => Codec::Pgs,
ct => Codec::Unknown(ct),
}
@@ -693,7 +705,6 @@ impl AudioChannels {
3 => AudioChannels::Stereo,
6 => AudioChannels::Surround51,
12 => AudioChannels::Surround71,
_ if af > 0 => AudioChannels::Unknown,
_ => AudioChannels::Unknown,
}
}
@@ -1176,21 +1187,19 @@ impl Disc {
Ok((capacity, buffered, udf_fs))
}
/// Scan a disc -- parse filesystem, playlists, streams, and set up AACS decryption.
/// Scan a disc — parse filesystem, playlists, streams, and set up
/// AACS decryption. This is the main entry point; after `scan()` the
/// Disc is ready (titles populated with streams, AACS inputs
/// captured, content readable and decryptable transparently).
///
/// This is the main entry point. After scan(), the Disc is ready:
/// - titles are populated with streams
/// - AACS keys are derived (if KEYDB available)
/// - content can be read and decrypted transparently
///
/// Scan a disc. One pipeline, one order:
/// One pipeline, one order:
/// 1. Read capacity + UDF filesystem
/// 2. AACS handshake + key resolution
/// 3. Parse playlists + streams
/// 4. Apply labels
///
/// The session must be open and unlocked (Drive::open handles this).
/// All disc reads use standard READ(10) via UDF -- no vendor SCSI commands.
/// The session must be open and unlocked (`Drive::open` handles this).
/// All disc reads use standard READ(10) via UDF — no vendor SCSI commands.
pub fn scan(session: &mut Drive, opts: &ScanOptions) -> Result<Self> {
// AACS handshake (Blu-ray/UHD). Routes through Disc::read_vid,
// which prefers the per-drive OEM CDB path when the drive is
@@ -1280,58 +1289,52 @@ impl Disc {
Self::scan_with(reader, capacity, None, None, opts, udf_fs)
}
/// Read a disc's AACS key-input files from a sector source: returns
/// `(Unit_Key_RO.inf, MKB)` raw bytes. Shared body for
/// [`Disc::read_aacs_inputs`] (ISO) and
/// [`Disc::read_aacs_inputs_from_drive`] (live drive).
///
/// Prefers MKB_RO, falls back to MKB_RW, then TRIMS to the real
/// record length. Both files are allocated to a fixed ~128 MiB and
/// zero-padded, so reading either ships up to ~124 MiB of nothing —
/// trim to the record stream so callers send/store a few MB, not
/// 128 MiB.
fn read_aacs_inputs_from_reader(
reader: &mut dyn SectorSource,
udf_fs: &udf::UdfFs,
) -> Result<(Vec<u8>, Vec<u8>)> {
let inf = udf_fs
.read_file(reader, "/AACS/Unit_Key_RO.inf")
.or_else(|_| udf_fs.read_file(reader, "/AACS/DUPLICATE/Unit_Key_RO.inf"))
.map_err(|_| Error::AacsNoKeys)?;
let mut mkb = udf_fs
.read_file(reader, "/AACS/MKB_RO.inf")
.or_else(|_| udf_fs.read_file(reader, "/AACS/MKB_RW.inf"))
.map_err(|_| Error::AacsNoKeys)?;
let n = crate::aacs::mkb_content_len(&mkb);
mkb.truncate(n);
Ok((inf, mkb))
}
/// Read a disc's AACS key-input files from an ISO image: returns
/// `(Unit_Key_RO.inf, MKB)` raw bytes. For callers that resolve a Unit Key
/// out-of-band: obtain the key however you like, then scan with
/// `ScanOptions { unit_key: Some(uk), .. }`. libfreemkv never makes a
/// network call.
/// out-of-band: obtain the key however you like, then apply it via
/// [`Disc::decrypt_with`]. libfreemkv never makes a network call.
pub fn read_aacs_inputs(iso_path: &std::path::Path) -> Result<(Vec<u8>, Vec<u8>)> {
let mut reader = crate::io::file_sector_source::FileSectorSource::open(iso_path)
.map_err(|_| Error::AacsNoKeys)?;
let udf_fs = udf::read_filesystem(&mut reader)?;
let inf = udf_fs
.read_file(&mut reader, "/AACS/Unit_Key_RO.inf")
.or_else(|_| udf_fs.read_file(&mut reader, "/AACS/DUPLICATE/Unit_Key_RO.inf"))
.map_err(|_| Error::AacsNoKeys)?;
// Prefer MKB_RO, fall back to MKB_RW, then TRIM to the real record
// length. Both files are allocated to a fixed ~128 MiB and zero-padded,
// so reading either ships up to ~124 MiB of nothing — trim to the
// record stream so callers send/store a few MB, not 128 MiB.
let mut mkb = udf_fs
.read_file(&mut reader, "/AACS/MKB_RO.inf")
.or_else(|_| udf_fs.read_file(&mut reader, "/AACS/MKB_RW.inf"))
.map_err(|_| Error::AacsNoKeys)?;
let n = crate::aacs::mkb_content_len(&mkb);
if n > 0 && n < mkb.len() {
mkb.truncate(n);
}
Ok((inf, mkb))
Self::read_aacs_inputs_from_reader(&mut reader, &udf_fs)
}
/// Same as [`Disc::read_aacs_inputs`] but reads from a live drive. The
/// out-of-band Unit Key path fetches the disc's key files from the drive,
/// resolves a key from them however it likes, then scans with
/// `ScanOptions { unit_key: Some(uk), .. }`. These files are plaintext UDF
/// metadata — no AACS handshake or keys are required to read them.
/// resolves a key from them however it likes, then applies it via
/// [`Disc::decrypt_with`]. These files are plaintext UDF metadata — no
/// AACS handshake or keys are required to read them.
pub fn read_aacs_inputs_from_drive(drive: &mut Drive) -> Result<(Vec<u8>, Vec<u8>)> {
let (_, mut reader, udf_fs) = Self::read_udf(drive)?;
let inf = udf_fs
.read_file(&mut reader, "/AACS/Unit_Key_RO.inf")
.or_else(|_| udf_fs.read_file(&mut reader, "/AACS/DUPLICATE/Unit_Key_RO.inf"))
.map_err(|_| Error::AacsNoKeys)?;
// Prefer MKB_RO, fall back to MKB_RW, then TRIM to the real record
// length. Both files are allocated to a fixed ~128 MiB and zero-padded,
// so reading either ships up to ~124 MiB of nothing — trim to the
// record stream so callers send/store a few MB, not 128 MiB.
let mut mkb = udf_fs
.read_file(&mut reader, "/AACS/MKB_RO.inf")
.or_else(|_| udf_fs.read_file(&mut reader, "/AACS/MKB_RW.inf"))
.map_err(|_| Error::AacsNoKeys)?;
let n = crate::aacs::mkb_content_len(&mkb);
if n > 0 && n < mkb.len() {
mkb.truncate(n);
}
Ok((inf, mkb))
Self::read_aacs_inputs_from_reader(&mut reader, &udf_fs)
}
/// Core scan pipeline — works with any SectorSource.
@@ -2119,13 +2122,11 @@ impl Disc {
let pipe: Pipeline<WorkItem, sweep::ConsumerSummary> =
Pipeline::spawn_named("freemkv-sweep-consumer", DEFAULT_PIPELINE_DEPTH, sink)?;
// Translate `Pipeline::send` failure (consumer gone) into the
// same `Error` shape the 0.17.x `send_or_abort` produced, so
// the producer-error semantics are unchanged.
// Translate `Pipeline::send` failure (consumer gone) into a
// numeric library error so the producer-error semantics are
// unchanged but no English leaks into an io::Error.
fn consumer_gone() -> Error {
Error::IoError {
source: std::io::Error::other("sweep consumer terminated unexpectedly"),
}
Error::PipelineConsumerGone
}
let mut buf = vec![0u8; batch as usize * 2048];
@@ -2637,6 +2638,10 @@ pub(crate) fn sleep_secs_or_halt(
}
}
/// Mapfile path for a regular output file: appends `.mapfile` to the
/// output path. For `/dev/null` (benchmark) output use
/// [`Disc::mapfile_for`], which special-cases it to a temp-dir path
/// derived from the disc title.
pub fn mapfile_path_for(iso_path: &std::path::Path) -> std::path::PathBuf {
let mut s = iso_path.as_os_str().to_os_string();
s.push(".mapfile");
@@ -2646,8 +2651,10 @@ pub fn mapfile_path_for(iso_path: &std::path::Path) -> std::path::PathBuf {
impl Disc {
/// Path to the mapfile for a given output path.
///
/// For `/dev/null` output, returns `/tmp/{volume_id_or_title}.mapfile`.
/// For regular files, returns `{path}.mapfile`.
/// For `/dev/null` output, returns
/// `{temp_dir}/{volume_id_or_title}.mapfile` (temp dir is
/// `TMPDIR`-aware and cross-platform). For regular files, returns
/// `{path}.mapfile`.
pub fn mapfile_for(&self, path: &std::path::Path) -> std::path::PathBuf {
if path.as_os_str() == "/dev/null" {
let name: String = self
@@ -2663,7 +2670,7 @@ impl Disc {
}
})
.collect();
std::path::PathBuf::from(format!("/tmp/{name}.mapfile"))
std::env::temp_dir().join(format!("{name}.mapfile"))
} else {
mapfile_path_for(path)
}
@@ -2755,25 +2762,25 @@ pub fn detect_max_batch_sectors(device_path: &str) -> u16 {
return DEFAULT_BATCH_SECTORS_OPTICAL;
}
// Check if optical drive (0x05 = CD/DVD)
let is_optical = (|| -> bool {
use std::path::Path;
let scsi_device_dir = "/sys/class/scsi_device/".to_string();
if let Ok(entries) = std::fs::read_dir(&scsi_device_dir) {
for entry in entries.flatten() {
let device_type_path = entry.path().join("device/type");
if Path::new(&device_type_path).exists() {
if let Ok(content) = std::fs::read_to_string(&device_type_path) {
// Type 0x05 (decimal 5) = CD/DVD drive
if content.trim().parse::<u32>() == Ok(5) {
return true;
}
}
}
}
}
false
})();
// Check whether THIS device (not any device on the host) is an
// optical drive: read the SCSI peripheral type of the target node
// only. Type 0x05 (decimal 5) = CD/DVD. A previous version scanned
// every /sys/class/scsi_device entry and returned true if any was
// optical, misclassifying a block device as optical on a host that
// also has an optical drive.
let is_optical = {
// For an sg node the type lives at scsi_generic/<sg>/device/type;
// for a block node (sr0/sdX) at /sys/block/<name>/device/type.
let type_path = if dev_name.starts_with("sg") {
format!("/sys/class/scsi_generic/{dev_name}/device/type")
} else {
format!("/sys/block/{dev_name}/device/type")
};
std::fs::read_to_string(&type_path)
.ok()
.map(|c| c.trim().parse::<u32>() == Ok(5))
.unwrap_or(false)
};
if is_optical {
// For sg devices, find the corresponding block device name
@@ -3417,7 +3424,6 @@ mod tests {
#[test]
fn sweep_to_dev_null_real() {
let _cleanup = CleanupGuard(std::path::PathBuf::from("/tmp/T2.mapfile"));
let sectors: u32 = 1000;
let bad: std::collections::HashSet<u32> = [500u32, 501, 502].into_iter().collect();
let mut reader = MockReader {
@@ -3425,6 +3431,7 @@ mod tests {
bad_sectors: bad,
};
let disc = make_test_disc(sectors, "T2");
let _cleanup = CleanupGuard(disc.mapfile_for(std::path::Path::new("/dev/null")));
let opts = CopyOptions {
decrypt: false,
multipass: true,
@@ -3450,13 +3457,13 @@ mod tests {
#[test]
fn sweep_dev_null_full_good() {
let _cleanup = CleanupGuard(std::path::PathBuf::from("/tmp/T3.mapfile"));
let sectors: u32 = 2000;
let mut reader = MockReader {
total_sectors: sectors,
bad_sectors: std::collections::HashSet::new(),
};
let disc = make_test_disc(sectors, "T3");
let _cleanup = CleanupGuard(disc.mapfile_for(std::path::Path::new("/dev/null")));
let opts = CopyOptions {
decrypt: false,
multipass: false,
@@ -3615,4 +3622,51 @@ mod tests {
let meta = std::fs::metadata(&iso_path).unwrap();
assert_eq!(meta.len(), sectors as u64 * 2048);
}
/// bytes_bad_in_title must overlap per-extent, not against a single
/// bounding box: a bad range in the gap between two extents of the
/// same title must NOT be counted.
#[test]
fn bytes_bad_in_title_ignores_inter_extent_gap() {
let mut title = title_with_video(Codec::Hevc, Resolution::R2160p);
// Two extents: sectors [0,10) and [100,110). Gap = [10,100).
title.extents = vec![
Extent {
start_lba: 0,
sector_count: 10,
},
Extent {
start_lba: 100,
sector_count: 10,
},
];
// A bad range entirely inside the gap (sector 50 == byte 50*2048).
let gap = vec![(50 * 2048, 2048)];
assert_eq!(
bytes_bad_in_title(&title, &gap),
0,
"bad bytes in the inter-extent gap must not be counted"
);
// A bad range overlapping the first extent counts.
let in_first = vec![(0, 4096)];
assert_eq!(bytes_bad_in_title(&title, &in_first), 4096);
// A bad range spanning both extents plus the gap counts only the
// bytes that fall inside the two extents (10 + 10 sectors).
let spanning = vec![(0, 110 * 2048)];
assert_eq!(bytes_bad_in_title(&title, &spanning), 20 * 2048);
}
/// 0xA2 is secondary DTS-HD MA (lossless), not lossy HR.
#[test]
fn coding_type_a2_is_dts_hd_ma() {
assert_eq!(Codec::from_coding_type(0xA2), Codec::DtsHdMa);
assert_eq!(Codec::from_coding_type(0x86), Codec::DtsHdMa);
}
/// chapter_name emits a bare 1-based ordinal (no localized prose).
#[test]
fn chapter_name_is_bare_ordinal() {
assert_eq!(chapter_name(0), "1");
assert_eq!(chapter_name(41), "42");
}
}
src/disc/patch.rs
+141 -69
View File
@@ -319,14 +319,14 @@ const CACHE_PRIME_SECTORS: u32 = 3;
/// scatter its sample LBAs across the failing region rather than
/// hammering the same neighborhood.
pub(super) fn skip_sectors_for_probe(idx: usize) -> u64 {
let base = PASSN_SKIP_SECTORS_BASE as i64;
let escalation = (idx * 3) as i64;
let shifted = if escalation < 64 {
base << escalation
} else {
base
};
shifted.min(PASSN_SKIP_SECTORS_CAP as i64) as u64
let escalation = (idx.saturating_mul(3)).min(u32::MAX as usize) as u32;
// Saturating shift: a large `idx` would overflow a fixed-width shift
// (32 << 60 = 2^65), so fall back to the cap instead of panicking
// (debug) or wrapping to 0 (release).
PASSN_SKIP_SECTORS_BASE
.checked_shl(escalation)
.unwrap_or(PASSN_SKIP_SECTORS_CAP)
.min(PASSN_SKIP_SECTORS_CAP)
}
/// Send a `PatchItem` and translate a `SendError` (consumer thread died
@@ -336,9 +336,7 @@ pub(super) fn send_or_abort(
pipe: &Pipeline<PatchItem, PatchSummary>,
item: PatchItem,
) -> Result<()> {
pipe.send(item).map_err(|_| Error::IoError {
source: std::io::Error::other("patch consumer terminated unexpectedly"),
})
pipe.send(item).map_err(|_| Error::PipelineConsumerGone)
}
/// Phase A pre-snapshot. Loads the mapfile, captures the fields the
@@ -667,21 +665,21 @@ pub(super) fn handle_read_success<R: SectorSource + ?Sized>(
state.damage_window.push(true);
if state.damage_window.len() > PASSN_DAMAGE_WINDOW {
state.damage_window.remove(0);
tracing::info!(
target: "freemkv::disc",
phase = "patch_read_ok",
lba,
count,
bytes,
blocks_read_ok = state.blocks_read_ok,
consecutive_failures = state.consecutive_failures,
read_duration_ms,
range_idx = frame.range_idx,
pos,
"Read succeeded"
);
}
tracing::info!(
target: "freemkv::disc",
phase = "patch_read_ok",
lba,
count,
bytes,
blocks_read_ok = state.blocks_read_ok,
consecutive_failures = state.consecutive_failures,
read_duration_ms,
range_idx = frame.range_idx,
pos,
"Read succeeded"
);
// Plaintext: DecryptingSectorSource applied AACS / CSS in-place
// during the read_sectors call above. The pre-0.18 inline
// decrypt_sectors call lived here.
@@ -909,8 +907,12 @@ pub(super) fn handle_read_failure<R: SectorSource + ?Sized>(
// Check if this is a NOT_READY error that should be retried
let sense = err.scsi_sense();
// ASC values indicating temporary drive unresponsiveness:
// 0x02 = medium not present, 0x03 = becoming ready, 0x04 = initialization required
// ASC values (under NOT READY, sense_key 0x02) indicating temporary
// drive unresponsiveness worth retrying:
// 0x02 = LUN not ready, no reference position (mechanism still seeking)
// 0x03 = LUN not ready, manual intervention required
// 0x04 = LUN not ready, in process of becoming ready / initializing
// (Medium-not-present is ASC 0x3A, not handled here — nothing to retry.)
let is_not_ready_retryable = sense
.map(|s| s.sense_key == 0x02 && (s.asc == 0x02 || s.asc == 0x03 || s.asc == 0x04))
.unwrap_or(false);
@@ -923,7 +925,7 @@ pub(super) fn handle_read_failure<R: SectorSource + ?Sized>(
lba,
consecutive_failures = state.consecutive_failures,
err_asc = sense.map(|s| s.asc as u32).unwrap_or(0),
"NOT_READY with ASC=0x03/0x04; pausing for drive recovery before retry"
"NOT_READY with ASC in 0x02/0x03/0x04; pausing for drive recovery before retry"
);
// Extended pause for NOT_READY - let drive complete internal mechanical recovery
@@ -1020,17 +1022,30 @@ pub(super) fn handle_read_failure<R: SectorSource + ?Sized>(
);
}
// Probe good sectors to differentiate wedge vs bad sector
// Probe good sectors to differentiate wedge vs bad sector.
// `skip_sectors_for_probe` returns a SECTOR distance; scale to bytes
// before adding to `pos` (a byte offset). The previous code compared
// a sector count against `block_bytes` and added a sector count to a
// byte offset, so the only probe that ran landed back on the failing
// LBA — the responsive-vs-wedged heuristic never scattered.
if state.consecutive_failures >= 3 && state.consecutive_failures % 5 == 0 {
let probe_offsets: [u64; 3] = [0, skip_sectors_for_probe(1), skip_sectors_for_probe(2)];
let probe_offsets_sectors: [u64; 3] =
[0, skip_sectors_for_probe(1), skip_sectors_for_probe(2)];
let mut probes_ok = 0;
for (probe_idx, &offset) in probe_offsets.iter().enumerate() {
if offset >= block_bytes || (offset == 0 && state.consecutive_failures < 5) {
for (probe_idx, &offset_sectors) in probe_offsets_sectors.iter().enumerate() {
let offset = offset_sectors.saturating_mul(2048);
let probe_pos = pos.saturating_add(offset);
// Skip the zero-distance re-read until failures are well
// established (it just re-confirms the current LBA), and
// never probe past the end of the current bad range (the
// probe scatters sample LBAs across the failing region —
// `block_bytes`, one block, was the wrong bound and in the
// wrong units).
if probe_pos >= frame.end || (offset == 0 && state.consecutive_failures < 5) {
continue;
}
let probe_pos = pos + offset;
let probe_lba = (probe_pos / 2048) as u32;
let probe_count = 1u16;
let mut probe_buf = [0u8; 2048];
@@ -1229,20 +1244,11 @@ pub(super) fn check_range_watchdog(
frame: &RangeFrame,
shared: &Mutex<SharedPatchState>,
) -> bool {
if state.range_start.elapsed().as_secs() > frame.range_budget_secs {
tracing::warn!(
target: "freemkv::disc",
phase = "patch_range_timeout",
range_lba = frame.range_pos / 2048,
range_sectors = frame.range_sectors,
elapsed_secs = state.range_start.elapsed().as_secs(),
budget_secs = frame.range_budget_secs,
bytes_recovered = state.range_bytes_good.saturating_sub(state.bytes_good_before),
"Range timeout - moving to next range"
);
return true;
}
// Refresh the forward-progress baseline FIRST, then do a single
// elapsed-vs-budget check. Reading bytes_good before the budget
// test means a range that committed a recovered sector since the
// previous tick resets its clock instead of being abandoned in the
// budget-boundary window.
let bytes_good_now = {
let g = shared
.lock()
@@ -1292,30 +1298,42 @@ pub(super) fn handle_skip_limit(
// them on a later pass when state has evolved (cache, mechanical
// settle). 2026-05-07 dd-as-oracle test confirmed ~36% of patch-
// marked Unreadable sectors are actually readable.
let unmarked_bytes = frame.block_end.saturating_sub(frame.range_pos);
if opts.reverse {
send_or_abort(
pipe,
PatchItem::NonTrimmed {
pos: frame.range_pos,
len: unmarked_bytes,
},
)?;
} else {
let remaining_start = frame.range_pos + (frame.end - frame.block_end);
if remaining_start < frame.end {
send_or_abort(
pipe,
PatchItem::NonTrimmed {
pos: remaining_start,
len: frame.end - remaining_start,
},
)?;
}
if let Some((pos, len)) =
skip_limit_remainder(opts.reverse, frame.range_pos, frame.end, frame.block_end)
{
send_or_abort(pipe, PatchItem::NonTrimmed { pos, len })?;
}
Ok(())
}
/// The never-attempted remainder of a range when the skip limit is
/// reached, as `Some((pos, len))` or `None` if nothing is left.
///
/// `block_end` is the per-iteration cursor. In reverse mode it moved
/// DOWN from `end` toward `range_pos`, so the attempted region is
/// `[block_end, end)` and the remainder is `[range_pos, block_end)`. In
/// forward mode it moved UP from `range_pos` toward `end`, so the
/// attempted region is `[range_pos, block_end)` and the remainder is
/// `[block_end, end)`. The pre-fix forward formula
/// `range_pos + (end - block_end)` was a mirror reflection that, once
/// `block_end` passed the midpoint, produced a start BELOW `block_end`
/// and overlapped the already-recovered region — downgrading Finished
/// sectors to NonTrimmed.
fn skip_limit_remainder(
reverse: bool,
range_pos: u64,
end: u64,
block_end: u64,
) -> Option<(u64, u64)> {
if reverse {
let len = block_end.saturating_sub(range_pos);
(len > 0).then_some((range_pos, len))
} else {
let len = end.saturating_sub(block_end);
(len > 0).then_some((block_end, len))
}
}
/// Damage-cluster size-aware skip decision. Inspects `state.damage_window`
/// against the `PASSN_DAMAGE_THRESHOLD_PCT` threshold; if crossed,
/// advances `frame.block_end` by an escalating skip (capped at 1/4 of
@@ -1348,7 +1366,9 @@ pub(super) fn compute_damage_skip(
};
let range_remaining_sectors = range_remaining_bytes / 2048;
let range_quarter = (range_remaining_sectors / 4).max(1);
let escalated = (PASSN_SKIP_SECTORS_BASE << state.consecutive_skips_without_recovery)
let escalated = PASSN_SKIP_SECTORS_BASE
.checked_shl(state.consecutive_skips_without_recovery)
.unwrap_or(PASSN_SKIP_SECTORS_CAP)
.min(PASSN_SKIP_SECTORS_CAP);
let skip_sectors = escalated.min(range_quarter);
let skip_bytes = skip_sectors * 2048;
@@ -1539,7 +1559,11 @@ impl Disc {
// "split decisions", not recorded failures
// - drop-to-1 retries the SAME starting position, so every
// sector in the failed batch is individually probed
let initial_batch = opts.block_sectors.unwrap_or(1);
// Clamp to at least 1 sector. block_sectors is public
// (Option<u16>); Some(0) would compute a zero-length read per
// iteration, never advance block_end, and busy-spin the range
// until its watchdog fired.
let initial_batch = opts.block_sectors.unwrap_or(1).max(1);
let recovery = opts.full_recovery;
let mut state = PatchLoopState::new(
bytes_good_before,
@@ -1785,3 +1809,51 @@ impl Disc {
))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn skip_sectors_for_probe_does_not_overflow_for_large_idx() {
// idx=20 (escalation 60) and idx=21 (63) previously overflowed
// i64 via `32i64 << escalation`. Must saturate to the cap.
for idx in [0usize, 1, 2, 20, 21, 100, usize::MAX] {
let v = skip_sectors_for_probe(idx);
assert!(
v <= PASSN_SKIP_SECTORS_CAP,
"idx {idx}: {v} exceeds cap {PASSN_SKIP_SECTORS_CAP}"
);
}
// Small indices still escalate as before.
assert_eq!(skip_sectors_for_probe(0), PASSN_SKIP_SECTORS_BASE);
assert_eq!(skip_sectors_for_probe(1), PASSN_SKIP_SECTORS_BASE << 3);
}
#[test]
fn skip_limit_remainder_forward_does_not_overlap_recovered_region() {
// Forward mode: range [1000, 2000), cursor advanced past the
// midpoint to block_end=1700. The recovered region is
// [1000, 1700); the never-attempted remainder must be exactly
// [1700, 2000) — NOT a mirror start below block_end.
let r = skip_limit_remainder(false, 1000, 2000, 1700);
assert_eq!(r, Some((1700, 300)));
// The pre-fix mirror formula would have produced start =
// 1000 + (2000 - 1700) = 1300, which overlaps [1000, 1700).
assert!(r.unwrap().0 >= 1700, "must not overlap recovered region");
}
#[test]
fn skip_limit_remainder_forward_none_when_fully_attempted() {
assert_eq!(skip_limit_remainder(false, 1000, 2000, 2000), None);
}
#[test]
fn skip_limit_remainder_reverse_marks_low_unattempted_region() {
// Reverse mode: cursor moved down to block_end=1300, so
// [1300, 2000) was attempted and [1000, 1300) is the remainder.
let r = skip_limit_remainder(true, 1000, 2000, 1300);
assert_eq!(r, Some((1000, 300)));
assert_eq!(skip_limit_remainder(true, 1000, 2000, 1000), None);
}
}
src/disc/read_error.rs
+144 -19
View File
@@ -34,14 +34,20 @@ pub struct ReadCtx {
/// Sliding window of recent read outcomes (true=ok, false=fail).
/// Capped at `damage_window_max`. Drives damage-jump decisions.
pub damage_window: Vec<bool>,
/// Maximum number of outcome entries kept in `damage_window`; the
/// oldest is evicted once this is exceeded. A whole count (e.g. 16).
pub damage_window_max: usize,
/// Fraction of `damage_window` entries that must be failures before
/// the window-based damage-jump fires, as a whole-number percentage
/// (e.g. `12` = 12%).
pub damage_threshold_pct: usize,
/// Trigger a damage-jump after this many consecutive outer-batch
/// failures, even when the damage_window isn't full yet. Pass 1
/// uses a small value (4) so we don't spend ~40 minutes grinding
/// to fill a 16-block window before the first jump on a damage
/// zone we entered cleanly. Pass N uses a larger value (or
/// disables this — see `bisect_on_marginal`) because Pass N's
/// uses a small value (1 — jump on the first outer failure; see
/// the 2026-05-11 rewrite in `for_sweep`) so we don't spend ~40
/// minutes grinding to fill a 16-block window before the first jump
/// on a damage zone we entered cleanly. Pass N uses a larger value
/// (or disables this — see `bisect_on_marginal`) because Pass N's
/// whole job IS to grind on the bad ranges.
pub fast_jump_threshold: u64,
/// Multiplier applied to damage-jump distance. Doubles each jump,
@@ -240,6 +246,13 @@ impl ReadCtx {
// drive recovered, so further wedges should reset the skip
// budget instead of accumulating toward a real abort.
self.wedge_count = 0;
// A successful read also means the bridge recovered, so the
// 15s-cooldown retry budget should be available again for the
// next bridge-degradation event. Without this reset the budget
// saturates permanently after 5 cumulative events across the
// whole pass and later degradations skip the cooldown retry,
// needlessly losing data.
self.bridge_degradation_count = 0;
// Outer-success only: a good single-sector read inside a
// bisect doesn't mean we've left the damaged batch. Only an
// outer-batch success resets the outer-failure counter.
@@ -259,6 +272,13 @@ impl ReadCtx {
if self.in_damage_zone && self.consecutive_good >= self.damage_window_max as u64 {
self.in_damage_zone = false;
self.last_error_family = None;
// Reset the damage-jump multiplier so the NEXT zone starts
// from the base jump distance. Without this the multiplier
// stays at whatever the prior zone inflated it to (up to
// MAX_JUMP_MULTIPLIER=64), so the next zone's first jump is
// 64x oversized and skips recoverable data. The field doc
// promises this reset.
self.jump_multiplier = 1;
}
}
@@ -375,9 +395,9 @@ const JUMP_BASE_SECTORS: u64 = 1024;
// When the BU40N (or similar drives) hits a physical-damage cluster,
// its firmware can transition into a "wedge" state where it returns
// HARDWARE_ERROR or ILLEGAL_REQUEST for every subsequent read —
// often for many LBAs after the actual bad sector. Per project docs
// "Bad-sector handling" rule #2: "Recovery requires eject+reload OR
// significant cool-down."
// often for many LBAs after the actual bad sector. Once wedged,
// recovery requires either a physical eject + reload or a significant
// cool-down period; hammering the same LBA only deepens the state.
//
// Pass 1's pre-fix behavior was to immediately AbortPass on the
// first HARDWARE_ERROR / ILLEGAL_REQUEST, killing the rip at
@@ -396,10 +416,10 @@ const JUMP_BASE_SECTORS: u64 = 1024;
/// One-gigabyte jump (1024 MiB) on each wedge. Big enough to clear
/// almost any single-cluster damage zone we've seen.
const WEDGE_JUMP_SECTORS: u64 = 524_288;
/// Cooldown pause after each wedge. Per project docs the drive needs
/// "significant cool-down"; 30 s strikes a balance between giving
/// the drive a chance to recover and not stalling the rip if the
/// drive is permanently stuck.
/// Cooldown pause after each wedge. A wedged drive needs a
/// significant cool-down to leave fast-fail; 30 s strikes a balance
/// between giving the drive a chance to recover and not stalling the
/// rip if the drive is permanently stuck.
const WEDGE_PAUSE_SECS: u64 = 30;
/// Bail after this many consecutive wedges with no good read in
/// between. At 1 GB jumps this lets us scan ~16 GB worth of fully
@@ -463,8 +483,13 @@ pub fn handle_read_error(err: &Error, ctx: &mut ReadCtx) -> ReadAction {
.unwrap_or(SenseFamily::Other);
// Zone-entry tracking: this is the first error after a clean run
// (or the first error of the sweep).
if !ctx.in_damage_zone && !ctx.bisecting {
// (or the first error of the sweep). Capture the genuine
// clean->damaged transition here, BEFORE mutating in_damage_zone,
// so the 30s zone-entry cooldown below keys off the real
// transition rather than re-deriving it from a counter that the
// fast-jump path resets after every jump.
let is_zone_entry_transition = !ctx.in_damage_zone && !ctx.bisecting;
if is_zone_entry_transition {
ctx.in_damage_zone = true;
ctx.zones_entered += 1;
}
@@ -569,9 +594,13 @@ pub fn handle_read_error(err: &Error, ctx: &mut ReadCtx) -> ReadAction {
// AbortPass after N consecutive wedges with no successful
// read in between.
if sense_key == scsi::SENSE_KEY_HARDWARE_ERROR || sense_key == scsi::SENSE_KEY_ILLEGAL_REQUEST {
if !ctx.bisecting {
ctx.wedge_count += 1;
}
// Count every wedge, including bisect-inner ones. A wedge is a
// firmware fast-fail state regardless of whether we're inside a
// bisect; if we did NOT count bisect-inner wedges, a drive that
// wedges mid-bisect would burn a 30s WEDGE_PAUSE cooldown per
// inner sector and never reach WEDGE_ABORT_THRESHOLD from inside
// the bisect — ~16 min of cooldown sleeping on a batch=32 bisect.
ctx.wedge_count += 1;
if ctx.wedge_count >= WEDGE_ABORT_THRESHOLD {
tracing::warn!(
target: "freemkv::disc",
@@ -665,8 +694,7 @@ pub fn handle_read_error(err: &Error, ctx: &mut ReadCtx) -> ReadAction {
// branch for future tuning. Pass N (bisect_on_marginal=true)
// uses the standard pauses — it's running single-sector retries
// on already-known-bad LBAs by design.
let is_zone_entry =
ctx.consecutive_outer_failures == 1 && !ctx.bisecting && !ctx.bisect_on_marginal;
let is_zone_entry = is_zone_entry_transition && !ctx.bisecting && !ctx.bisect_on_marginal;
let pause_secs = if is_zone_entry {
ZONE_ENTRY_COOLDOWN_SECS
} else if ctx.consecutive_failures >= CONSECUTIVE_FAIL_LONG_PAUSE_THRESHOLD {
@@ -692,7 +720,7 @@ pub fn handle_read_error(err: &Error, ctx: &mut ReadCtx) -> ReadAction {
// Two triggers, evaluated in order:
//
// a. **Fast-entry** — `consecutive_outer_failures >= fast_jump_threshold`.
// Fires on Pass 1 (threshold=4) so we don't spend ~40 min
// Fires on Pass 1 (threshold=1) so we don't spend ~40 min
// grinding to fill a 16-block damage window before the
// first jump on a damage zone we entered cleanly. Doesn't
// fire on Pass N (threshold=u64::MAX).
@@ -1029,6 +1057,43 @@ mod tests {
}
}
#[test]
fn pass_1_subsequent_in_zone_errors_skip_long_cooldown() {
// Regression: the fast-jump path resets consecutive_outer_failures
// to 0 after each jump, so the next in-zone error re-increments it
// to 1. Zone-entry must key off the genuine clean->damaged
// transition (in_damage_zone), not the counter, otherwise every
// error in a damaged region pays the 30 s cooldown.
let mut ctx = ReadCtx::for_sweep(32);
// First error: genuine zone entry, gets the long cooldown.
let first = handle_read_error(&medium_err(), &mut ctx);
match first {
ReadAction::JumpAhead { pause_secs, .. } => assert_eq!(
pause_secs,
ZONE_ENTRY_COOLDOWN_SECS + POST_JUMP_EXTRA_PAUSE_SECS
),
other => panic!("expected JumpAhead on first error, got {other:?}"),
}
// We are now still in the damage zone; the jump reset the outer
// counter. A second error must NOT re-arm the 30 s cooldown.
assert!(ctx.in_damage_zone);
let second = handle_read_error(&medium_err(), &mut ctx);
let pause = match second {
ReadAction::JumpAhead { pause_secs, .. } => pause_secs,
ReadAction::SkipBlock { pause_secs } => pause_secs,
other => panic!("expected pausing action, got {other:?}"),
};
assert_ne!(
pause,
ZONE_ENTRY_COOLDOWN_SECS + POST_JUMP_EXTRA_PAUSE_SECS,
"subsequent in-zone error must not pay the 30 s zone-entry cooldown"
);
assert!(
pause <= FAIL_PAUSE_SECS + POST_JUMP_EXTRA_PAUSE_SECS,
"subsequent in-zone pause should be the standard fail pause, got {pause}"
);
}
#[test]
fn pass_n_pauses_uniformly_on_failed_read() {
// Pass N (bisect_on_marginal=true) is exempt from the
@@ -1067,6 +1132,66 @@ mod tests {
);
}
#[test]
fn jump_multiplier_resets_after_damage_zone_exit() {
// A zone that doubles the multiplier must not carry the inflated
// value into the next zone — otherwise the next zone's first
// jump is up to 64x oversized and skips recoverable data.
let mut ctx = ReadCtx::for_sweep(32);
// First zone: a few errors push jumps and double the multiplier.
for _ in 0..4 {
handle_read_error(&medium_err(), &mut ctx);
}
assert!(
ctx.jump_multiplier > 1,
"expected the multiplier to inflate inside a damage zone"
);
// Exit the zone: damage_window_max consecutive good reads.
ctx.bisecting = false;
for _ in 0..ctx.damage_window_max {
ctx.on_success();
}
assert!(!ctx.in_damage_zone, "zone should have exited");
assert_eq!(
ctx.jump_multiplier, 1,
"jump_multiplier must reset to 1 on zone exit"
);
}
#[test]
fn bridge_degradation_count_resets_on_success() {
// After a good read the bridge recovered; the 15s-cooldown retry
// budget must be available again instead of staying saturated
// for the whole pass.
let mut ctx = ReadCtx::for_patch(1);
ctx.bridge_degradation_count = BRIDGE_DEGRADATION_MAX_RETRIES;
ctx.on_success();
assert_eq!(ctx.bridge_degradation_count, 0);
}
#[test]
fn wedge_abort_reachable_during_bisect() {
// A drive that wedges mid-bisect must still reach the abort
// threshold rather than burning a WEDGE_PAUSE cooldown per inner
// sector forever.
let mut ctx = ReadCtx::for_patch(32);
ctx.bisecting = true;
let mut aborted = false;
for _ in 0..WEDGE_ABORT_THRESHOLD {
if matches!(
handle_read_error(&hardware_err(), &mut ctx),
ReadAction::AbortPass
) {
aborted = true;
break;
}
}
assert!(
aborted,
"wedge abort threshold must be reachable from inside a bisect"
);
}
#[test]
fn on_success_resets_failure_counters_and_pushes_window() {
let mut ctx = ReadCtx::for_sweep(32);
src/disc/sweep.rs
+22 -49
View File
@@ -8,15 +8,13 @@
//! during the post-read work; throughput tops out at the *sum* of
//! both costs.
//!
//! 0.17.11 introduced a bespoke producer/consumer split (the now-
//! removed `disc/sweep_pipeline.rs`) to overlap the two stages. 0.18
//! collapses that split — together with the analogous splits patch
//! and mux need — onto the generic [`crate::io::Pipeline`] +
//! [`crate::io::Sink`] primitive. This module is the sweep-specific
//! `Sink` impl; the producer-side state machine (read_error context,
//! decrypt, set_speed, halt) stays in `Disc::sweep` in `disc/mod.rs`.
//! A producer/consumer split overlaps the two stages on the generic
//! [`crate::io::Pipeline`] + [`crate::io::Sink`] primitive. This module
//! is the sweep-specific `Sink` impl; the producer-side state machine
//! (read_error context, decrypt, set_speed, halt) stays in
//! `Disc::sweep` in `disc/mod.rs`.
//!
//! Correctness invariants preserved (same as 0.17.11):
//! Correctness invariants preserved:
//! - Mapfile is single-writer (consumer-only). No locking.
//! - All `read_error::ReadCtx` state stays on the producer thread.
//! - `set_speed` calls happen on the producer thread (same thread that
@@ -25,9 +23,8 @@
//! intact in the consumer (write before record), so the on-disk
//! invariant "mapfile only marks Finished what the file has
//! received" survives a crash mid-pass.
//! - The BU40N+Initio bridge wedge concern is unchanged: only one
//! SCSI command in flight at a time, error-path timing identical,
//! no new retry logic.
//! - Only one SCSI command is in flight at a time; error-path timing
//! is identical and no new retry logic is introduced.
use std::io::{Seek, SeekFrom, Write};
use std::sync::mpsc::{Receiver, SyncSender, sync_channel};
@@ -40,7 +37,7 @@ use super::mapfile::{MapStats, Mapfile, SectorStatus};
/// Reusable zero buffer for SkipFill / GapFill / BisectBad. 64 KB
/// matches the existing zero_gap chunk size used by the pre-split
/// sweep loop.
const ZERO_CHUNK: usize = 65 * 1024;
const ZERO_CHUNK: usize = 64 * 1024;
/// Producer → Consumer messages. The consumer applies these in FIFO
/// order; ordering of file writes and mapfile records across items is
@@ -151,55 +148,31 @@ impl Sink<WorkItem> for SweepSink {
WorkItem::Good { pos, buf } => {
// Decrypt is on the producer; consumer assumes plaintext.
let len = buf.len() as u64;
self.file
.seek(SeekFrom::Start(pos))
.map_err(|e| Error::IoError { source: e })?;
self.file
.write_all(&buf)
.map_err(|e| Error::IoError { source: e })?;
self.map
.record(pos, len, SectorStatus::Finished)
.map_err(|e| Error::IoError { source: e })?;
self.file.seek(SeekFrom::Start(pos))?;
self.file.write_all(&buf)?;
self.map.record(pos, len, SectorStatus::Finished)?;
}
WorkItem::BisectGood { pos, buf } => {
self.file
.seek(SeekFrom::Start(pos))
.map_err(|e| Error::IoError { source: e })?;
self.file
.write_all(&buf[..])
.map_err(|e| Error::IoError { source: e })?;
self.map
.record(pos, 2048, SectorStatus::Finished)
.map_err(|e| Error::IoError { source: e })?;
self.file.seek(SeekFrom::Start(pos))?;
self.file.write_all(&buf[..])?;
self.map.record(pos, 2048, SectorStatus::Finished)?;
}
WorkItem::BisectBad { pos } => {
self.file
.seek(SeekFrom::Start(pos))
.map_err(|e| Error::IoError { source: e })?;
self.file
.write_all(&self.zero[..2048])
.map_err(|e| Error::IoError { source: e })?;
self.map
.record(pos, 2048, SectorStatus::NonTrimmed)
.map_err(|e| Error::IoError { source: e })?;
self.file.seek(SeekFrom::Start(pos))?;
self.file.write_all(&self.zero[..2048])?;
self.map.record(pos, 2048, SectorStatus::NonTrimmed)?;
}
WorkItem::SkipFill { pos, len } | WorkItem::GapFill { pos, len } => {
self.file
.seek(SeekFrom::Start(pos))
.map_err(|e| Error::IoError { source: e })?;
self.file.seek(SeekFrom::Start(pos))?;
// Subsequent writes are sequential; `WritebackFile`'s
// seek-elision keeps them on the writeback pipeline path.
let mut filled = 0u64;
while filled < len {
let chunk = (len - filled).min(self.zero.len() as u64) as usize;
self.file
.write_all(&self.zero[..chunk])
.map_err(|e| Error::IoError { source: e })?;
self.file.write_all(&self.zero[..chunk])?;
filled += chunk as u64;
}
self.map
.record(pos, len, SectorStatus::NonTrimmed)
.map_err(|e| Error::IoError { source: e })?;
self.map.record(pos, len, SectorStatus::NonTrimmed)?;
}
WorkItem::StatsRequest => {
let stats = self.map.stats();
@@ -230,7 +203,7 @@ impl Sink<WorkItem> for SweepSink {
// Non-regular outputs (/dev/null, pipes) always fail
// sync_all; that's not a real error.
}
self.map.flush().map_err(|e| Error::IoError { source: e })?;
self.map.flush()?;
Ok(ConsumerSummary {
stats: self.map.stats(),