mirror of
https://github.com/freemkv/libfreemkv.git
synced 2026-10-07 18:13:10 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3244fdd683 | ||
|
|
ef5487e5a5 | ||
|
|
e7dffa63d2 | ||
|
|
e5989c8270 | ||
|
|
e1c938b82a | ||
|
|
ed801a708b | ||
|
|
d94451954c | ||
|
|
385c9f094c | ||
|
|
9f2a13739d | ||
|
|
d9ce69bc9d | ||
|
|
7cd2c937ed | ||
|
|
107524da5e | ||
|
|
055a3c5276 | ||
|
|
50b04790ce | ||
|
|
a00b2c884b | ||
|
|
44e2c64d7e | ||
|
|
d6535b8f57 | ||
|
|
90ab00ed45 | ||
|
|
6bfd7dfd13 | ||
|
|
b9a7f601d4 | ||
|
|
cf0a61f8b4 | ||
|
|
f98f07b2d3 | ||
|
|
b53454fa09 | ||
|
|
f28b6ee6d9 | ||
|
|
d9d778fa64 | ||
|
|
bbfb887a35 | ||
|
|
9884346c24 | ||
|
|
766b7c6636 | ||
|
|
925c30686b | ||
|
|
f52e4c5d22 | ||
|
|
8d67790d2e | ||
|
|
246a439990 | ||
|
|
bd19041648 | ||
|
|
198268b725 | ||
|
|
31424c8203 | ||
|
|
c747ecc589 | ||
|
|
283a561c12 | ||
|
|
2667d68675 | ||
|
|
57f2c22e30 | ||
|
|
5f3545d244 | ||
|
|
6e17ef0859 | ||
|
|
6ec97af104 | ||
|
|
3a6c1aa5a3 | ||
|
|
1ba3264747 | ||
|
|
ae2909fe8d | ||
|
|
6a3d19a453 | ||
|
|
a35596d2d1 | ||
|
|
318f654fed | ||
|
|
5e82ef65de | ||
|
|
8534607329 | ||
|
|
97e1a4cad3 | ||
|
|
3f6bf33de5 | ||
|
|
359301e8fc | ||
|
|
bfa527162a | ||
|
|
45defd47f3 | ||
|
|
fe8933dad2 | ||
|
|
0a3c1b7b70 | ||
|
|
2bcdc97341 | ||
|
|
4863e9c545 | ||
|
|
03db038dd9 | ||
|
|
95001ba2c3 | ||
|
|
a1f4dff6f6 | ||
|
|
7dd5001d45 | ||
|
|
67fe93c0b8 | ||
|
|
5c4f575c46 | ||
|
|
e2cb7a78bb | ||
|
|
a6f1bd19bc | ||
|
|
9e3d0f1383 | ||
|
|
1e6eb0698d | ||
|
|
bd744171e4 | ||
|
|
7de9d4d42c | ||
|
|
9ac0c2fcdf | ||
|
|
3aa27d5d1b | ||
|
|
5083f70cff | ||
|
|
477cf7ee0e | ||
|
|
0b079e27ce | ||
|
|
5f69fd2a44 | ||
|
|
befe6ef69b | ||
|
|
29491d6eaf | ||
|
|
2e602f7e37 | ||
|
|
a126f31b70 | ||
|
|
6ff10df83a | ||
|
|
52b8522a75 | ||
|
|
13eb4331b6 | ||
|
|
1a3f7ce676 | ||
|
|
e1a1f730be |
@@ -14,6 +14,10 @@ jobs:
|
||||
with:
|
||||
components: clippy, rustfmt
|
||||
- run: cargo fmt --check
|
||||
# libfreemkv is a library — Cargo.lock is gitignored. --locked
|
||||
# would always fail on a fresh runner because there's no committed
|
||||
# lockfile to lock against. The binary crates (freemkv, autorip,
|
||||
# bdemu) track Cargo.lock and DO use --locked.
|
||||
- run: cargo clippy -- -D warnings
|
||||
|
||||
test:
|
||||
|
||||
@@ -28,6 +28,8 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
# libfreemkv is a library — Cargo.lock isn't tracked, so --locked
|
||||
# would always fail (no lockfile to lock against on a fresh runner).
|
||||
- run: cargo test
|
||||
|
||||
publish:
|
||||
|
||||
+505
@@ -1,5 +1,510 @@
|
||||
# Changelog
|
||||
|
||||
## 0.18.4 (2026-05-09)
|
||||
|
||||
### Build / CI hardening — no library code changes
|
||||
|
||||
- All `cargo` invocations in `.github/workflows/*.yml` now use
|
||||
`--locked`. Previously `cargo build` / `cargo test` could silently
|
||||
re-resolve `Cargo.lock` if a lock-pinned version wasn't yet on
|
||||
crates.io — masking dependency races between same-tag releases of
|
||||
sibling crates. The 0.18.3 release hit this: autorip's docker image
|
||||
built `libfreemkv v0.18.2` because libfreemkv 0.18.3 hadn't
|
||||
published yet at the time autorip's CI ran, and cargo silently fell
|
||||
back to the previous version. With `--locked`, that race now hard-
|
||||
fails the build with a clear "package X is not in registry" error
|
||||
and we retrigger after the upstream lands.
|
||||
|
||||
- `cargo publish --locked` in the release workflow guards against
|
||||
publishing a lockfile-mismatched release.
|
||||
|
||||
## 0.18.3 (2026-05-09)
|
||||
|
||||
### Behaviour change
|
||||
|
||||
- **`Disc::titles[0]` is now the canonical main feature on branching
|
||||
discs, not the longest playlist.** Previously titles were sorted
|
||||
purely by `duration_secs` descending, which puts a "play-all"
|
||||
virtual playlist (alternate angles / seamless branching) at index 0
|
||||
on UHDs that ship one — its inflated duration overshoots the real
|
||||
movie. Concrete example: *The Amateur (2025)* 4K UHD on a 58.5 GB
|
||||
disc has Title 1 = `00020.mpls` 4h13m / 92.4 GB / 253 clips
|
||||
(impossible — the size exceeds the disc capacity, proving it's a
|
||||
virtual composite) and Title 2 = `00800.mpls` 2h02m / 57.2 GB /
|
||||
1 clip (the actual film, matching TMDB).
|
||||
|
||||
New sort priority:
|
||||
1. Real titles (`size_bytes <= capacity_bytes`) before virtual
|
||||
composites.
|
||||
2. Among real titles, fewer clips first (1-clip wins).
|
||||
3. Tiebreak on longer duration first.
|
||||
|
||||
**Migration:** consumers calling `disc.titles.first()` /
|
||||
`disc.titles[0]` automatically get the corrected title — no code
|
||||
changes needed. CLI users invoking `freemkv -t 1 disc:// …` now hit
|
||||
the actual main feature on branching discs (this was the user-
|
||||
visible bug). On non-branching discs the order is unchanged.
|
||||
|
||||
The comparator is exposed as
|
||||
[`Disc::canonical_title_order`](#) for callers that need to
|
||||
re-sort a custom title set with the same logic.
|
||||
|
||||
Regression tests: `disc::tests::canonical_order_*` (three cases:
|
||||
branching-UHD, normal disc, clip-count tiebreak).
|
||||
|
||||
## 0.18.2 (2026-05-09)
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- **Nav-file scramble during AACS rip** (`decrypt::decrypt_sectors`). 0.18's
|
||||
`DecryptingSectorSource` decorator broadened the call surface of
|
||||
`decrypt_sectors` to every sector flowing through sweep, including UDF
|
||||
navigation files (MPLS playlists, CLPI clip-info). The byte-0 heuristic
|
||||
in `aacs::is_unit_encrypted` correctly fires on m2ts source-packet copy
|
||||
markers but false-positives on any binary file whose first byte happens
|
||||
to have the top 2 bits set — most notably MPLS files (start with 'M' =
|
||||
0x4D) and CLPI files (start with 'H' = 0x48). `decrypt_unit_full`
|
||||
already self-checks the result via TS-sync verification and returns
|
||||
`false` on a misfire, but the chunk had been mutated by then;
|
||||
`decrypt_sectors` discarded the return value, leaving scrambled bytes
|
||||
in the ISO. Fix: snapshot the chunk before decryption and restore on
|
||||
verification failure (same pattern `decrypt_unit_try_keys` already used
|
||||
for multi-key discs). Symptom: `freemkv info iso://UHD.iso` and
|
||||
`iso:// → mkv://` returned E6009 NoStreams on freshly-ripped UHD ISOs;
|
||||
affected only the iso-source path, not the disc-source path or the
|
||||
m2ts video payload itself. Regression test:
|
||||
`decrypt::tests::nav_file_unit_survives_decrypt_attempt`.
|
||||
|
||||
- **Sweep progress display can regress to zero** (`Disc::sweep`). The
|
||||
consumer-side `bytes_good` snapshot lags producer-side `bytes_done`
|
||||
whenever the consumer is behind on draining the work channel. Until
|
||||
the first snapshot arrived, the placeholder branch reported
|
||||
`bytes_done` correctly; once a stale snapshot landed, the report
|
||||
switched to `snap.bytes_good` and could regress below `bytes_done`.
|
||||
Fix: `bytes_good_total = max(snap.bytes_good, bytes_done)`, so the
|
||||
user-visible counter never moves backward.
|
||||
|
||||
## 0.18.1 (2026-05-09)
|
||||
|
||||
### I/O stack redesign — primitives over orchestration
|
||||
|
||||
0.18 reshapes the read/write surface from "library does the multipass dance"
|
||||
to "library hands the caller flat verbs and a few composable primitives." All
|
||||
in-tree consumers (autorip, the `freemkv` CLI) drove their own multipass
|
||||
loops in 0.18 rounds 1-3; 0.18.1 lands the cleanup with the deprecated names
|
||||
still alive for one minor-version window.
|
||||
|
||||
The 0.18 design notes are in
|
||||
`(internal)/memory/0_18_redesign.md` (private) — this entry sticks to
|
||||
what changed at the public surface.
|
||||
|
||||
#### Flat verbs
|
||||
|
||||
`Disc::sweep` is the forward Pass 1. `Disc::patch` is one retry pass over the
|
||||
mapfile. Neither knows about pass index, retry budget, or accept-loss policy
|
||||
— the caller invokes them in whatever sequence its use case dictates. The
|
||||
old multipass-aware `Disc::copy` dispatcher is deprecated and slated for
|
||||
deletion in 0.18.2; no in-tree caller still uses it.
|
||||
|
||||
#### Trait splits — direction-typed at compile time
|
||||
|
||||
- `pes::Stream` (combined read+write) is split into `FrameSource` and
|
||||
`FrameSink`. Calling `read()` on a write-only sink is now a compile error,
|
||||
not the runtime `E9001` (`StreamWriteOnly`) it used to be.
|
||||
- `SectorReader` is split into `SectorSource` (read) and `SectorSink`
|
||||
(write). `Drive` impls `SectorSource` only; `FileSectorSource` /
|
||||
`FileSectorSink` replace `FileSectorReader` for ISO-backed I/O.
|
||||
- A blanket impl bridges legacy `SectorReader` callers onto the new
|
||||
`SectorSource` so existing code keeps compiling through the deprecation
|
||||
window.
|
||||
|
||||
#### New primitives
|
||||
|
||||
- `Halt` — one cancellation token (cloneable, `Arc<AtomicBool>` under the
|
||||
hood) replaces the three near-duplicate halt flags scattered through the
|
||||
workspace. Threaded through every long-running loop.
|
||||
- `Pipeline<I, R>` + `Sink<I>` — generic producer/consumer primitive in
|
||||
`crate::io`. Replaces the bespoke `disc/sweep_pipeline.rs` and now also
|
||||
drives `Disc::patch` and the autorip mux loop. `DEFAULT_PIPELINE_DEPTH`
|
||||
for streaming reads (`4`); `WRITE_THROUGH_DEPTH` for write-through patch
|
||||
semantics (`1`).
|
||||
- `WritebackFile` — was `crate::io::Writer`. The renamed type makes its job
|
||||
explicit: a `File` wrapper that runs continuous `sync_file_range` +
|
||||
`posix_fadvise(DONTNEED)` to keep the kernel dirty-page cache bounded on
|
||||
long sequential writes.
|
||||
- `DecryptingSectorSource<S>` — a single decorator wrapping any
|
||||
`SectorSource` to yield plaintext sectors. One audit surface for AACS /
|
||||
CSS / passthrough; the previous two-site decrypt (sweep producer +
|
||||
`DiscStream` demux) is gone.
|
||||
|
||||
#### Throughput
|
||||
|
||||
The round-2 producer/consumer split is now applied uniformly to sweep,
|
||||
patch, and mux. Mux on NFS-staged UHD measured ~16 MB/s sustained on the
|
||||
test bed (was ~12 MB/s pre-round-2).
|
||||
|
||||
#### Module reorg
|
||||
|
||||
- `sector/` and `io/` are now module directories.
|
||||
- `disc/sweep.rs`, `disc/patch.rs`, and `disc/mapfile.rs` split out of the
|
||||
monolithic `disc/mod.rs`.
|
||||
|
||||
#### Renames (no behavior change)
|
||||
|
||||
- `crate::io::Writer` → `crate::io::WritebackFile`.
|
||||
- `Apply` → `Flow` (`Sink::apply` return value).
|
||||
- `DEFAULT_DEPTH` → `DEFAULT_PIPELINE_DEPTH`.
|
||||
- `PatchOpts` → `PatchOptions`.
|
||||
|
||||
#### Deprecated (alive in 0.18.1, deletion target 0.18.2)
|
||||
|
||||
`Disc::copy`, `pes::Stream`, `SectorReader`, `FileSectorReader`,
|
||||
`CopyOptions`, `CopyResult`, `DiscStream::set_halt`. Each compiles with a
|
||||
deprecation warning; all in-tree call sites have migrated.
|
||||
|
||||
## 0.17.13 (2026-05-09)
|
||||
|
||||
### Use `crate::io::Writer` uniformly for all binary file output
|
||||
|
||||
0.17.10 introduced the bounded-cache writeback wrapper (`sync_file_range`
|
||||
+ `posix_fadvise(DONTNEED)` per chunk) and 0.17.11 wired it into
|
||||
`Disc::sweep`. The other two big-write paths in the crate were still
|
||||
opening raw `std::fs::File` and would have hit the same dirty-page
|
||||
burst pathology against slow / network-attached staging.
|
||||
|
||||
This release threads `crate::io::Writer` through the remaining sites:
|
||||
|
||||
- **`Disc::patch`** (`disc/mod.rs:1981`): the ISO file reopened for
|
||||
Pass-N recovery now wraps in `Writer` before any seek / write.
|
||||
Recovery writes are sparse, but the wrapper costs nothing when
|
||||
there's no chunk crossing — and on heavily-damaged discs it
|
||||
matters as patch accumulates GB of recovered data.
|
||||
- **MKV mux output** (`mux/resolve.rs:243`): `BufWriter::with_capacity`
|
||||
now wraps `Writer::new(file)` instead of a raw `File`. UHD MKVs
|
||||
routinely exceed 70 GB of sequential writes; pre-0.17.13 those
|
||||
bursts went straight to the kernel writeback queue.
|
||||
- **M2TS mux output** (`mux/resolve.rs:251`): same change for
|
||||
parity with the MKV path — anyone using `m2ts://` URLs gets it
|
||||
too.
|
||||
|
||||
No new public surface. `Writer::sync_all()` is called from `Disc::patch`
|
||||
and `sweep_pipeline`'s consumer at end of pass; mux exits via Drop on
|
||||
the wrapping `BufWriter`, which propagates flush down to `Writer`'s
|
||||
final `note_progress` (kernel finishes the in-flight chunk on file
|
||||
close — there's no explicit `sync_all` for mux today, same behaviour
|
||||
as before).
|
||||
|
||||
## 0.17.12 (2026-05-09)
|
||||
|
||||
### Mapfile time-batched persistence — unblock NFS staging
|
||||
|
||||
Pre-0.17.12 every `Mapfile::record()` wrote the entire mapfile to disk
|
||||
via tempfile-create + buffered-write + atomic-rename. On local LVM
|
||||
that's effectively free (page cache + microsecond-scale renames). On
|
||||
NFS (autorip's intended staging path for centralised media) each
|
||||
record() became three RPCs through the unraid user share's shfs-fuse
|
||||
layer — measured end-to-end at multiple ms each. With ~170 record()
|
||||
calls per second of sustained sweep, the mapfile path alone burned
|
||||
multiple seconds of wall time per real-world second of work, dragging
|
||||
the rip from ~11 MB/s on local to ~1.5 MB/s on NFS.
|
||||
|
||||
Fix: time-batch the persistence inside `Mapfile`.
|
||||
|
||||
- `record()` always updates in-memory state and stats (so callers'
|
||||
`stats()` reads stay coherent in the same process).
|
||||
- The `write_to_disk()` rename only fires when ≥ `FLUSH_INTERVAL`
|
||||
(1 s) has elapsed since the last persist.
|
||||
- New `flush()` method forces an out-of-band persist; called by
|
||||
`sweep_pipeline`'s consumer at end-of-sweep and by `Disc::patch` at
|
||||
end-of-patch, after the file's `sync_all()`.
|
||||
- `Drop` impl best-effort flushes so an early-return / unwind doesn't
|
||||
silently lose pending state.
|
||||
|
||||
Crash-safety changes from "lose at most one block" to "lose at most
|
||||
1 s of recorded progress" — the ISO file's payload bytes are
|
||||
unaffected; only the mapfile's authority over which sectors are
|
||||
already-good is at risk, and a resume re-reads anything Pass 1 had
|
||||
already covered. Acceptable for a 7× throughput recovery on the
|
||||
target deployment.
|
||||
|
||||
The internal `round_trip_load` test now calls `flush()` before
|
||||
`Mapfile::load` to read back what the in-memory state asserts.
|
||||
External patch / copy tests are unaffected: `patch` and
|
||||
`sweep_pipeline` flush at completion before returning, so any
|
||||
`Mapfile::load` at the call-site sees fully persisted state.
|
||||
|
||||
## 0.17.11 (2026-05-09)
|
||||
|
||||
### Sweep producer/consumer split — overlap drive read with file write
|
||||
|
||||
Pre-0.17.11 the sweep loop ran strictly serialised: SCSI read → decrypt
|
||||
→ seek + write → mapfile.record → next read. The drive idled for the
|
||||
post-read work; throughput capped at the **sum** of both costs. On a
|
||||
healthy disc that's ~7-12 ms read + ~5-15 ms write/record per 64 KB
|
||||
batch — limiting sustained throughput to ~10-12 MB/s on the test bed
|
||||
(BU40N + UHD inner zone), well below the drive's ceiling of ~14-16 MB/s.
|
||||
|
||||
This release decouples them with a producer / consumer split:
|
||||
|
||||
- **Producer** (caller's thread): owns the `SectorReader`, the entire
|
||||
`read_error` state machine (Retry, Bisect, SkipBlock, JumpAhead,
|
||||
AbortPass), `set_speed` damage-zone transitions, halt check, and
|
||||
decrypt. Hands plaintext bytes to the consumer.
|
||||
- **Consumer** (one spawned thread): owns the `crate::io::Writer` and
|
||||
the `Mapfile`. Receives `WorkItem` messages and applies the file
|
||||
write + mapfile record per item.
|
||||
- **Channel**: bounded `mpsc::sync_channel(4)` — natural back-pressure
|
||||
via blocking `send` when consumer falls behind.
|
||||
|
||||
While the consumer is writing batch N to disk and updating the mapfile,
|
||||
the producer is already reading batch N+1 from the drive. Steady-state
|
||||
throughput is now bound by the slower of the two pipelines (the drive,
|
||||
on a healthy disc) instead of their sum.
|
||||
|
||||
Side effects of the refactor:
|
||||
|
||||
- **Bisect path now decrypts.** Pre-0.17.11 the bisect inner loop wrote
|
||||
raw cyphertext when `decrypt=true` and a single sector was recovered
|
||||
via single-sector retry — a quiet correctness bug exercised only by
|
||||
the (rare) batch-fail-then-bisect-succeed path on encrypted discs.
|
||||
The new producer-side decrypt covers both the main success path and
|
||||
the bisect inner success path.
|
||||
- All `read_error::ReadCtx` state stays single-threaded on the producer
|
||||
(damage window, jump multiplier, consecutive-good count, etc.). No
|
||||
locking added.
|
||||
- Mapfile remains single-writer (consumer-only). No locking.
|
||||
- Halt-flag responsiveness unchanged: producer breaks the loop on
|
||||
signal, sends `Finish`, consumer drains its ≤4 in-flight items and
|
||||
exits within ~1 batch (~12 ms typical).
|
||||
- BU40N + Initio bridge wedge concern unchanged: still one SCSI command
|
||||
in flight, error-path timing identical, no new retry logic.
|
||||
|
||||
New module: `src/disc/sweep_pipeline.rs` (`WorkItem`, `ProgressSnapshot`,
|
||||
`ConsumerInputs`, `spawn_consumer`, `consumer_loop`, send/recv helpers).
|
||||
Public API surface unchanged — `Disc::copy` / `CopyOptions` /
|
||||
`CopyResult` look identical to callers.
|
||||
|
||||
Patch (Pass N) is **not** affected by this release. Patch is bound by
|
||||
drive recovery time (60 s timeouts on bad sectors), not the read↔write
|
||||
serialisation; a similar split there would yield negligible benefit.
|
||||
|
||||
## 0.17.10 (2026-05-09)
|
||||
|
||||
### Bounded-cache writeback for big sequential writes
|
||||
|
||||
Pass 1 sweep speed on a healthy disc previously dipped from ~15 MB/s
|
||||
to ~1 MB/s every ~30 s on a host with default Linux dirty-page
|
||||
settings. Cause: the kernel's `vm.dirty_ratio` (~20 % of RAM) lets
|
||||
hundreds of MB of dirty pages accumulate, then bursts a flush at
|
||||
99 % disk utilisation that blocks app writes for ~1 s. Confirmed
|
||||
empirically on the BU40N test bed: dirty pages grew 112 → 563 MB
|
||||
between bursts; lowering `vm.dirty_bytes` to 64 MB at the host
|
||||
sysctl level eliminated the dips.
|
||||
|
||||
This release ships the equivalent inside libfreemkv so users don't
|
||||
need to tune the host kernel:
|
||||
|
||||
- New `crate::io::Writer` — drop-in `File` wrapper implementing
|
||||
`Write` + `Seek`. Wraps a per-platform `WritebackPipeline` that on
|
||||
Linux schedules `sync_file_range(WRITE)` + lagging
|
||||
`sync_file_range(WAIT_AFTER)` + `posix_fadvise(DONTNEED)` calls in
|
||||
32 MB chunks, keeping dirty cache bounded at ~64 MB. macOS and
|
||||
Windows ship a no-op stub — their default cache policies don't
|
||||
exhibit the same pathology for our access pattern.
|
||||
- Disc::sweep wraps its output `File` in `Writer`. No changes to the
|
||||
loop body — `Writer` forwards `seek`/`write_all` to `File` and
|
||||
drives the pipeline transparently.
|
||||
- Module is purpose-built for any large sequential output (sweep,
|
||||
patch, future mux) — they can adopt `crate::io::Writer` with a
|
||||
one-line wrapper and inherit the same behaviour.
|
||||
|
||||
## 0.17.7 (2026-05-08)
|
||||
|
||||
### Sync release — no functional libfreemkv changes
|
||||
|
||||
Version bump to keep the four freemkv crates at unified versioning.
|
||||
autorip shipped UI smoothness + audit fixes today (v0.17.6, v0.17.7);
|
||||
libfreemkv carries no code changes, but is bumped + republished to
|
||||
crates.io so downstream `libfreemkv = "0.17"` consumers stay aligned
|
||||
on the latest patch version. See autorip CHANGELOG for the
|
||||
user-visible work in this release window.
|
||||
|
||||
## 0.17.5 (2026-05-08)
|
||||
|
||||
### Pass N recovery — kernel block-device fallback + per-range fixes
|
||||
|
||||
Live testing on direct-SATA BU40N + AACS-encrypted UHD disc revealed that
|
||||
the v0.17.3 single-shot SCSI READ path matched 0/22 of the small bad-sector
|
||||
LBAs that `dd if=/dev/sr0` recovers on the same drive. This release closes
|
||||
that gap and fixes several adjacent bugs that were silently capping
|
||||
recovery.
|
||||
|
||||
- **`/dev/sr0` pread fallback in `Drive::read` (Linux)**: when a SCSI READ
|
||||
via `/dev/sg*` returns Err, fall back to `posix_fadvise(POSIX_FADV_DONTNEED)
|
||||
+ pread()` against the corresponding block device. The kernel `sr_mod`
|
||||
driver runs ~5 internal retries per command without the per-attempt
|
||||
error-escalation overhead that userspace SG_IO retries pay, which is the
|
||||
source of dd's recovery advantage. End-to-end byte-verification confirms
|
||||
the fallback path returns real disc data (md5-equivalent to fresh dd from
|
||||
the same drive session). The block fd is opened in `Drive::open` by
|
||||
resolving `/sys/class/scsi_generic/sgN/device/block`; best-effort, with no
|
||||
fallback if open fails.
|
||||
|
||||
- **`Disc::patch` per-range watchdog fix**: when a range hit `MAX_RANGE_SECS`,
|
||||
the old code did `wedged_exit = true; break 'outer;` — a single slow range
|
||||
killed the entire patch. Now `break;` (skip this range, advance the outer
|
||||
for loop). Pre-fix, patch was dying after 4 sectors of the first slow
|
||||
range and never reaching the other 46.
|
||||
|
||||
- **Per-sector range budget**: replaced flat `MAX_RANGE_SECS = 180` with
|
||||
`range_budget_secs = (range_sectors × SECONDS_PER_SECTOR).min(RANGE_BUDGET_CAP_SECS)`.
|
||||
Tiny ranges exit fast (1-sector range = 25 s); medium ranges get
|
||||
proportional time (51-sector range ≈ 1275 s); large ranges still bounded
|
||||
by the 1800 s cap so they cannot monopolise pass 1.
|
||||
|
||||
- **`consecutive_failures` resets per range**: the wedge-exit detector
|
||||
(`>= 50 consecutive failures`) is for "stuck on the same range"; pre-fix
|
||||
the counter persisted across ranges, so 50 small post-bisection ranges
|
||||
with one failure each falsely tripped the wedge mid-pass. Reset at every
|
||||
range boundary.
|
||||
|
||||
- **Reverted inline 5× retry in patch**: a brief experiment that was
|
||||
measurably harmful — each "2 s" SCSI timeout paid ~1.5 s kernel SCSI
|
||||
mid-layer error-escalation overhead, so 5× retry took ~17 s per LBA and
|
||||
triggered the per-range watchdog in 4 sectors. Restored
|
||||
`READ_RECOVERY_TIMEOUT_MS = 60_000` (the v0.17.3 baseline; the kernel-
|
||||
auto-retry pattern is now provided by the `/dev/sr0` fallback above).
|
||||
|
||||
### Empirical results (Dune Part Two UHD, BU40N direct SATA)
|
||||
|
||||
- Pass 1: **94.6 MB recovered** (28% of formerly-bad data, 33 sr0 fallback
|
||||
saves), **11 s of main-title content** restored. Patch completed all 47
|
||||
retryable ranges naturally (was wedging at range 1 of 47 in v0.17.3).
|
||||
- Pass 2 cumulative: 95.2 MB (+0.6 MB; diminishing returns curve).
|
||||
- Remaining ~233 MB on the test disc appears physically unrecoverable on
|
||||
this hardware (kernel auto-retry can't decode it either).
|
||||
|
||||
### Behavioural notes
|
||||
|
||||
- The `/dev/sr0` fallback is Linux only; macOS and Windows fall back to
|
||||
the existing single-shot SCSI behaviour. The fallback is gated to
|
||||
`recovery=true` reads (only fires from the patch path, not the sweep
|
||||
path) to avoid page-cache pressure during multi-GB sequential ripping.
|
||||
|
||||
## 0.17.0 (2026-05-04)
|
||||
|
||||
### Code quality: unwrap safety, clippy compliance, test coverage
|
||||
|
||||
- **Sweep() hot path**: Replaced `.err().unwrap()` in `disc/mod.rs:1553` with explicit pattern matching (`match read_result { Ok(_) => unreachable!(), Err(e) => e }`). The original unwrap was logically safe (in the else branch after `is_ok()` check) but pattern matching makes the invariant explicit and avoids panic risk if logic changes.
|
||||
- **cargo clippy --lib**: Clean build with `-D warnings` across all targets. No lint failures.
|
||||
- **cargo test --lib**: All 256 tests pass (0 failed, 1 ignored). Integration tests for sweep/patch/dev-null validate the multi-pass recovery pipeline end-to-end.
|
||||
|
||||
### Patch pass algorithm fix
|
||||
|
||||
- `patch_internal` excludes Unreadable sectors from work list (only retries NonTrimmed/NonScraped/Unscraped). Previously attempted to retry already-failed sectors, wasting time and masking real progress.
|
||||
- Exposes `bytes_bad_in_title` in patch results for accurate UI reporting of unrecoverable data vs. recovered data.
|
||||
|
||||
## 0.16.1 (2026-04-30)
|
||||
|
||||
### Unified progress display for sweep and patch
|
||||
|
||||
- Patch passes now show `bytes_good / disc_total` as GB (grows when sectors are recovered), with pass progress `%` using `work_done / work_total` (always advances). `Xs unreadable` declines as data is recovered. Same UI for all pass types.
|
||||
|
||||
## 0.16.0 (2026-04-30)
|
||||
|
||||
### IOKit registry-based drive enumeration, BSD name matching, reverse patch default
|
||||
|
||||
- **`shim_open_exclusive`** now matches the correct `IOBDServices` for the requested BSD name. Walks all IOBDServices entries in the IOKit registry, matches child IOMedia `"BSD Name"` property. Falls back to IOMedia parent walk, then first-match. Fixes multi-drive systems.
|
||||
- **`shim_list_drives`** (new): registry-based drive enumeration. Reads IOBDServices `"Device Characteristics"` for vendor/model/firmware and child IOMedia for BSD name. Zero SCSI, zero exclusive access, zero unmounts. Fixes the enumeration blast that unmounted every disk on the system.
|
||||
- **`list_drives()`** and **`find_drives()`** rewritten to use `shim_list_drives`. No longer iterates `/dev/disk0..15` opening exclusive SCSI on each.
|
||||
- **`patch_internal`** defaults `reverse: true`. Sweep jumps forward with escalating gaps, so NonTrimmed ranges have good data at their tail. Reverse patch hits good data first, converges on actual bad block boundaries.
|
||||
|
||||
## 0.15.1 (2026-04-30)
|
||||
|
||||
### Fix damage-jump detection, fix dispatch covers_disc check
|
||||
|
||||
- Damage-jump tuning: `DAMAGE_WINDOW=16`, `DAMAGE_THRESHOLD_PCT=12%` (was 50/25%). Old params were too diluted by good reads between sparse failures; new params trigger on 2nd scattered failure.
|
||||
- `Disc::copy()` dispatch: `covers_disc` now checks `map.total_size() == disc_size` (byte-for-byte, not approximate). Fixes false sweep dispatch when mapfile exists but doesn't cover full disc.
|
||||
- Sweep resume: when dispatched from existing mapfile with NonTried, passes `resume: true`.
|
||||
|
||||
## 0.15.0 (2026-04-30)
|
||||
|
||||
### Multipass dispatch rewrite, speed control on damage zone entry/exit
|
||||
|
||||
- `CopyOptions { decrypt, multipass, progress, halt }`. `Disc::copy()` auto-detects sweep vs patch from mapfile state: no mapfile → sweep, NonTried → sweep with resume, only NonTrimmed/NonScraped/Unreadable → patch, clean → no-op.
|
||||
- `Disc::mapfile_for()`: `/dev/null` output → `/tmp/<disc_name>.mapfile`, otherwise `mapfile_path_for(path)`.
|
||||
- Speed control: damage zone entry → `set_speed(0x0000)` (minimum), 16 consecutive good reads → `set_speed(0xFFFF)` (maximum). Drive manages optimal speed in clean sections.
|
||||
- `SET CD SPEED` SCSI command via `SectorReader::set_speed()` (default no-op, Drive impl sends SCSI).
|
||||
|
||||
## 0.14.0 (2026-04-30)
|
||||
|
||||
### Damage-jump algorithm replaces probe, bridge degradation detection
|
||||
|
||||
- Damage-jump algorithm: when damage threshold exceeded in sliding window, jump ahead by `256×batch×multiplier` sectors. Doubles multiplier on each jump. Zero-fills gap as NonTrimmed.
|
||||
- `ecc_sectors()`: returns ECC block size per disc format (32 for UHD, 16 for BD, 16 for DVD).
|
||||
- Bridge degradation detection: NOT READY with sense key 2/ASC 0x04/ASCQ 0x3E triggers 10s cooldown, up to 5 times before treating as bad sector.
|
||||
|
||||
## 0.13.43 (2026-04-29)
|
||||
|
||||
### Pass 1 transport-failure recovery loop
|
||||
|
||||
- Transport failure (USB bridge crash) no longer kills the entire rip.
|
||||
- Autorip re-discovers the drive after USB re-enumeration and resumes
|
||||
from the mapfile. Up to 10 attempts.
|
||||
- `Error::is_scsi_transport_failure()` now matches DiscRead with
|
||||
status 0xFF (bridge crash) in addition to ScsiError.
|
||||
- DriveSession tracks device_path for re-discovery.
|
||||
|
||||
## 0.13.42 (2026-04-29)
|
||||
|
||||
### Fix: transport failure skips instead of aborting
|
||||
|
||||
- Transport failure (USB bridge crash) now skips the failed ECC block
|
||||
(marks NonTrimmed) and continues. 3 consecutive transport failures
|
||||
still abort the copy. Previously, a single transport failure killed
|
||||
the entire rip.
|
||||
|
||||
## 0.13.41 (2026-04-29)
|
||||
|
||||
### Debug logging for sector-0 regression diagnosis
|
||||
|
||||
- Add debug logging to Drive::read and Disc::copy first reads.
|
||||
- No functional changes.
|
||||
|
||||
## 0.13.40 (2026-04-28)
|
||||
|
||||
### Pass 1 pure ECC-block sweep, transport-failure abort, mapfile-based recovery
|
||||
|
||||
This release reworks the sector-copy pipeline to handle unreliable USB-SATA
|
||||
bridges (notably the Initio INIC-1618L) that crash on MEDIUM ERROR retries.
|
||||
|
||||
**Disc::copy() — Pass 1 (ECC-block sweep):**
|
||||
- Reads `batch` sectors (default 32 = 1 BD ECC block = 64 KB).
|
||||
- Success → mark Finished. MEDIUM ERROR → zero-fill, mark NonTrimmed, advance.
|
||||
- Transport failure (host_status=7) → abort immediately, return error.
|
||||
- No single-sector reads, no retry, no state machine in Pass 1.
|
||||
|
||||
**Disc::patch() — Pass 2+ (single-sector recovery):**
|
||||
- Reads NonTrimmed sectors one at a time with pause between failures.
|
||||
- Succeeds → mark Finished. Fails → mark Unreadable (or leave for next pass).
|
||||
- Multi-pass: caller runs patch repeatedly until 0 recovered.
|
||||
|
||||
**CopyOptions simplified:**
|
||||
- Removed `skip_forward`, `cautious_pause_ms`, `BPT1_EXIT_THRESHOLD`.
|
||||
- Fields: `decrypt`, `resume`, `batch_sectors`, `skip_on_error`, `progress`, `halt`.
|
||||
|
||||
**Other fixes since 0.13.26:**
|
||||
- `open()` just opens the device — no side effects. `drive_has_disc()` is a
|
||||
standalone TUR, not a probe sequence.
|
||||
- `enumerate_sg_names()` skips unreadable `/sys` type files.
|
||||
- SCSI sense data preserved in DiscRead errors (status + key + ASC/ASCQ).
|
||||
- MapStats splits `bytes_pending` into `nontried` / `retryable`.
|
||||
- Wallclock rip budget — halt after max(disc_runtime_secs, 3600).
|
||||
- Patch instrumentation: counters for reads_ok/err, writes_ok/err, finished/unreadable.
|
||||
- `as_encoded_bytes()` replaces `as_bytes()` for portable OsStr handling.
|
||||
- Removed inline retry/reset from Drive::read — orchestration layer handles recovery.
|
||||
- Removed all `(internal)` references from public code.
|
||||
|
||||
## 0.13.26 (2026-04-27)
|
||||
|
||||
### Extend DiscRead with SCSI status/sense for 30% wedge diagnostics
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "libfreemkv"
|
||||
version = "0.13.34"
|
||||
version = "0.18.7"
|
||||
edition = "2024"
|
||||
rust-version = "1.86"
|
||||
license = "AGPL-3.0-only"
|
||||
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
# libfreemkv — Feature List
|
||||
|
||||
## v0.10.10 (current)
|
||||
|
||||
### Done
|
||||
- [x] Drive access: open, identify, unlock, firmware upload, speed calibration, eject
|
||||
- [x] 206 bundled drive profiles (MediaTek MT1959 A + B variants)
|
||||
- [x] SCSI transport: Linux SG_IO, macOS IOKit, Windows SPTI
|
||||
- [x] UDF 2.50 filesystem parser (metadata partitions, Blu-ray profile)
|
||||
- [x] MPLS playlist parser (play items, STN table, secondary streams)
|
||||
- [x] CLPI clip info parser (EP map, sector extents)
|
||||
- [x] AACS 1.0 decryption (4 VUK paths: KEYDB, media key, processing key, device key)
|
||||
- [x] AACS 2.0 SCSI handshake (P-256/SHA-256 ECDH, bus decryption, read data key)
|
||||
- [x] DVD IFO parser (VMG, VTS, PGC chains, cell addresses)
|
||||
- [x] DVD CSS decryption (bus auth, disc key via player keys, title key, sector descramble)
|
||||
- [x] CSS Stevenson plaintext attack for ISO key recovery
|
||||
- [x] MPEG-2 PS demuxer (DVD Program Stream with PES extraction)
|
||||
- [x] MPEG-2 video codec parser (sequence header, quantizer matrices, keyframe detection)
|
||||
- [x] KEYDB.cfg download, verify, save (raw TCP, zero HTTP deps)
|
||||
- [x] Content reading with adaptive batch sizing, error recovery, 12+ MB/s
|
||||
- [x] Stream labels: 5 BD-J format parsers (Paramount, Criterion, Pixelogic, CTRM, Deluxe)
|
||||
- [x] MKV muxer: 14 codec parsers (H.264, HEVC, MPEG-2, AC-3, DTS, TrueHD, PGS, DVD Sub, VC-1, LPCM, +4 more)
|
||||
- [x] SectorReader trait: decouples disc scanning from SCSI
|
||||
- [x] 7 stream types: Disc, ISO, MKV, M2TS, Network, Stdio, Null
|
||||
- [x] PES pipeline with unified Stream trait (any source → any dest)
|
||||
- [x] FMKV metadata header for M2TS and network streams
|
||||
- [x] Numeric error codes only (no English text in library)
|
||||
- [x] Event system for progress callbacks
|
||||
|
||||
### Planned
|
||||
- [ ] Windows testing on real hardware
|
||||
- [ ] Pioneer Renesas platform support (48 drives, need GET_CONFIG 010C)
|
||||
- [ ] TranscodeStream (ffmpeg integration)
|
||||
- [ ] ISO write with BD-compliant UDF structure
|
||||
@@ -18,7 +18,7 @@ Part of the [freemkv](https://github.com/freemkv) project.
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
libfreemkv = "0.13"
|
||||
libfreemkv = "0.18"
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
@@ -53,27 +53,48 @@ output.finish()?;
|
||||
|
||||
### Multi-pass recovery rip
|
||||
|
||||
For damaged discs, the library offers a two-stage rip model: fast sweep with zero-fill and a ddrescue-format mapfile, then targeted retry of bad ranges. See [`docs/rip-recovery.md`](docs/rip-recovery.md) for the full architecture.
|
||||
For damaged discs the library exposes two flat verbs — `Disc::sweep` for the
|
||||
forward Pass 1 and `Disc::patch` for retrying bad ranges. The library never
|
||||
loops; the multipass policy is the caller's job. See
|
||||
[`docs/rip-recovery.md`](docs/rip-recovery.md) and the design notes in
|
||||
`(internal)/memory/0_18_redesign.md`.
|
||||
|
||||
```rust
|
||||
use libfreemkv::disc::{CopyOptions, PatchOptions};
|
||||
use libfreemkv::{SweepOptions, PatchOptions};
|
||||
use libfreemkv::disc::{mapfile, mapfile_path_for};
|
||||
use std::path::Path;
|
||||
|
||||
// Pass 1: disc → ISO. Fast 64 KB reads, skip-forward on failure,
|
||||
// zero-fill bad blocks, write a sidecar .mapfile.
|
||||
let mut result = disc.copy(
|
||||
&mut drive,
|
||||
Path::new("disc.iso"),
|
||||
&CopyOptions { skip_on_error: true, skip_forward: true, ..Default::default() },
|
||||
)?;
|
||||
let iso = Path::new("disc.iso");
|
||||
|
||||
// Pass 2..N: retry bad ranges with full drive recovery.
|
||||
// Idempotent — call as many times as you want.
|
||||
while result.bytes_unreadable + result.bytes_pending > 0 {
|
||||
let pr = disc.patch(&mut drive, Path::new("disc.iso"), &PatchOptions::default())?;
|
||||
if pr.bytes_recovered_this_pass == 0 { break; }
|
||||
// Pass 1: disc → ISO. Skip-on-error, zero-fill, write the sidecar mapfile.
|
||||
disc.sweep(&mut drive, iso, &SweepOptions {
|
||||
decrypt: true,
|
||||
resume: false,
|
||||
batch_sectors: None,
|
||||
skip_on_error: true,
|
||||
progress: None,
|
||||
halt: None,
|
||||
})?;
|
||||
|
||||
// Pass 2..N: retry every non-finished range. Idempotent.
|
||||
loop {
|
||||
let map = mapfile::Mapfile::load(&mapfile_path_for(iso))?;
|
||||
let stats = map.stats();
|
||||
if stats.bytes_pending + stats.bytes_unreadable == 0 { break; }
|
||||
|
||||
let outcome = disc.patch(&mut drive, iso, &PatchOptions {
|
||||
decrypt: true,
|
||||
block_sectors: None,
|
||||
full_recovery: true,
|
||||
reverse: true,
|
||||
wedged_threshold: 50,
|
||||
progress: None,
|
||||
halt: None,
|
||||
})?;
|
||||
if outcome.bytes_recovered_this_pass == 0 { break; }
|
||||
}
|
||||
|
||||
// Then mux from the ISO via the normal stream pipeline (no drive involvement).
|
||||
// Mux from the ISO via the normal stream pipeline (no drive involvement).
|
||||
```
|
||||
|
||||
## What It Does
|
||||
@@ -92,14 +113,14 @@ while result.bytes_unreadable + result.bytes_pending > 0 {
|
||||
| Stream | Input | Output | Transport |
|
||||
|--------|-------|--------|-----------|
|
||||
| DiscStream | Yes | -- | Optical drive via SCSI |
|
||||
| IsoStream | Yes | -- | Blu-ray ISO image file (read via stream pipeline; written via `Disc::copy()`) |
|
||||
| IsoStream | Yes | -- | Blu-ray ISO image file (read via stream pipeline; written via `Disc::sweep()`) |
|
||||
| MkvStream | Yes | Yes | Matroska container |
|
||||
| M2tsStream | Yes | Yes | BD transport stream with FMKV metadata header |
|
||||
| NetworkStream | Yes (listen) | Yes (connect) | TCP with FMKV metadata header |
|
||||
| StdioStream | Yes (stdin) | Yes (stdout) | Raw byte pipe |
|
||||
| NullStream | -- | Yes | Discard sink (byte counter for benchmarks) |
|
||||
|
||||
Streams implement `pes::Stream` (frame-level). `input()` / `output()` resolve URL strings to PES stream instances. All URLs use the `scheme://path` format — bare paths are rejected.
|
||||
Streams implement `FrameSource` (read) and/or `FrameSink` (write); direction is type-checked. `input()` / `output()` resolve URL strings to PES stream instances. All URLs use the `scheme://path` format — bare paths are rejected.
|
||||
|
||||
AACS decryption requires a KEYDB.cfg file. If available at `~/.config/aacs/KEYDB.cfg` or passed via `ScanOptions`, the library handles everything — handshake, key derivation, and per-sector decryption — without the application needing to know anything about encryption.
|
||||
|
||||
@@ -122,7 +143,8 @@ Disc — scan titles, streams, AACS/CSS state
|
||||
└── KEYDB — download + verify + save
|
||||
|
||||
Streams — unified PES pipeline
|
||||
├── pes::Stream — read()/write() PES frames
|
||||
├── FrameSource — read() PES frames (direction-typed)
|
||||
├── FrameSink — write() PES frames (direction-typed)
|
||||
├── DiscStream — sectors → decrypt → TS demux → PES
|
||||
├── IsoStream — ISO file → decrypt → TS demux → PES
|
||||
├── MkvStream — MKV mux/demux
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
# Troubleshooting Guide
|
||||
|
||||
Common problems and solutions for optical drive ripping with freemkv.
|
||||
|
||||
---
|
||||
|
||||
## 1. USB-SATA Bridge Issues
|
||||
|
||||
This is the single most common source of problems when ripping discs over USB.
|
||||
|
||||
### Symptoms
|
||||
|
||||
- The drive disappears mid-rip. The ripping tool reports the device is gone, and `ls /dev/sg*` no longer shows it.
|
||||
- The device re-enumerates under a different name: `sg4` becomes `sg5`, then `sg7`, then `sg11` after each USB port reset.
|
||||
- `dmesg` shows USB port resets: `usb X-Y: reset high-speed USB device`, `xhci_hcd 0000:00:14.0: Cannot enable. Maybe the USB cable is bad?`, or `usb-storage: device reset failed`.
|
||||
- The SCSI layer reports `host_status=7` (Linux USB transport error) in sense data.
|
||||
- The drive works fine for reading data discs or burning, but crashes when hitting damaged sectors during a rip.
|
||||
- After the crash, the drive is completely invisible until physically unplugged and reconnected.
|
||||
|
||||
### Root Cause
|
||||
|
||||
USB-SATA bridges translate between the USB Mass Storage protocol (BOT or UAS) and the drive's native SATA interface. When the optical drive encounters an unreadable sector, it returns a SCSI CHECK CONDITION with sense key 0x03 (MEDIUM ERROR). Some bridge chipsets -- particularly the Initio INIC-36xx family -- have buggy firmware that mishandles this error response.
|
||||
|
||||
Specific failure modes:
|
||||
|
||||
- **Incorrect residue reporting.** The bridge claims a different number of bytes transferred than what actually occurred. The Linux USB storage driver sees this discrepancy as a protocol violation and resets the port to recover. The `US_FL_IGNORE_RESIDUE` quirk exists specifically for this class of bug (see `drivers/usb/storage/transport.c` in the Linux kernel).
|
||||
- **Bridge firmware crash.** On some Initio bridges, a malformed SCSI error response from the drive causes the bridge MCU to hang entirely. The USB controller sees the device stop responding and initiates a port reset. The bridge recovers (it re-enumerates), but the rip is dead -- all state is lost.
|
||||
- **Sense data corruption.** The bridge forwards garbled or truncated sense data to the host, which the SCSI midlayer cannot parse, leading to a transport reset.
|
||||
|
||||
This is a hardware + firmware problem, not a software bug. The same drive connected via direct SATA does not exhibit these symptoms.
|
||||
|
||||
### Known Affected Bridges
|
||||
|
||||
| Chipset | USB IDs | Notes |
|
||||
|---------|---------|-------|
|
||||
| Initio INIC-3609 | `13fd:3609` | Very common in cheap SATA-to-USB enclosures. Highly problematic. |
|
||||
| Initio INIC-3619 | `13fd:3940` | Same firmware family as INIC-3609. |
|
||||
| Initio INIC-3069 | `13fd:0840` | Older variant, same residue bug. |
|
||||
| ASMedia ASM1051 | `174c:5106` | Early ASM SATA bridge. Residue issues on error paths. |
|
||||
| JMicron JMB36x | `152d:0561` | Some firmware versions. Not all JMicroon chips are affected. |
|
||||
|
||||
If your drive came in a pre-built external enclosure (Vantec, Sabrent, OWC, etc.), it almost certainly uses one of these bridge chips internally.
|
||||
|
||||
### The Fix: USB Storage Quirk
|
||||
|
||||
Apply the `US_FL_IGNORE_RESIDUE` kernel quirk for your bridge. This tells the Linux USB storage driver to ignore the residue field in SCSI response frames, preventing the port reset on mismatched byte counts.
|
||||
|
||||
**Step 1: Identify your bridge's vendor:product ID.**
|
||||
|
||||
```bash
|
||||
lsusb
|
||||
```
|
||||
|
||||
Look for your drive's entry. Example output:
|
||||
|
||||
```
|
||||
Bus 002 Device 005: ID 13fd:0840 Initio Corporation INIC-3609
|
||||
```
|
||||
|
||||
Here the vendor ID is `13fd` and the product ID is `0840`.
|
||||
|
||||
**Step 2: Apply the quirk at runtime.**
|
||||
|
||||
```bash
|
||||
echo "13fd:0840:i" > /sys/module/usb_storage/parameters/quirks
|
||||
```
|
||||
|
||||
Replace `13fd:0840` with your device's actual IDs. The `:i` flag means `US_FL_IGNORE_RESIDUE`.
|
||||
|
||||
You can combine multiple flags. Common additions:
|
||||
|
||||
- `:i` -- ignore residue (`US_FL_IGNORE_RESIDUE`)
|
||||
- `:u` -- force BOT mode instead of UAS, for bridges with UAS bugs
|
||||
|
||||
**Step 3: Reconnect the drive.** Unplug and replug the USB cable, or bind/unbind the device. The quirk is applied per-module-load, so existing sessions may need the drive reconnected.
|
||||
|
||||
### Making It Persistent
|
||||
|
||||
Add the quirk to your kernel boot parameters so it survives reboots.
|
||||
|
||||
Edit `/etc/default/grub` (GRUB) and add to `GRUB_CMDLINE_LINUX_DEFAULT`:
|
||||
|
||||
```
|
||||
GRUB_CMDLINE_LINUX_DEFAULT="quiet usb_storage.quirks=13fd:0840:i"
|
||||
```
|
||||
|
||||
Then rebuild the GRUB config:
|
||||
|
||||
```bash
|
||||
sudo update-grub
|
||||
```
|
||||
|
||||
For systemd-boot, add to your loader entry or `/etc/kernel/cmdline`:
|
||||
|
||||
```
|
||||
usb_storage.quirks=13fd:0840:i
|
||||
```
|
||||
|
||||
Multiple devices can be separated by commas:
|
||||
|
||||
```
|
||||
usb_storage.quirks=13fd:0840:i,174c:5106:u
|
||||
```
|
||||
|
||||
### Recommended Bridges
|
||||
|
||||
If you are buying a USB-SATA adapter or enclosure for optical drive use:
|
||||
|
||||
| Bridge | USB IDs | Notes |
|
||||
|--------|---------|-------|
|
||||
| ASMedia ASM1153 | `174c:1153` | Reliable. Widely available in SATA-USB 3.0 cables. |
|
||||
| JMicron JMS578 | `152d:0578` | Good firmware. Supports UASP. |
|
||||
| Icy Box IB-AC640-C3 | N/A | Uses a known-good bridge internally. Plug-and-play. |
|
||||
|
||||
Avoid any enclosure or adapter listing an Initio chipset.
|
||||
|
||||
### Best Solution: Direct SATA
|
||||
|
||||
Connect your optical drive directly to a motherboard SATA port. This eliminates the USB-SATA bridge entirely and is the most reliable configuration:
|
||||
|
||||
- No USB protocol overhead or translation errors.
|
||||
- No bridge firmware bugs.
|
||||
- No port resets or re-enumeration.
|
||||
- Full SATA error recovery handled natively by the kernel's libata driver.
|
||||
- Sustained read speeds are limited only by the drive, not the USB bus.
|
||||
|
||||
If your machine has a free SATA port, use it.
|
||||
|
||||
---
|
||||
|
||||
## 2. Damaged Disc Handling
|
||||
|
||||
### Symptoms
|
||||
|
||||
- SCSI MEDIUM ERROR (sense key 0x03) at specific LBAs. `dmesg` shows `sr X:0:0:0: [srY] Unrecoverable read error` or similar.
|
||||
- Read speed drops to near zero when approaching a damaged area.
|
||||
- The drive makes audible retrying noises (laser repositioning, spindle speed changes).
|
||||
- On USB-connected drives: the bridge crashes (see section 1 above) when the drive returns the error.
|
||||
|
||||
### How freemkv Handles This
|
||||
|
||||
freemkv uses a three-layer recovery model. See [`docs/rip-recovery.md`](docs/rip-recovery.md) for full details.
|
||||
|
||||
- **Pass 1 (Disc::copy):** Fast sweep with 64 KB reads. On failure, zero-fills the block and skips forward. Writes a ddrescue-format mapfile for later retry.
|
||||
- **Pass 2+ (Disc::patch):** Targeted re-reads of bad ranges with a long 30-second timeout per CDB. The drive firmware performs its own ECC and laser power retries within that window.
|
||||
- **In-stream (DiscStream):** Adaptive batch halving -- reduces request size on failure to isolate bad sectors within a larger block.
|
||||
|
||||
This means a disc with some bad sectors will still produce a usable ISO. The damaged areas are zero-filled in pass 1 and retried in subsequent passes. Structure-protected sectors (deliberate unreadable regions from copy protection) will never yield, which is expected.
|
||||
|
||||
### The Drive Taint Issue (LG BU40N)
|
||||
|
||||
Some drives, notably the LG BU40N, exhibit a "taint" behavior after encountering MEDIUM ERRORs:
|
||||
|
||||
1. The drive hits a damaged sector and returns a MEDIUM ERROR.
|
||||
2. From that point forward, **all subsequent reads fail** -- even reads to sectors that were previously successful.
|
||||
3. The only recovery is to physically unplug and reconnect the drive (or power-cycle it).
|
||||
|
||||
This is not a freemkv bug. It is a drive firmware behavior triggered by the interaction between the drive's internal error recovery and the USB-SATA bridge's handling of the error response. The drive firmware enters a degraded state that it does not recover from without a power cycle.
|
||||
|
||||
Workarounds:
|
||||
|
||||
- **Use a direct SATA connection.** This eliminates the bridge interaction that triggers the taint.
|
||||
- **Use a different bridge.** The ASM1153 and JMS578 are less likely to trigger this behavior.
|
||||
- **Accept the partial ISO.** freemkv's skip-forward recovery will zero-fill the unreadable blocks and continue. The resulting ISO may be playable with minor glitches in the affected areas.
|
||||
- **Physical replug between retry passes.** If running multi-pass patch, replug the drive between passes to clear the taint state.
|
||||
|
||||
freemkv deliberately does not attempt inline SCSI resets or eject cycles to recover from this state, because those operations were found to make the problem worse on affected hardware (see the design rationale in [`docs/rip-recovery.md`](docs/rip-recovery.md)).
|
||||
|
||||
---
|
||||
|
||||
## 3. Drive Not Detected
|
||||
|
||||
### Check Hardware Visibility
|
||||
|
||||
```bash
|
||||
lsusb
|
||||
```
|
||||
|
||||
Verify the drive appears in the USB device list. If it does not show up, the drive is not visible to the host at all -- check cables, power, and USB port.
|
||||
|
||||
```bash
|
||||
ls /dev/sg*
|
||||
```
|
||||
|
||||
On Linux, optical drives appear as `/dev/sg*` devices (the SCSI Generic interface). freemkv uses `/dev/sg*`, not `/dev/sr*`. If `lsusb` shows the device but no `/dev/sg*` entry exists, the `sg` kernel module may not be loaded:
|
||||
|
||||
```bash
|
||||
sudo modprobe sg
|
||||
```
|
||||
|
||||
### Check Kernel Messages
|
||||
|
||||
```bash
|
||||
dmesg | grep -i usb | tail -30
|
||||
dmesg | grep -i sg | tail -10
|
||||
```
|
||||
|
||||
Look for:
|
||||
- USB enumeration errors or failed port resets.
|
||||
- `sg_add` messages confirming the sg device was registered.
|
||||
- Permission denied or access errors.
|
||||
|
||||
### Permission Issues
|
||||
|
||||
On most Linux distributions, `/dev/sg*` devices are owned by `root:disk` or `root:cdrom` with restricted permissions. Running freemkv as an unprivileged user will fail with permission errors.
|
||||
|
||||
Options:
|
||||
|
||||
- Add your user to the appropriate group:
|
||||
|
||||
```bash
|
||||
sudo usermod -aG disk $USER
|
||||
```
|
||||
|
||||
Then log out and back in for the change to take effect. On some distributions the group is `cdrom` or `optical` instead of `disk`.
|
||||
|
||||
- Run with elevated privileges:
|
||||
|
||||
```bash
|
||||
sudo freemkv ...
|
||||
```
|
||||
|
||||
- Install a udev rule for persistent per-device permissions. Create `/etc/udev/rules.d/99-sg-optical.rules`:
|
||||
|
||||
```
|
||||
SUBSYSTEM=="scsi_generic", ATTRS{type}=="5", MODE="0666"
|
||||
```
|
||||
|
||||
Then reload udev rules:
|
||||
|
||||
```bash
|
||||
sudo udevadm control --reload-rules && sudo udevadm trigger
|
||||
```
|
||||
|
||||
### Spin-Up Delay
|
||||
|
||||
Optical drives take 30-60 seconds to spin up and become ready after hot-plug or disc insertion. During this window, SCSI commands may return NOT READY or timeout.
|
||||
|
||||
freemkv's `Drive::wait_ready()` handles this automatically by polling with TEST UNIT READY until the drive responds. If you are writing your own code using the library, always call `wait_ready()` before `init()`:
|
||||
|
||||
```rust
|
||||
let mut drive = Drive::open(Path::new("/dev/sg4"))?;
|
||||
drive.wait_ready()?; // blocks until disc is ready, up to 30s
|
||||
drive.init()?;
|
||||
```
|
||||
|
||||
If the drive was just plugged in, wait a full minute before concluding it is not detected.
|
||||
|
||||
---
|
||||
|
||||
## 4. How to Identify Your USB-SATA Bridge
|
||||
|
||||
If you are experiencing the issues described in section 1, you need to know which bridge chipset your adapter or enclosure uses.
|
||||
|
||||
### Step 1: Find the Device
|
||||
|
||||
```bash
|
||||
lsusb
|
||||
```
|
||||
|
||||
Look for entries matching your drive or enclosure. Bridges may appear under their own manufacturer name or as a generic SATA device. Common examples:
|
||||
|
||||
```
|
||||
Bus 002 Device 005: ID 13fd:0840 Initio Corporation
|
||||
Bus 002 Device 006: ID 174c:1153 ASMedia Technology Inc. ASM1153
|
||||
Bus 002 Device 007: ID 152d:0578 JMicron Technology Corp. JMS578
|
||||
```
|
||||
|
||||
### Step 2: Match the IDs
|
||||
|
||||
| Vendor | Product ID | Chipset | Status |
|
||||
|--------|-----------|---------|--------|
|
||||
| `13fd` | `3609` | Initio INIC-3609 | Affected. Apply quirk. |
|
||||
| `13fd` | `3940` | Initio INIC-3619 | Affected. Apply quirk. |
|
||||
| `13fd` | `0840` | Initio INIC-3069 | Affected. Apply quirk. |
|
||||
| `174c` | `5106` | ASMedia ASM1051 | Affected (early firmware). Apply quirk. |
|
||||
| `174c` | `1153` | ASMedia ASM1153 | Good. No quirk needed. |
|
||||
| `152d` | `0561` | JMicron JMB36x | Affected (some firmware). Apply quirk if issues occur. |
|
||||
| `152d` | `0578` | JMicron JMS578 | Good. No quirk needed. |
|
||||
|
||||
### Step 3: Check dmesg for the Bridge Name
|
||||
|
||||
```bash
|
||||
dmesg | grep -i "usb-storage\|uas\|initio\|asmedia\|jmicron"
|
||||
```
|
||||
|
||||
This often reveals the bridge chipset even when `lsusb` shows a generic name.
|
||||
|
||||
### Step 4: If the Enclosure Is Sealed
|
||||
|
||||
Many external drive enclosures (Vantec NexStar, Sabrent, OWC, etc.) do not advertise the bridge chipset on the packaging. In this case:
|
||||
|
||||
1. Check `lsusb` while the enclosure is connected.
|
||||
2. Search the vendor:product ID online -- there are community-maintained lists of which chipsets popular enclosures use.
|
||||
3. If you cannot determine the chipset and are experiencing bridge crashes, assume it is an Initio and apply the quirk with its IDs.
|
||||
4. The definitive test: connect the bare drive to a motherboard SATA port. If the problems disappear, the bridge was the cause.
|
||||
|
||||
---
|
||||
|
||||
## 5. General Debugging Checklist
|
||||
|
||||
When something goes wrong during a rip, gather this information before filing an issue:
|
||||
|
||||
1. **freemkv version:** `freemkv --version` or the crate version in `Cargo.toml`.
|
||||
2. **Drive model:** from the drive label, or from `freemkv info`.
|
||||
3. **Connection type:** USB (with bridge chipset if known) or direct SATA.
|
||||
4. **Operating system and kernel:** `uname -a`.
|
||||
5. **Kernel messages during the failure:** `dmesg | tail -50` immediately after the crash.
|
||||
6. **SCSI device:** which `/dev/sg*` the drive was on, and whether it changed after the failure.
|
||||
7. **The disc:** title, format (BD/DVD/UHD), condition.
|
||||
|
||||
Include all of the above in bug reports. SCSI transport errors that resolve with the `US_FL_IGNORE_RESIDUE` quirk or by switching to direct SATA are bridge firmware bugs, not freemkv bugs.
|
||||
@@ -3,5 +3,34 @@ fn main() {
|
||||
if target == "macos" {
|
||||
println!("cargo:rustc-link-lib=framework=IOKit");
|
||||
println!("cargo:rustc-link-lib=framework=CoreFoundation");
|
||||
|
||||
let out_dir = std::env::var("OUT_DIR").unwrap();
|
||||
let obj = format!("{out_dir}/macos_shim.o");
|
||||
let lib = format!("{out_dir}/libmacos_scsi.a");
|
||||
|
||||
std::process::Command::new("cc")
|
||||
.args([
|
||||
"-c",
|
||||
"src/scsi/macos_shim.c",
|
||||
"-o",
|
||||
&obj,
|
||||
"-framework",
|
||||
"IOKit",
|
||||
"-framework",
|
||||
"CoreFoundation",
|
||||
"-Wall",
|
||||
"-O2",
|
||||
])
|
||||
.status()
|
||||
.expect("failed to compile macos_shim.c");
|
||||
|
||||
std::process::Command::new("ar")
|
||||
.args(["rcs", &lib, &obj])
|
||||
.status()
|
||||
.expect("failed to create static lib");
|
||||
|
||||
println!("cargo:rustc-link-search=native={out_dir}");
|
||||
println!("cargo:rustc-link-lib=static=macos_scsi");
|
||||
println!("cargo:rerun-if-changed=src/scsi/macos_shim.c");
|
||||
}
|
||||
}
|
||||
|
||||
+39
-37
@@ -44,35 +44,30 @@ while let Ok(Some(frame)) = input.read() {
|
||||
output.finish()?;
|
||||
```
|
||||
|
||||
The `pes::Stream` trait:
|
||||
The `FrameSource` and `FrameSink` traits — direction is type-checked, so
|
||||
calling `read()` on a write-only sink (or `write()` on a read-only source)
|
||||
is a compile error rather than a runtime fault:
|
||||
|
||||
```rust
|
||||
pub trait Stream {
|
||||
fn read(&mut self) -> io::Result<Option<PesFrame>>;
|
||||
fn write(&mut self, frame: &PesFrame) -> io::Result<()>;
|
||||
fn finish(&mut self) -> io::Result<()>;
|
||||
pub trait FrameSource: Send {
|
||||
fn read(&mut self) -> Result<Option<PesFrame>, Error>;
|
||||
fn info(&self) -> &DiscTitle;
|
||||
fn codec_private(&self, track: usize) -> Option<Vec<u8>>;
|
||||
fn headers_ready(&self) -> bool;
|
||||
fn codec_private(&self, track: usize) -> Option<Vec<u8>> { None }
|
||||
fn headers_ready(&self) -> bool { true }
|
||||
}
|
||||
```
|
||||
|
||||
## IOStream (byte-level API)
|
||||
|
||||
For raw byte copies (disc→ISO, resume, benchmarks). Lower level than PES.
|
||||
|
||||
```rust
|
||||
let opts = InputOptions::default();
|
||||
let mut input = open_input("iso://Disc.iso", &opts)?;
|
||||
let mut output = open_output("mkv://Movie.mkv", input.info())?;
|
||||
io::copy(&mut *input, &mut *output)?;
|
||||
output.finish()?;
|
||||
pub trait FrameSink: Send {
|
||||
fn write(&mut self, frame: &PesFrame) -> Result<(), Error>;
|
||||
fn finish(self: Box<Self>) -> Result<(), Error>;
|
||||
fn info(&self) -> &DiscTitle;
|
||||
}
|
||||
```
|
||||
|
||||
## Streams
|
||||
|
||||
All streams implement `IOStream` (byte-level) and/or `pes::Stream` (frame-level).
|
||||
URL-based resolvers open any stream by string.
|
||||
All streams implement `FrameSource` (read) and/or `FrameSink` (write); the
|
||||
directional split prevents runtime "wrong-direction" errors. URL-based
|
||||
resolvers open any stream by string.
|
||||
|
||||
| Stream | Input | Output | URL | Transport |
|
||||
|--------|-------|--------|-----|-----------|
|
||||
@@ -87,21 +82,14 @@ URL-based resolvers open any stream by string.
|
||||
All URLs require a `scheme://path` format. Bare paths are rejected.
|
||||
|
||||
```rust
|
||||
// PES pipeline (frame-level)
|
||||
// PES pipeline (frame-level) — input() returns Box<dyn FrameSource>,
|
||||
// output() returns Box<dyn FrameSink>.
|
||||
let input = libfreemkv::input("disc:///dev/sg4", &opts)?; // DiscStream
|
||||
let input = libfreemkv::input("iso://Dune.iso", &opts)?; // IsoStream
|
||||
let output = libfreemkv::output("mkv://Dune.mkv", &title)?; // MkvOutputStream
|
||||
let output = libfreemkv::output("m2ts://Dune.m2ts", &title)?; // M2tsOutputStream
|
||||
let output = libfreemkv::output("network://10.0.0.1:9000", &title)?; // NetworkOutputStream
|
||||
let output = libfreemkv::output("null://", &title)?; // NullOutputStream
|
||||
|
||||
// IOStream (byte-level)
|
||||
let input = open_input("disc://", &opts)?; // DiscStream
|
||||
let input = open_input("iso://Dune.iso", &opts)?; // IsoStream
|
||||
let output = open_output("iso://Copy.iso", &meta)?; // IsoStream (write)
|
||||
let output = open_output("mkv://Dune.mkv", &meta)?; // MkvStream
|
||||
let output = open_output("m2ts://Dune.m2ts", &meta)?; // M2tsStream
|
||||
let output = open_output("null://", &meta)?; // NullStream
|
||||
```
|
||||
|
||||
### FMKV Metadata Header
|
||||
@@ -175,20 +163,34 @@ libfreemkv/src/
|
||||
├── lib.rs Public exports
|
||||
├── error.rs Error codes (no English)
|
||||
├── event.rs Event types for callbacks
|
||||
├── halt.rs Halt cancellation token (Arc<AtomicBool> wrapper)
|
||||
├── io/ Pipeline + WritebackFile primitives
|
||||
│ ├── mod.rs Re-exports WritebackFile, Pipeline, Sink, Flow
|
||||
│ ├── pipeline.rs Generic Pipeline<I, R> + Sink trait
|
||||
│ ├── writeback_file.rs WritebackFile (was crate::io::Writer)
|
||||
│ └── writeback.rs sync_file_range pipeline
|
||||
├── drive/ Drive (open, init, single-shot read)
|
||||
│ ├── mod.rs Drive struct, init, read (single-shot), reset, eject
|
||||
│ ├── capture.rs Drive profile capture for contribution
|
||||
│ ├── linux.rs Linux drive discovery
|
||||
│ ├── macos.rs macOS drive discovery
|
||||
│ └── windows.rs Windows drive discovery
|
||||
├── disc/ Disc (scan, titles, AACS setup)
|
||||
├── disc/ Disc (scan, titles, AACS setup, sweep, patch)
|
||||
│ ├── mod.rs Disc struct, scan, titles, formats
|
||||
│ ├── sweep.rs Disc::sweep (Pass 1 forward sweep)
|
||||
│ ├── patch.rs Disc::patch (Pass N retry over mapfile)
|
||||
│ ├── mapfile.rs ddrescue-format mapfile
|
||||
│ └── read_error.rs ReadCtx / ReadAction state machine
|
||||
├── scsi/ SCSI transport (Linux SG_IO, macOS IOKit, Windows SPTI)
|
||||
├── platform/ Drive unlock (MT1959 A/B)
|
||||
├── aacs/ AACS decryption (handshake, keys, keydb, decrypt)
|
||||
├── css/ DVD CSS cipher
|
||||
├── decrypt.rs Unified decrypt dispatcher (AACS/CSS/None)
|
||||
├── pes.rs PES frame types, Stream trait
|
||||
├── sector.rs SectorReader trait
|
||||
├── pes.rs PES frame types, FrameSource / FrameSink traits
|
||||
├── sector/ Sector I/O (was sector.rs in 0.17)
|
||||
│ ├── mod.rs SectorSource, SectorSink traits
|
||||
│ ├── file.rs FileSectorSource, FileSectorSink (ISO-backed)
|
||||
│ └── decrypting.rs DecryptingSectorSource decorator
|
||||
├── udf.rs UDF 2.50 filesystem parser
|
||||
├── mpls.rs MPLS playlist parser
|
||||
├── clpi.rs CLPI clip info parser
|
||||
@@ -199,15 +201,15 @@ libfreemkv/src/
|
||||
├── profile.rs Bundled drive profiles
|
||||
├── speed.rs DriveSpeed enum
|
||||
├── mux/
|
||||
│ ├── mod.rs IOStream trait, public exports
|
||||
│ ├── resolve.rs URL parser + open_input/open_output + input/output
|
||||
│ ├── mod.rs Public mux exports
|
||||
│ ├── resolve.rs URL parser + input/output (Box<dyn FrameSource/Sink>)
|
||||
│ ├── meta.rs FMKV header format
|
||||
│ ├── disc.rs DiscStream (optical drive → PES)
|
||||
│ ├── iso.rs IsoStream (ISO image read/write)
|
||||
│ ├── iso.rs IsoStream (ISO image read)
|
||||
│ ├── isowriter.rs ISO image writer (UDF, AVDP, multi-extent)
|
||||
│ ├── mkvstream.rs MkvStream (bidirectional Matroska, IOStream)
|
||||
│ ├── mkvstream.rs MkvStream (bidirectional Matroska)
|
||||
│ ├── mkvout.rs MkvOutputStream (PES → MKV)
|
||||
│ ├── m2ts.rs M2tsStream (BD-TS, IOStream)
|
||||
│ ├── m2ts.rs M2tsStream (BD-TS)
|
||||
│ ├── pesout.rs PES output streams (M2ts, Network, Stdio, Null)
|
||||
│ ├── network.rs NetworkStream (TCP + FMKV header)
|
||||
│ ├── stdio.rs StdioStream (stdin/stdout pipe)
|
||||
|
||||
@@ -65,8 +65,12 @@ libfreemkv (lib.rs)
|
||||
│
|
||||
├── Streaming
|
||||
│ ├── mux/ Stream implementations (Disc, ISO, MKV, M2TS, Network, Stdio, Null)
|
||||
│ ├── pes PES frame types, Stream trait (read/write frames)
|
||||
│ └── sector SectorReader trait — abstracts disc vs ISO vs file
|
||||
│ ├── pes PES frame types; FrameSource / FrameSink direction-typed traits
|
||||
│ └── sector/ SectorSource / SectorSink traits, FileSector{Source,Sink}, DecryptingSectorSource
|
||||
│
|
||||
├── I/O Primitives
|
||||
│ ├── halt Halt cancellation token (one Arc<AtomicBool>, cloneable)
|
||||
│ └── io/ Pipeline<I, R> + Sink trait + WritebackFile (bounded-cache writer)
|
||||
│
|
||||
├── Support
|
||||
│ ├── keydb KEYDB.cfg download, parse, verify, save
|
||||
@@ -102,7 +106,7 @@ Recovery is layered above `Drive::read`, not inside it. Layer 1
|
||||
Layer 3 (`DiscStream::fill_extents` adaptive batch sizer) handles in-loop
|
||||
request-size adaptation. Inline recovery (gentle retry → SCSI reset → retry)
|
||||
was removed in 0.13.6 — see [`rip-recovery.md`](rip-recovery.md) and
|
||||
`(internal)/postmortems/2026-04-25-stop-wedge-and-zero-kbs.md`.
|
||||
the stop-wedge postmortem (2026-04-25).
|
||||
|
||||
---
|
||||
|
||||
|
||||
+8
-4
@@ -97,7 +97,9 @@ drive.probe_disc()?;
|
||||
// Scan disc (UDF + playlists + AACS — all automatic)
|
||||
let disc = Disc::scan(&mut drive, &ScanOptions::default())?;
|
||||
|
||||
// Stream pipeline — PES frames from any source to any output
|
||||
// Stream pipeline — PES frames from any source to any output.
|
||||
// 0.18: input() returns Box<dyn FrameSource>, output() returns Box<dyn FrameSink>;
|
||||
// direction is type-checked, so calling .write() on an input is a compile error.
|
||||
let opts = InputOptions::default();
|
||||
let mut input = libfreemkv::input("disc:///dev/sg4", &opts)?;
|
||||
let title = input.info().clone();
|
||||
@@ -121,11 +123,13 @@ output.finish()?;
|
||||
| aacs/ | [aacs.md](aacs.md) | Key resolution + content decrypt + bus handshake |
|
||||
| css/ | -- | DVD CSS cipher |
|
||||
| decrypt.rs | -- | Unified decrypt dispatcher (AACS/CSS/None) |
|
||||
| disc/ | -- | High-level scan + read API |
|
||||
| disc/ | [rip-recovery.md](rip-recovery.md) | Disc::scan + Disc::sweep + Disc::patch + mapfile |
|
||||
| labels/ | -- | BD-J stream labels (5 format parsers) |
|
||||
| mux/ | -- | Stream implementations (7 stream types) |
|
||||
| pes.rs | -- | PES frame types + Stream trait |
|
||||
| sector.rs | -- | SectorReader trait |
|
||||
| pes.rs | -- | PES frame types + FrameSource / FrameSink traits |
|
||||
| sector/ | -- | SectorSource / SectorSink + DecryptingSectorSource decorator |
|
||||
| io/ | -- | Pipeline<I, R> + Sink trait + WritebackFile |
|
||||
| halt.rs | -- | Halt cancellation token |
|
||||
| keydb.rs | -- | KEYDB download, parse, save |
|
||||
| error.rs | -- | Error codes (E1xxx-E8xxx) |
|
||||
| event.rs | -- | Drive event system |
|
||||
|
||||
+14
-4
@@ -52,8 +52,8 @@ selects the per-CDB timeout:
|
||||
|
||||
| `recovery` | Timeout | Used by |
|
||||
|------------|----------|------------------------------------------|
|
||||
| `false` | 1.5 s | `Disc::copy` fast skip-forward sweep, `DiscStream::fill_extents` |
|
||||
| `true` | 30 s | `Disc::patch` multi-pass over the mapfile |
|
||||
| `false` | 1.5 s | `Disc::sweep` fast skip-forward pass, `DiscStream::fill_extents` |
|
||||
| `true` | 30 s | `Disc::patch` retry pass over the mapfile |
|
||||
|
||||
On any SCSI failure or timeout, `read` returns `Err(DiscRead)` immediately.
|
||||
There are no inline retries, no SCSI reset, no Phase 1/2/3 escalation.
|
||||
@@ -67,7 +67,7 @@ Recovery is layered above `Drive::read`:
|
||||
streak.
|
||||
|
||||
Inline recovery (5× gentle retry → close + reset + reopen → 5× more) was
|
||||
removed in 0.13.6. See `(internal)/postmortems/2026-04-25-stop-wedge-and-zero-kbs.md`
|
||||
removed in 0.13.6. See the stop-wedge postmortem (2026-04-25)
|
||||
for rationale: the inline reset wedged drive firmware on the LG BU40N (Initio
|
||||
USB-SATA bridge) without ever recovering a sector. See
|
||||
[`rip-recovery.md`](rip-recovery.md) for the full three-layer model.
|
||||
@@ -99,7 +99,6 @@ descriptors or calls ioctls outside of a `ScsiTransport` implementation.
|
||||
|----------|---------------|--------|
|
||||
| Linux | `SgIoTransport` — async `write`/`poll`/`read` on `/dev/sg*` | `/dev/sg*` |
|
||||
| macOS | `MacScsiTransport` — IOKit SCSITask | IOKit service |
|
||||
| Windows | `WindowsScsiTransport` — SPTI | `\\.\CdRomN` |
|
||||
|
||||
The Linux backend uses the sg driver's asynchronous interface: `write()` submits
|
||||
the command, `poll()` waits with an enforceable wall-clock timeout, `read()`
|
||||
@@ -107,6 +106,17 @@ retrieves the result. If `poll()` times out, the fd is abandoned (closed in a
|
||||
background thread) and a fresh fd opened — the kernel's USB error recovery
|
||||
cannot block us. Opens with `O_RDWR | O_NONBLOCK`.
|
||||
|
||||
The macOS backend uses a C shim (`macos_shim.c`) for IOKit exclusive access.
|
||||
The shim handles:
|
||||
1. `shim_open_exclusive(bsd_name)` — unmounts the target device via `diskutil`,
|
||||
then walks the IOKit registry to find the `IOBDServices` matching the
|
||||
requested BSD name (IOBDServices → IOBDBlockStorageDriver → IOMedia → "BSD Name"),
|
||||
then creates MMCDeviceInterface → SCSITaskDeviceInterface → ObtainExclusiveAccess.
|
||||
2. `shim_list_drives()` — registry-based enumeration with zero SCSI, zero exclusive
|
||||
access, zero unmounts. Reads IOBDServices "Device Characteristics" for
|
||||
vendor/model/firmware and child IOMedia "BSD Name" for the device path.
|
||||
3. `shim_execute()` / `shim_close()` — raw CDB dispatch and cleanup.
|
||||
|
||||
On non-zero SCSI status, the transport parses sense key from the sense buffer
|
||||
and returns `Error::ScsiError`.
|
||||
|
||||
|
||||
+64
-44
@@ -12,21 +12,24 @@ reach into the others.
|
||||
|
||||
| Layer | Where it lives | What it does |
|
||||
|-------|---------------|--------------|
|
||||
| 1 — Bad-range retry | `Disc::patch` (multi-pass over the mapfile) | Re-reads non-`+` ranges with the long timeout. Idempotent; call N times. |
|
||||
| 1 — Bad-range retry | `Disc::patch` (one pass over the mapfile per call) | Re-reads non-`+` ranges with the long timeout. Idempotent; caller invokes N times. |
|
||||
| 2 — Single-shot primitive | `Drive::read` in `src/drive/mod.rs` | One CDB, one timeout, one result. No inline retries, no SCSI reset. |
|
||||
| 3 — In-loop request adaptation | `DiscStream::fill_extents` adaptive batch sizer | Halves the batch on failure, retries at the same LBA, walks back up on a clean-read streak. |
|
||||
|
||||
The caller orchestrates layer 1. Autorip's `rip_disc` loops `copy` then
|
||||
N × `patch` per the `MAX_RETRIES` config, then hands the ISO off to the
|
||||
existing mux pipeline. Layer 3 runs inside any consumer of `DiscStream`
|
||||
(direct PES pipeline, ISO playback, etc.) without caller involvement.
|
||||
The library exposes flat verbs; the caller drives the multipass loop. Autorip
|
||||
runs `Disc::sweep` once, then loops `Disc::patch` until either the mapfile is
|
||||
clean or the configured retry budget is exhausted, then hands the ISO off to
|
||||
the mux pipeline. The `freemkv` CLI does the same shape with a
|
||||
terminal-output progress sink. Layer 3 runs inside any consumer of
|
||||
`DiscStream` (direct PES pipeline, ISO playback, etc.) without caller
|
||||
involvement.
|
||||
|
||||
Three primitives compose the disc-side flow:
|
||||
|
||||
| Primitive | What it does |
|
||||
|---------------------------|-----------------------------------------------------------------------|
|
||||
| `Disc::copy` | disc → ISO. Writes a sidecar `.mapfile`. Opt-in skip-forward on failure. |
|
||||
| `Disc::patch` | Re-reads bad ranges from the drive. Idempotent; call N times. |
|
||||
| `Disc::sweep` | disc → ISO, one forward pass. Writes a sidecar `.mapfile`. Opt-in skip-on-error. |
|
||||
| `Disc::patch` | Re-reads bad ranges from the drive. One pass per call; caller invokes N times. |
|
||||
| `DiscStream` (ISO source) | Reads sectors from the ISO, feeds decrypt → demux → codec → mux. |
|
||||
|
||||
## Data model
|
||||
@@ -60,54 +63,71 @@ Status characters match ddrescue:
|
||||
|
||||
Position and size are hex byte offsets into the ISO.
|
||||
|
||||
### `CopyOptions` and `PatchOptions`
|
||||
### `SweepOptions` and `PatchOptions`
|
||||
|
||||
Defaults preserve pre-`0.11.21` behavior — abort on the first unreadable
|
||||
sector. Opt in to the recovery-friendly path:
|
||||
The library no longer dispatches between sweep and patch internally — the
|
||||
caller picks the verb explicitly per pass. The two option structs are flat
|
||||
and have no overlap:
|
||||
|
||||
```rust
|
||||
CopyOptions {
|
||||
skip_on_error: true, // zero-fill bad blocks, continue
|
||||
skip_forward: true, // exponential skip-forward after a failure
|
||||
resume: true, // pick up from an existing ISO + mapfile
|
||||
decrypt: false, // keep the ISO a raw disc image
|
||||
..Default::default()
|
||||
SweepOptions {
|
||||
decrypt: true,
|
||||
resume: false,
|
||||
batch_sectors: None,
|
||||
skip_on_error: true, // damage-jump + zero-fill on read failure
|
||||
progress: Some(&reporter),
|
||||
halt: Some(flag),
|
||||
}
|
||||
|
||||
PatchOptions {
|
||||
decrypt: true,
|
||||
block_sectors: None,
|
||||
full_recovery: true,
|
||||
reverse: true, // walk bad ranges high → low LBA
|
||||
wedged_threshold: 50,
|
||||
progress: Some(&reporter),
|
||||
halt: Some(flag),
|
||||
}
|
||||
```
|
||||
|
||||
Caller-orchestrated dispatch (the policy `Disc::copy` used to embed):
|
||||
|
||||
- No mapfile → `sweep` (fresh Pass 1).
|
||||
- Mapfile with `?` ranges → `sweep` with `resume: true`.
|
||||
- Mapfile covers full disc, only `*` / `/` / `-` ranges → `patch`.
|
||||
- Mapfile clean → done; no further pass needed.
|
||||
|
||||
Each consumer (autorip, `freemkv` CLI) implements the loop in roughly five
|
||||
lines of `Mapfile::stats()` checks.
|
||||
|
||||
## Algorithm
|
||||
|
||||
### Pass 1 — fast sweep (`Disc::copy`)
|
||||
### Pass 1 — fast sweep (`Disc::sweep`)
|
||||
|
||||
1. Read 64 KB (32 sectors, one BD ECC block) at the current LBA via
|
||||
`Drive::read(.., recovery=false)` — short 1.5 s timeout, single shot.
|
||||
2. On success: mark the range `+`, advance by one block.
|
||||
3. On failure (with `skip_on_error`): zero-fill the block in the ISO, mark
|
||||
it `*`, advance.
|
||||
4. If `skip_forward` is set: after a failure, jump ahead by an exponentially
|
||||
growing amount (256 KB initial, doubling on consecutive failures, capped at
|
||||
1% of disc). The skipped bytes are also marked `*` — `patch` will visit
|
||||
them later.
|
||||
5. Reset the skip size to 256 KB on the first success after a failure.
|
||||
1. Read one ECC block (32 sectors for UHD, 16 for BD/DVD) at the current LBA.
|
||||
2. On success: write data to ISO, mark `+`, advance.
|
||||
3. On failure (with `multipass`): zero-fill, mark `*`, advance.
|
||||
4. Track a sliding window of the last 16 ECC block results. When ≥12% are failures
|
||||
→ **damage-jump**: skip ahead by `256×batch×multiplier` sectors (8 MB base for
|
||||
UHD). Double the multiplier on each jump (8→16→32→64 MB...). Zero-fill the gap as `*`.
|
||||
5. On 16 consecutive good reads: reset jump multiplier to 1, restore max read speed.
|
||||
6. Speed control: damage zone entry → minimum speed, exit → maximum speed.
|
||||
7. Only transport failures (USB bridge crash) abort the pass.
|
||||
|
||||
Pass 1 completes when every byte has terminal status (`+`, `-`, or the caller
|
||||
bails via the halt flag).
|
||||
Pass 1 completes when every byte has been visited (either `+` or `*`).
|
||||
|
||||
### Pass 2+ — patch (`Disc::patch`)
|
||||
|
||||
`Disc::patch` reads the mapfile and iterates every non-`+` range. For each:
|
||||
`Disc::patch` reads the mapfile and iterates every non-`+` range. Default: **reverse** mode
|
||||
(walks ranges from highest LBA to lowest, within each range from end to start).
|
||||
|
||||
1. Issue a drive read via `Drive::read(.., recovery=true)` — long 30 s
|
||||
timeout, still single shot. Drive firmware does its own ECC and retries
|
||||
inside that window; userspace does not pile on additional retries here.
|
||||
2. On success: write the good bytes into the ISO at the exact byte offset,
|
||||
mark `+`.
|
||||
3. On failure: mark `-`.
|
||||
4. Update the mapfile after every block — crash-safe resume.
|
||||
|
||||
Idempotent. Call `patch` N times for N retry attempts; typically the caller
|
||||
stops early if a pass recovers zero bytes (structure-protected sectors will
|
||||
never yield).
|
||||
1. Issue a single-sector read with 60 s timeout (`recovery=true`). Drive firmware
|
||||
does its own ECC recovery inside that window.
|
||||
2. On success: write the good bytes into the ISO, mark `+`.
|
||||
3. On failure with non-marginal SCSI sense: bail immediately (drive won't produce data).
|
||||
4. On failure with marginal sense: mark `-`, continue.
|
||||
5. Update the mapfile after every block — crash-safe resume.
|
||||
6. Wedged-drive exit: 50 consecutive failures with zero recovery → bail this pass.
|
||||
|
||||
### In-stream — adaptive batch halving (`DiscStream::fill_extents`)
|
||||
|
||||
@@ -133,8 +153,8 @@ implement retry logic.
|
||||
no eject cycle. The `recovery` flag controls only the per-CDB timeout
|
||||
(1.5 s vs. 30 s); on any failure it returns `Err(DiscRead)` immediately.
|
||||
Inline recovery (5× gentle retry → close + SCSI reset + reopen → 5× more)
|
||||
was removed in 0.13.6. See `(internal)/postmortems/2026-04-25-stop-wedge-and-zero-kbs.md`
|
||||
for rationale: the inline reset on the LG BU40N (Initio USB-SATA bridge)
|
||||
was removed in 0.13.6. See the stop-wedge postmortem (2026-04-25) for rationale:
|
||||
the inline reset on the LG BU40N (Initio USB-SATA bridge)
|
||||
wedged drive firmware below the bridge without ever recovering a sector,
|
||||
and the gentle-retry phase produced long stretches of 0 KB/s with no
|
||||
recoveries to show for it. Recovery responsibility is now layered: layer 1
|
||||
@@ -178,4 +198,4 @@ scrape vs. retry with direction reversal) if there's measured benefit.
|
||||
|
||||
- [ddrescue manual, Algorithm chapter](https://www.gnu.org/software/ddrescue/manual/ddrescue_manual.html)
|
||||
- [ddrescue optical media notes](https://www.electric-spoon.com/doc/gddrescue/html/Optical-media.html)
|
||||
- Source: [`src/disc/mapfile.rs`](../src/disc/mapfile.rs), [`src/disc/mod.rs`](../src/disc/mod.rs) (`Disc::copy`, `Disc::patch`), [`src/drive/mod.rs`](../src/drive/mod.rs) (`Drive::read`), [`src/mux/disc.rs`](../src/mux/disc.rs) (`DiscStream::fill_extents`).
|
||||
- Source: [`src/disc/mapfile.rs`](../src/disc/mapfile.rs), [`src/disc/sweep.rs`](../src/disc/sweep.rs) (`Disc::sweep`), [`src/disc/patch.rs`](../src/disc/patch.rs) (`Disc::patch`), [`src/drive/mod.rs`](../src/drive/mod.rs) (`Drive::read`), [`src/mux/disc.rs`](../src/mux/disc.rs) (`DiscStream::fill_extents`).
|
||||
|
||||
+1
-1
@@ -126,7 +126,7 @@ USB optical drives have ~500ms round-trip latency per SCSI command. Since `read_
|
||||
|
||||
`Disc::scan()` wraps the drive in a `BufferedSectorReader` before reading. On a single-sector read, the buffer prefetches a batch of sectors (sized from the kernel's `max_hw_sectors_kb` for the device) and caches them. Subsequent reads to nearby LBAs return from cache with zero SCSI overhead. After parsing the UDF directory structure, the entire metadata partition is pre-read into the cache, so all ICB lookups during title scanning and encryption resolution are instant.
|
||||
|
||||
The buffer is transparent -- `read_filesystem()`, `read_file()`, and all downstream code still call `read_sectors(lba, 1, buf)` as before. The batching happens inside the `SectorReader` implementation.
|
||||
The buffer is transparent -- `read_filesystem()`, `read_file()`, and all downstream code still call `read_sectors(lba, 1, buf)` as before. The batching happens inside the `SectorSource` implementation.
|
||||
|
||||
### UDF Filename Encoding
|
||||
|
||||
|
||||
+50
-1
@@ -62,7 +62,23 @@ pub fn decrypt_sectors(
|
||||
|
||||
for chunk in buf.chunks_mut(unit_len) {
|
||||
if chunk.len() == unit_len && aacs::is_unit_encrypted(chunk) {
|
||||
aacs::decrypt_unit_full(chunk, &uk, rdk);
|
||||
// `is_unit_encrypted` is a byte-0 heuristic: it fires on any
|
||||
// unit whose first byte has the top 2 bits set, which is
|
||||
// correct for m2ts source packets (where those bits are the
|
||||
// copy-control marker) but false-positives on any other binary
|
||||
// data with similarly-shaped first bytes — notably MPLS/CLPI
|
||||
// navigation files that begin with ASCII magic ('M', 'H'…)
|
||||
// and survive sweep mixed in with encrypted m2ts payloads.
|
||||
// `decrypt_unit_full` self-checks via TS-sync verification and
|
||||
// returns false on a misfire, but it has already mutated the
|
||||
// chunk by then. Snapshot and restore on verification failure
|
||||
// — same pattern `decrypt_unit_try_keys` uses for multi-key
|
||||
// discs. Real m2ts units verify and stay decrypted; nav-file
|
||||
// sectors get scrambled briefly and then put back as-was.
|
||||
let original: Vec<u8> = chunk.to_vec();
|
||||
if !aacs::decrypt_unit_full(chunk, &uk, rdk) {
|
||||
chunk.copy_from_slice(&original);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -74,3 +90,36 @@ pub fn decrypt_sectors(
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Regression for the 0.18.1 nav-file scramble bug. A non-m2ts unit whose
|
||||
/// first byte has the top 2 bits set (here: the ASCII letter 'M' that
|
||||
/// MPLS files start with, 0x4D = 0b01001101) trips `is_unit_encrypted`,
|
||||
/// gets AES-decrypted with the unit key, fails the TS-sync verification,
|
||||
/// and must be restored to its original bytes — not left scrambled.
|
||||
#[test]
|
||||
fn nav_file_unit_survives_decrypt_attempt() {
|
||||
let mut unit = vec![0u8; aacs::ALIGNED_UNIT_LEN];
|
||||
unit[0] = b'M';
|
||||
unit[1] = b'P';
|
||||
unit[2] = b'L';
|
||||
unit[3] = b'S';
|
||||
for (i, b) in unit.iter_mut().enumerate().skip(4) {
|
||||
*b = (i as u8).wrapping_mul(31);
|
||||
}
|
||||
let snapshot = unit.clone();
|
||||
|
||||
let keys = DecryptKeys::Aacs {
|
||||
unit_keys: vec![(0, [0xAB; 16])],
|
||||
read_data_key: None,
|
||||
};
|
||||
decrypt_sectors(&mut unit, &keys, 0).unwrap();
|
||||
assert_eq!(
|
||||
unit, snapshot,
|
||||
"non-m2ts unit must be restored after failed decrypt"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+149
-13
@@ -17,11 +17,22 @@
|
||||
//!
|
||||
//! Status chars: `?` non-tried · `*` non-trimmed · `/` non-scraped · `-` unreadable · `+` finished.
|
||||
//!
|
||||
//! The mapfile is flushed to disk on every `record()` call so a crashed
|
||||
//! rip loses at most one block of recorded state.
|
||||
//! The mapfile is flushed to disk at most once per `FLUSH_INTERVAL`
|
||||
//! during `record()` calls, plus on explicit `flush()` and on `Drop`.
|
||||
//! This bounds atomic-rename RPC rate on networked staging (e.g. NFS)
|
||||
//! where per-record persists otherwise serialize the rip pipeline.
|
||||
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Minimum interval between mapfile persists. `record()` updates in-memory
|
||||
/// state every call but only writes to disk when this interval has elapsed
|
||||
/// since the last persist (or when `flush()` is called explicitly, or on
|
||||
/// `Drop`). Bounds RPC rate on NFS staging where atomic-rename per record
|
||||
/// otherwise dominates throughput. On crash the worst-case progress loss
|
||||
/// is one interval's worth of records.
|
||||
const FLUSH_INTERVAL: Duration = Duration::from_millis(1000);
|
||||
|
||||
/// Status of a byte range in the mapfile. ddrescue-compatible.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -91,15 +102,34 @@ pub struct MapStats {
|
||||
/// retry" UI bucket; `bytes_pending` over-counts because it folds
|
||||
/// in `bytes_nontried`.
|
||||
pub bytes_retryable: u64,
|
||||
/// Number of unreadable ranges (for UI display). Computed from
|
||||
/// `ranges_with(&[Unreadable])`.
|
||||
pub num_bad_ranges: u32,
|
||||
/// Largest gap among unreadable ranges in milliseconds. Computed as
|
||||
/// largest range size / bytes_per_sec * 1000. Set by caller (autorip)
|
||||
/// since bytes_per_sec is application-specific.
|
||||
pub main_lost_ms: f64,
|
||||
}
|
||||
|
||||
/// Write-through mapfile. Every `record()` persists to disk immediately
|
||||
/// so a crash during rip loses at most one block.
|
||||
/// Time-batched mapfile. `record()` keeps in-memory state up-to-date on
|
||||
/// every call; persists to disk at most once per `FLUSH_INTERVAL`.
|
||||
/// Explicit `flush()` and `Drop` guarantee state is on disk after a sweep
|
||||
/// or patch finishes. On hard crash the worst-case loss is one flush
|
||||
/// interval of records — the file's payload bytes are unaffected.
|
||||
pub struct Mapfile {
|
||||
path: PathBuf,
|
||||
entries: Vec<MapEntry>,
|
||||
total_size: u64,
|
||||
version: String,
|
||||
/// Incrementally maintained stats — updated on every `record()` call
|
||||
/// so `stats()` is O(1) instead of O(n).
|
||||
stats: MapStats,
|
||||
/// True when in-memory state has changed but `write_to_disk` has not
|
||||
/// yet captured it.
|
||||
dirty: bool,
|
||||
/// Wall-clock timestamp of the last successful `write_to_disk` (or
|
||||
/// the moment the mapfile was constructed, whichever is later).
|
||||
last_flushed: Instant,
|
||||
}
|
||||
|
||||
impl Mapfile {
|
||||
@@ -107,7 +137,7 @@ impl Mapfile {
|
||||
/// Writes to disk immediately so a resume can pick up even if the caller
|
||||
/// never records anything.
|
||||
pub fn create(path: &Path, total_size: u64, version: &str) -> io::Result<Self> {
|
||||
let mf = Self {
|
||||
let mut mf = Self {
|
||||
path: path.to_path_buf(),
|
||||
entries: vec![MapEntry {
|
||||
pos: 0,
|
||||
@@ -116,8 +146,19 @@ impl Mapfile {
|
||||
}],
|
||||
total_size,
|
||||
version: version.to_string(),
|
||||
stats: MapStats {
|
||||
bytes_total: total_size,
|
||||
bytes_pending: total_size,
|
||||
bytes_nontried: total_size,
|
||||
..Default::default()
|
||||
},
|
||||
dirty: false,
|
||||
last_flushed: Instant::now(),
|
||||
};
|
||||
// Eager initial persist so a resume can pick this up even if
|
||||
// `record()` is never called.
|
||||
mf.write_to_disk()?;
|
||||
mf.last_flushed = Instant::now();
|
||||
Ok(mf)
|
||||
}
|
||||
|
||||
@@ -177,11 +218,15 @@ impl Mapfile {
|
||||
}
|
||||
entries.sort_by_key(|e| e.pos);
|
||||
let total_size = entries.last().map(|e| e.pos + e.size).unwrap_or(0);
|
||||
let stats = Self::compute_stats(&entries, total_size);
|
||||
Ok(Self {
|
||||
path: path.to_path_buf(),
|
||||
entries,
|
||||
total_size,
|
||||
version,
|
||||
stats,
|
||||
dirty: false,
|
||||
last_flushed: Instant::now(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -244,8 +289,30 @@ impl Mapfile {
|
||||
merged.push(e);
|
||||
}
|
||||
|
||||
// Recompute stats from merged entries. record() is already O(n) due to
|
||||
// drain-and-rebuild, so this is a constant-factor overhead. The critical
|
||||
// win is that stats() is now O(1) — called millions of times in the hot
|
||||
// path during sweep/patch, it just returns the cached value.
|
||||
self.stats = Self::compute_stats(&merged, self.total_size);
|
||||
self.entries = merged;
|
||||
self.write_to_disk()?;
|
||||
self.dirty = true;
|
||||
if self.last_flushed.elapsed() >= FLUSH_INTERVAL {
|
||||
self.write_to_disk()?;
|
||||
self.dirty = false;
|
||||
self.last_flushed = Instant::now();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist any pending in-memory changes to disk. No-op if clean.
|
||||
/// Callers (sweep/patch finalisation) invoke this after their last
|
||||
/// `record()` to guarantee state is durable before returning.
|
||||
pub fn flush(&mut self) -> io::Result<()> {
|
||||
if self.dirty {
|
||||
self.write_to_disk()?;
|
||||
self.dirty = false;
|
||||
self.last_flushed = Instant::now();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -283,11 +350,15 @@ impl Mapfile {
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> MapStats {
|
||||
self.stats
|
||||
}
|
||||
|
||||
fn compute_stats(entries: &[MapEntry], total_size: u64) -> MapStats {
|
||||
let mut s = MapStats {
|
||||
bytes_total: self.total_size,
|
||||
bytes_total: total_size,
|
||||
..Default::default()
|
||||
};
|
||||
for e in &self.entries {
|
||||
for e in entries {
|
||||
match e.status {
|
||||
SectorStatus::Finished => s.bytes_good += e.size,
|
||||
SectorStatus::Unreadable => s.bytes_unreadable += e.size,
|
||||
@@ -316,11 +387,7 @@ impl Mapfile {
|
||||
{
|
||||
let file = std::fs::File::create(&tmp)?;
|
||||
let mut w = std::io::BufWriter::new(file);
|
||||
writeln!(
|
||||
w,
|
||||
"# Rescue Logfile. Created by libfreemkv v{}",
|
||||
self.version
|
||||
)?;
|
||||
writeln!(w, "# Rescue Logfile. Created by {}", self.version)?;
|
||||
writeln!(w, "# Current pos / status / pass / pass_time")?;
|
||||
writeln!(w, "0x000000000 ? 1 0")?;
|
||||
writeln!(w, "# pos size status")?;
|
||||
@@ -340,6 +407,16 @@ impl Mapfile {
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Mapfile {
|
||||
/// Best-effort flush on drop so a sweep / patch that returns early
|
||||
/// (or unwinds) doesn't lose its in-memory state. Errors here are
|
||||
/// swallowed because Drop has no way to surface them; explicit
|
||||
/// `flush()` on the success path gives callers proper error handling.
|
||||
fn drop(&mut self) {
|
||||
let _ = self.flush();
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_hex(s: &str) -> io::Result<u64> {
|
||||
let s = s.strip_prefix("0x").unwrap_or(s);
|
||||
u64::from_str_radix(s, 16).map_err(|_| {
|
||||
@@ -440,6 +517,8 @@ mod tests {
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
mf.record(100, 200, SectorStatus::Finished).unwrap();
|
||||
mf.record(500, 100, SectorStatus::Unreadable).unwrap();
|
||||
// record() batches; explicit flush before reading back from disk.
|
||||
mf.flush().unwrap();
|
||||
let loaded = Mapfile::load(&p).unwrap();
|
||||
assert_eq!(loaded.entries(), mf.entries());
|
||||
let _ = std::fs::remove_file(&p);
|
||||
@@ -471,4 +550,61 @@ mod tests {
|
||||
assert_eq!(bad, vec![(100, 50), (300, 50)]);
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_consistent_after_overlapping_records() {
|
||||
let p = tmpfile("stats_consistent_after_overlapping");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
// Record some finished, some unreadable, some nontrimmed
|
||||
mf.record(0, 300, SectorStatus::Finished).unwrap();
|
||||
mf.record(300, 200, SectorStatus::NonTrimmed).unwrap();
|
||||
mf.record(500, 100, SectorStatus::Unreadable).unwrap();
|
||||
mf.record(600, 400, SectorStatus::Finished).unwrap();
|
||||
|
||||
// Final entries: [0..300 Finished, 300..500 NonTrimmed, 500..600 Unreadable, 600..1000 Finished]
|
||||
let s = mf.stats();
|
||||
assert_eq!(s.bytes_good, 700); // 300 + 400
|
||||
assert_eq!(s.bytes_unreadable, 100); // 100
|
||||
assert_eq!(s.bytes_pending, 200); // NonTrimmed only (NonTried=0)
|
||||
assert_eq!(s.bytes_nontried, 0);
|
||||
assert_eq!(s.bytes_retryable, 200); // NonTrimmed
|
||||
assert_eq!(s.bytes_total, 1000);
|
||||
|
||||
// Overwrite a NonTrimmed range with Finished
|
||||
mf.record(300, 100, SectorStatus::Finished).unwrap();
|
||||
// Entries: [0..400 Finished, 400..500 NonTrimmed, 500..600 Unreadable, 600..1000 Finished]
|
||||
let s2 = mf.stats();
|
||||
assert_eq!(s2.bytes_good, 800); // 400 + 400
|
||||
assert_eq!(s2.bytes_unreadable, 100);
|
||||
assert_eq!(s2.bytes_pending, 100); // NonTrimmed only
|
||||
assert_eq!(s2.bytes_retryable, 100);
|
||||
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_consistent_after_split_record() {
|
||||
let p = tmpfile("stats_consistent_after_split");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
// Mark middle as NonTrimmed
|
||||
mf.record(200, 400, SectorStatus::NonTrimmed).unwrap();
|
||||
// Entries: [0..200 NonTried, 200..600 NonTrimmed, 600..1000 NonTried]
|
||||
let s = mf.stats();
|
||||
assert_eq!(s.bytes_pending, 1000); // NonTried(600) + NonTrimmed(400)
|
||||
assert_eq!(s.bytes_retryable, 400); // NonTrimmed only
|
||||
assert_eq!(s.bytes_nontried, 600); // 200 + 400
|
||||
|
||||
// Overwrite the NonTrimmed with Finished (splitting the remaining NonTried)
|
||||
mf.record(200, 400, SectorStatus::Finished).unwrap();
|
||||
// Entries: [0..200 NonTried, 200..600 Finished, 600..1000 NonTried]
|
||||
let s2 = mf.stats();
|
||||
assert_eq!(s2.bytes_good, 400);
|
||||
assert_eq!(s2.bytes_pending, 600); // NonTried(200 + 400)
|
||||
assert_eq!(s2.bytes_nontried, 600);
|
||||
assert_eq!(s2.bytes_retryable, 0);
|
||||
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
}
|
||||
|
||||
+2008
-359
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,243 @@
|
||||
//! Producer / consumer split for `Disc::patch`.
|
||||
//!
|
||||
//! Background: pre-0.18 patch ran strictly serial — single-sector
|
||||
//! recovery read → seek + write recovered bytes → mapfile.record →
|
||||
//! next iteration. The drive sat idle while the previous block's
|
||||
//! recovered bytes were committed. On a damaged disc with many bad
|
||||
//! sectors that adds up: per-sector write + mapfile.record costs a
|
||||
//! handful of milliseconds each, which the drive could be using to
|
||||
//! issue the next per-sector retry.
|
||||
//!
|
||||
//! This module decouples them. A consumer thread owns the
|
||||
//! [`crate::io::WritebackFile`] (the ISO file) and the
|
||||
//! [`super::mapfile::Mapfile`]. The producer thread (`Disc::patch`)
|
||||
//! keeps the [`crate::sector::SectorReader`], the wedge / damage-window
|
||||
//! state, the per-range watchdog, decrypt — so what enters the channel
|
||||
//! is already-clean cleartext bytes (or an "Unreadable" terminal mark).
|
||||
//!
|
||||
//! Producer and consumer run concurrently; the channel uses
|
||||
//! [`crate::io::pipeline::WRITE_THROUGH_DEPTH`] (=1) so back-pressure
|
||||
//! kicks in immediately. We want the drive's per-sector retry budget
|
||||
//! to stay in lockstep with the writer — sweep's `DEFAULT_PIPELINE_DEPTH`
|
||||
//! (4) would let several sectors of recovered bytes queue up between
|
||||
//! the producer's retry decisions and the writer, and patch's recovery
|
||||
//! loop reads stats (`bytes_good`, range progress) inline to drive its
|
||||
//! skip / wedge decisions. WRITE_THROUGH_DEPTH gives "read N+1 while
|
||||
//! writing N", no further pipelining — exactly the model the producer
|
||||
//! logic was written against.
|
||||
//!
|
||||
//! Correctness invariants preserved:
|
||||
//! - Mapfile is single-writer (consumer-only). No locking on it.
|
||||
//! - All recovery state (damage window, consecutive_failures, skip
|
||||
//! escalation, range watchdog) stays on the producer thread.
|
||||
//! - `set_speed` calls happen on the producer thread (same thread that
|
||||
//! owns the `SectorReader`). No new SCSI concurrency.
|
||||
//! - Per-iteration ordering of file-write → mapfile-record is kept
|
||||
//! intact in the consumer (write before record), so the on-disk
|
||||
//! invariant "mapfile only marks Finished what the file has received"
|
||||
//! survives a crash mid-pass.
|
||||
//! - The BU40N+Initio bridge wedge concern is unchanged: only one
|
||||
//! SCSI command in flight at a time, error-path timing identical,
|
||||
//! no new retry logic. The threading primitive only overlaps the
|
||||
//! *write* with the *next read*; the per-sector single-shot read
|
||||
//! budget that the bridge wedge concern was originally about is
|
||||
//! untouched.
|
||||
//!
|
||||
//! Per-range watchdog (`MAX_RANGE_SECS` / `RANGE_BUDGET_CAP_SECS`)
|
||||
//! checks `bytes_good` for forward progress. With work in flight on
|
||||
//! the consumer, the producer would otherwise see stale values; the
|
||||
//! sink publishes a [`SharedPatchState`] snapshot after every record
|
||||
//! so the producer's stall guards observe consumer side-effects with
|
||||
//! at most one item of lag (which is fine — the watchdog uses minute-
|
||||
//! scale budgets, not single-record latency).
|
||||
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::io::pipeline::{Flow, Sink};
|
||||
|
||||
use super::mapfile::{MapStats, Mapfile, SectorStatus};
|
||||
|
||||
/// Item the producer hands to the patch consumer. One per per-sector
|
||||
/// recovery decision.
|
||||
pub(super) enum PatchItem {
|
||||
/// Sector / small batch successfully recovered (and decrypted on the
|
||||
/// producer side if `opts.decrypt` was set). Consumer seeks to
|
||||
/// `pos`, writes `buf`, records the range as `Finished`.
|
||||
Recovered { pos: u64, buf: Vec<u8> },
|
||||
|
||||
/// Producer exhausted retries on `[pos, pos+len)`. Consumer records
|
||||
/// the range as `Unreadable`. No file write — the existing zero-fill
|
||||
/// from sweep is preserved in place.
|
||||
Unreadable { pos: u64, len: u64 },
|
||||
|
||||
/// Producer hit the per-range skip limit and is leaving the
|
||||
/// remaining bytes as `NonTrimmed` for a future pass. CRITICAL:
|
||||
/// this is not the same as `Unreadable` — sectors we never tried
|
||||
/// stay hopeful. (See the comment at the skip-limit branch in
|
||||
/// `Disc::patch`: ~36% of patch-marked Unreadable sectors are
|
||||
/// actually readable on a later pass.) No file write.
|
||||
NonTrimmed { pos: u64, len: u64 },
|
||||
}
|
||||
|
||||
/// Mapfile snapshot the sink republishes after every record so the
|
||||
/// producer can drive its stall / progress logic without holding the
|
||||
/// mapfile lock for long. `bad_ranges` mirrors what
|
||||
/// `Mapfile::ranges_with(&[NonTrimmed, Unreadable, NonScraped, NonTried])`
|
||||
/// would return — same set the pre-split patch loop computed inline
|
||||
/// for the progress callback.
|
||||
pub(super) struct SharedPatchState {
|
||||
pub stats: MapStats,
|
||||
pub bad_ranges: Vec<(u64, u64)>,
|
||||
}
|
||||
|
||||
impl SharedPatchState {
|
||||
fn from_map(map: &Mapfile) -> Self {
|
||||
Self {
|
||||
stats: map.stats(),
|
||||
bad_ranges: map.ranges_with(&[
|
||||
SectorStatus::NonTrimmed,
|
||||
SectorStatus::Unreadable,
|
||||
SectorStatus::NonScraped,
|
||||
SectorStatus::NonTried,
|
||||
]),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Final summary returned by [`Sink::close`] when the consumer drains
|
||||
/// cleanly. Mirrors what the pre-split patch loop computed at the end
|
||||
/// of the function — final mapfile stats plus whether `sync_all`
|
||||
/// failed on a regular file (the only kind of fsync error patch ever
|
||||
/// surfaced; `/dev/null` and pipes always fail `sync_all`, that's not
|
||||
/// a real error).
|
||||
pub(super) struct PatchSummary {
|
||||
pub stats: MapStats,
|
||||
}
|
||||
|
||||
/// Consumer-side of the patch pipeline. Owns the ISO writeback file
|
||||
/// and the mapfile; publishes a shared snapshot after every record so
|
||||
/// the producer can read `bytes_good` for stall detection and
|
||||
/// progress reporting.
|
||||
pub(super) struct PatchSink {
|
||||
file: crate::io::WritebackFile,
|
||||
map: Mapfile,
|
||||
/// Whether the output is a regular file (so a `sync_all` failure
|
||||
/// is real). `/dev/null` etc. always fail `sync_all`; ignore those.
|
||||
is_regular: bool,
|
||||
/// Snapshot the producer reads. Updated after every successful
|
||||
/// `record()` call. `Mutex` rather than separate atomics because
|
||||
/// the producer wants stats + bad_ranges as a coherent pair.
|
||||
shared: Arc<Mutex<SharedPatchState>>,
|
||||
}
|
||||
|
||||
impl PatchSink {
|
||||
/// Open `path` as a [`crate::io::WritebackFile`] and pair it with
|
||||
/// `map` for the consumer. The producer holds onto the returned
|
||||
/// `Arc<Mutex<SharedPatchState>>` so it can poll mapfile state
|
||||
/// while the consumer is mutating it.
|
||||
pub(super) fn new(
|
||||
path: &std::path::Path,
|
||||
map: Mapfile,
|
||||
is_regular: bool,
|
||||
) -> Result<(Self, Arc<Mutex<SharedPatchState>>)> {
|
||||
let file =
|
||||
crate::io::WritebackFile::open(path).map_err(|e| Error::IoError { source: e })?;
|
||||
let shared = Arc::new(Mutex::new(SharedPatchState::from_map(&map)));
|
||||
let shared_clone = shared.clone();
|
||||
Ok((
|
||||
Self {
|
||||
file,
|
||||
map,
|
||||
is_regular,
|
||||
shared,
|
||||
},
|
||||
shared_clone,
|
||||
))
|
||||
}
|
||||
|
||||
fn republish(&self) {
|
||||
// Best-effort lock — only the producer reads, only the consumer
|
||||
// writes; contention is single-acquire so the lock is never
|
||||
// poisoned in practice. If it ever did get poisoned we'd want
|
||||
// the underlying error surfaced rather than silently swallowed,
|
||||
// so we propagate the poison panic. (Same posture as
|
||||
// `sweep_pipeline.rs` — it never recovers from a poisoned
|
||||
// mutex either.)
|
||||
let mut guard = self
|
||||
.shared
|
||||
.lock()
|
||||
.expect("PatchSink shared state mutex poisoned");
|
||||
*guard = SharedPatchState::from_map(&self.map);
|
||||
}
|
||||
}
|
||||
|
||||
impl Sink<PatchItem> for PatchSink {
|
||||
type Output = PatchSummary;
|
||||
|
||||
fn apply(&mut self, item: PatchItem) -> std::result::Result<Flow, Error> {
|
||||
match item {
|
||||
PatchItem::Recovered { pos, buf } => {
|
||||
let len = buf.len() as u64;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&buf)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, len, SectorStatus::Finished)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
PatchItem::Unreadable { pos, len } => {
|
||||
self.map
|
||||
.record(pos, len, SectorStatus::Unreadable)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
PatchItem::NonTrimmed { pos, len } => {
|
||||
self.map
|
||||
.record(pos, len, SectorStatus::NonTrimmed)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
}
|
||||
self.republish();
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(mut self) -> std::result::Result<Self::Output, Error> {
|
||||
// Drain in-flight writeback then issue a full fsync. A failure
|
||||
// here matters only on regular files — pipes / `/dev/null` etc.
|
||||
// always fail `sync_all`.
|
||||
if let Err(e) = self.file.sync_all() {
|
||||
if self.is_regular {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "patch_sync_failed",
|
||||
error = %e,
|
||||
os_error = e.raw_os_error(),
|
||||
error_kind = ?e.kind(),
|
||||
"patch: sync_all failed"
|
||||
);
|
||||
return Err(Error::IoError { source: e });
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "freemkv::disc",
|
||||
phase = "patch_sync_skipped",
|
||||
error = %e,
|
||||
"patch: sync_all failed for non-regular file; ignoring"
|
||||
);
|
||||
}
|
||||
self.map.flush().map_err(|e| Error::IoError { source: e })?;
|
||||
// Final republish so anyone reading the shared snapshot after
|
||||
// `Pipeline::finish` sees the post-flush state. (The producer
|
||||
// already has its own copy of the final `MapStats` in the
|
||||
// returned `PatchSummary`, but the snapshot is part of the
|
||||
// public-ish contract of the consumer: it stays current
|
||||
// through close.)
|
||||
self.republish();
|
||||
Ok(PatchSummary {
|
||||
stats: self.map.stats(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,561 @@
|
||||
//! Single source of truth for what to do when a sector read fails.
|
||||
//!
|
||||
//! Both Pass 1 (`Disc::sweep`) and Pass 2-N (`Disc::patch`) call into
|
||||
//! `handle_read_error` after every failed `read_sectors`. The handler
|
||||
//! classifies the error, updates the in-flight context (counters,
|
||||
//! damage window, retry budgets), and returns a `ReadAction` the caller
|
||||
//! dispatches on. Every read goes through the same gate — no path can
|
||||
//! silently skip pause/skip/jump/abort logic.
|
||||
//!
|
||||
//! Adding a new error class = add one arm in `handle_read_error`.
|
||||
//! Adding new logging on errors = one place.
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::scsi;
|
||||
|
||||
/// In-flight bookkeeping a read loop must keep across iterations. The
|
||||
/// handler reads and mutates this. Caller owns the storage.
|
||||
pub struct ReadCtx {
|
||||
/// Number of sectors per read attempt. The handler uses this to
|
||||
/// decide whether to bisect (only worthwhile when batch > 1).
|
||||
pub batch: u16,
|
||||
/// Successful reads since the last failure. Resets to 0 on failure.
|
||||
/// Used by callers to drive damage-zone exit / speed restoration.
|
||||
pub consecutive_good: u64,
|
||||
/// Failed reads since the last success. Resets to 0 on success.
|
||||
/// Drives long-pause escalation on persistent failure.
|
||||
pub consecutive_failures: u64,
|
||||
/// Failed OUTER batch reads since the last outer success — bisect
|
||||
/// inner-sector failures are NOT counted here. Drives the
|
||||
/// fast-entry damage-jump on Pass 1 (skip the disc-level grind
|
||||
/// once we're clearly in a damaged region; Pass N will recover
|
||||
/// the actual sectors). Reset on outer success.
|
||||
pub consecutive_outer_failures: u64,
|
||||
/// Sliding window of recent read outcomes (true=ok, false=fail).
|
||||
/// Capped at `damage_window_max`. Drives damage-jump decisions.
|
||||
pub damage_window: Vec<bool>,
|
||||
pub damage_window_max: usize,
|
||||
pub damage_threshold_pct: usize,
|
||||
/// Trigger a damage-jump after this many consecutive outer-batch
|
||||
/// failures, even when the damage_window isn't full yet. Pass 1
|
||||
/// uses a small value (4) so we don't spend ~40 minutes grinding
|
||||
/// to fill a 16-block window before the first jump on a damage
|
||||
/// zone we entered cleanly. Pass N uses a larger value (or
|
||||
/// disables this — see `bisect_on_marginal`) because Pass N's
|
||||
/// whole job IS to grind on the bad ranges.
|
||||
pub fast_jump_threshold: u64,
|
||||
/// Multiplier applied to damage-jump distance. Doubles each jump,
|
||||
/// resets to 1 after `damage_window_max` consecutive good reads.
|
||||
pub jump_multiplier: u64,
|
||||
/// NOT_READY retries used so far for the current LBA. Reset to 0
|
||||
/// on any non-NOT_READY response.
|
||||
pub not_ready_retries: u32,
|
||||
/// Bridge-degradation cooldowns used so far.
|
||||
pub bridge_degradation_count: u32,
|
||||
/// Whether we're currently inside a damage-jump bisect attempt.
|
||||
/// Caller sets this true when entering single-sector mode for a
|
||||
/// failed batch, so the handler doesn't recursively request another
|
||||
/// bisect on the inner-sector failures.
|
||||
pub bisecting: bool,
|
||||
/// Whether to return `Bisect` on a marginal-media batch failure.
|
||||
/// Pass 1 sweep sets this false: a failed batch becomes
|
||||
/// SkipBlock (mark the whole 32-sector ECC block NonTrimmed,
|
||||
/// advance, let Pass N recover the salvageable sectors with
|
||||
/// proper recovery semantics). Pass N sets this true: bisection
|
||||
/// is its core job, and it has the right tools (single-sector
|
||||
/// reads, 60s recovery timeout, retry budget, escalating skip).
|
||||
pub bisect_on_marginal: bool,
|
||||
}
|
||||
|
||||
impl ReadCtx {
|
||||
/// Initial context for a Pass 1 sweep. The job is "fast and
|
||||
/// accurate, get the most data in the shortest time" — Pass N
|
||||
/// is the one that grinds on the bad ranges. So bisect-on-
|
||||
/// marginal is OFF (failed batches become SkipBlock; whole 32-
|
||||
/// sector blocks marked NonTrimmed for Pass N to revisit), and
|
||||
/// the damage-jump fast-path triggers after just 4 consecutive
|
||||
/// outer-batch failures.
|
||||
pub fn for_sweep(batch: u16) -> Self {
|
||||
Self {
|
||||
batch,
|
||||
consecutive_good: 0,
|
||||
consecutive_failures: 0,
|
||||
consecutive_outer_failures: 0,
|
||||
damage_window: Vec::with_capacity(16),
|
||||
damage_window_max: 16,
|
||||
damage_threshold_pct: 12,
|
||||
fast_jump_threshold: 4,
|
||||
jump_multiplier: 1,
|
||||
not_ready_retries: 0,
|
||||
bridge_degradation_count: 0,
|
||||
bisecting: false,
|
||||
bisect_on_marginal: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Initial context for a Pass 2-N patch. Pass N's whole reason to
|
||||
/// exist is to recover sectors Pass 1 skipped — bisection on
|
||||
/// marginal media is part of the job, and the fast-jump
|
||||
/// threshold is loose so we don't bail too early on a range that
|
||||
/// has scattered good sectors mixed in.
|
||||
pub fn for_patch(batch: u16) -> Self {
|
||||
Self {
|
||||
batch,
|
||||
consecutive_good: 0,
|
||||
consecutive_failures: 0,
|
||||
consecutive_outer_failures: 0,
|
||||
damage_window: Vec::with_capacity(16),
|
||||
damage_window_max: 16,
|
||||
damage_threshold_pct: 12,
|
||||
// Pass N is allowed to grind: window-based jump only,
|
||||
// matching the historical behaviour for patch passes.
|
||||
fast_jump_threshold: u64::MAX,
|
||||
jump_multiplier: 1,
|
||||
not_ready_retries: 0,
|
||||
bridge_degradation_count: 0,
|
||||
bisecting: false,
|
||||
bisect_on_marginal: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Caller calls this after every successful read.
|
||||
pub fn on_success(&mut self) {
|
||||
self.consecutive_good += 1;
|
||||
self.consecutive_failures = 0;
|
||||
self.not_ready_retries = 0;
|
||||
// Outer-success only: a good single-sector read inside a
|
||||
// bisect doesn't mean we've left the damaged batch. Only an
|
||||
// outer-batch success resets the outer-failure counter.
|
||||
if !self.bisecting {
|
||||
self.consecutive_outer_failures = 0;
|
||||
}
|
||||
self.damage_window.push(true);
|
||||
if self.damage_window.len() > self.damage_window_max {
|
||||
self.damage_window.remove(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What the caller should do after a read failure. The caller owns the
|
||||
/// I/O side-effects (sleep, write zeros, advance pos) — the handler
|
||||
/// only decides which side-effects.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ReadAction {
|
||||
/// Pause `pause_secs` then retry the same LBA / batch. Used for
|
||||
/// transient conditions (NOT_READY, bridge degradation) that the
|
||||
/// drive may recover from on its own.
|
||||
Retry { pause_secs: u64 },
|
||||
/// Re-issue the failed batch as `batch` single-sector reads. Each
|
||||
/// inner read is itself dispatched through `handle_read_error` with
|
||||
/// `bisecting = true` so it cannot recurse.
|
||||
Bisect,
|
||||
/// Mark the failed range NonTrimmed (zero-fill, retry in Pass N+),
|
||||
/// then pause `pause_secs` before resuming the next LBA.
|
||||
SkipBlock { pause_secs: u64 },
|
||||
/// Mark the failed range NonTrimmed AND advance position by
|
||||
/// `sectors` (zero-filling the gap as NonTrimmed). Then pause
|
||||
/// `pause_secs`. Used when the damage-window threshold is crossed.
|
||||
JumpAhead { sectors: u64, pause_secs: u64 },
|
||||
/// Unrecoverable at this layer. Caller propagates `Err` up to the
|
||||
/// outer pass loop / autorip, which can attempt USB re-enumeration,
|
||||
/// drop session, etc.
|
||||
AbortPass,
|
||||
}
|
||||
|
||||
// Pause budget constants. Tuned from 2026-05-07 BU40N traces showing
|
||||
// bridge wedges 524 ms after a 5.4-second internal ECC retry. The
|
||||
// post-failure pauses give the drive — and the bridge — time to settle.
|
||||
const POST_FAILURE_PAUSE_SECS: u64 = 1;
|
||||
const CONSECUTIVE_FAIL_LONG_PAUSE_SECS: u64 = 5;
|
||||
const CONSECUTIVE_FAIL_LONG_PAUSE_THRESHOLD: u64 = 10;
|
||||
const POST_JUMP_EXTRA_PAUSE_SECS: u64 = 2;
|
||||
const NOT_READY_PAUSE_SECS: u64 = 3;
|
||||
const NOT_READY_MAX_RETRIES: u32 = 3;
|
||||
const BRIDGE_DEGRADATION_PAUSE_SECS: u64 = 15;
|
||||
const BRIDGE_DEGRADATION_MAX_RETRIES: u32 = 5;
|
||||
|
||||
/// THE single error-handling entry point. Updates `ctx`, returns the
|
||||
/// action the caller must apply.
|
||||
///
|
||||
/// New error class = add a new arm here. New logging on errors = add
|
||||
/// it once at the top. New retry policy = adjust the constants. No
|
||||
/// other read site needs to change.
|
||||
pub fn handle_read_error(err: &Error, ctx: &mut ReadCtx) -> ReadAction {
|
||||
ctx.consecutive_failures += 1;
|
||||
ctx.consecutive_good = 0;
|
||||
// Outer-failure counter — only OUTER batch failures count toward
|
||||
// the fast-jump trigger. Bisect inner failures are part of
|
||||
// recovering an already-failed batch and don't represent
|
||||
// independent damage signal.
|
||||
if !ctx.bisecting {
|
||||
ctx.consecutive_outer_failures += 1;
|
||||
}
|
||||
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "read_error",
|
||||
consecutive_failures = ctx.consecutive_failures,
|
||||
batch = ctx.batch,
|
||||
bisecting = ctx.bisecting,
|
||||
sense_key = err.scsi_sense().map(|s| s.sense_key),
|
||||
asc = err.scsi_sense().map(|s| s.asc),
|
||||
ascq = err.scsi_sense().map(|s| s.ascq),
|
||||
error = %err,
|
||||
"read failed; classifying"
|
||||
);
|
||||
|
||||
// 1. Transport failure: bridge crash / USB disconnect. The outer
|
||||
// pass loop knows how to handle this (rediscover sg path,
|
||||
// re-open drive). Inline single-sector retry here was tried in
|
||||
// pre-v0.17.0 builds and observed to make wedges worse.
|
||||
if err.is_scsi_transport_failure() {
|
||||
return ReadAction::AbortPass;
|
||||
}
|
||||
|
||||
// 2. Bridge degradation: NOT_READY with the well-known signature
|
||||
// (sense_key=2, ASC=0x04, ASCQ=0x3E). Drive's bridge is in a
|
||||
// semi-stuck state but typically recovers after a long cooldown.
|
||||
// If we've exhausted our retry budget, fall through to the
|
||||
// marginal/skip path below.
|
||||
if err.is_bridge_degradation() && ctx.bridge_degradation_count < BRIDGE_DEGRADATION_MAX_RETRIES
|
||||
{
|
||||
ctx.bridge_degradation_count += 1;
|
||||
return ReadAction::Retry {
|
||||
pause_secs: BRIDGE_DEGRADATION_PAUSE_SECS,
|
||||
};
|
||||
}
|
||||
|
||||
let sense_key = err.scsi_sense().map(|s| s.sense_key).unwrap_or(0);
|
||||
|
||||
// 3. Generic NOT_READY (other ASC codes): drive's mechanical
|
||||
// pickup may be moving. Pause and retry briefly.
|
||||
if sense_key == scsi::SENSE_KEY_NOT_READY && ctx.not_ready_retries < NOT_READY_MAX_RETRIES {
|
||||
ctx.not_ready_retries += 1;
|
||||
return ReadAction::Retry {
|
||||
pause_secs: NOT_READY_PAUSE_SECS,
|
||||
};
|
||||
}
|
||||
if sense_key != scsi::SENSE_KEY_NOT_READY {
|
||||
ctx.not_ready_retries = 0;
|
||||
}
|
||||
|
||||
// 4. Hardware / illegal request: the drive said "no, won't do it".
|
||||
// Retrying won't change the answer. Surface to the outer layer
|
||||
// so it can eject + prompt user.
|
||||
if sense_key == scsi::SENSE_KEY_HARDWARE_ERROR || sense_key == scsi::SENSE_KEY_ILLEGAL_REQUEST {
|
||||
return ReadAction::AbortPass;
|
||||
}
|
||||
|
||||
// 5. Marginal media (MEDIUM_ERROR / ABORTED_COMMAND) on a multi-
|
||||
// sector batch: the drive can often read the same sectors
|
||||
// individually. Bisect into single-sector reads (gentler on the
|
||||
// bridge too — shorter SCSI transactions). Avoid recursive
|
||||
// bisect.
|
||||
//
|
||||
// Pass 1 sweep sets `bisect_on_marginal=false` to skip this:
|
||||
// its job is "fast and accurate, get the most data in the
|
||||
// shortest time." Pass N is purpose-built to recover
|
||||
// individual sectors with proper recovery semantics, and Pass
|
||||
// 1 grinding through 32-sector bisects costs ~2.5 min per bad
|
||||
// block AND fills the damage window slower than it should.
|
||||
// Whole-block NonTrimmed → SkipBlock → advance → Pass N
|
||||
// revisits.
|
||||
let is_marginal = matches!(
|
||||
sense_key,
|
||||
scsi::SENSE_KEY_MEDIUM_ERROR | scsi::SENSE_KEY_ABORTED_COMMAND
|
||||
);
|
||||
if is_marginal && ctx.batch > 1 && !ctx.bisecting && ctx.bisect_on_marginal {
|
||||
return ReadAction::Bisect;
|
||||
}
|
||||
|
||||
// 6. Single-sector failure or unbisectable error — record in
|
||||
// damage window, decide between skip-in-place vs damage-jump.
|
||||
//
|
||||
// SKIP damage-window updates while bisecting: the window
|
||||
// represents per-batch outcomes, not per-sector. Updating it
|
||||
// inside a bisect inner loop (potentially 32+ sector failures
|
||||
// per batch) would over-weight the window and cause runaway
|
||||
// JumpAhead distance via excessive multiplier doublings.
|
||||
if !ctx.bisecting {
|
||||
ctx.damage_window.push(false);
|
||||
if ctx.damage_window.len() > ctx.damage_window_max {
|
||||
ctx.damage_window.remove(0);
|
||||
}
|
||||
}
|
||||
|
||||
let bad_count = ctx.damage_window.iter().filter(|&&b| !b).count();
|
||||
let bad_pct = if ctx.damage_window.is_empty() {
|
||||
0
|
||||
} else {
|
||||
bad_count * 100 / ctx.damage_window.len()
|
||||
};
|
||||
|
||||
// Pass 1's job is to get to the end of the disc fast. Inter-
|
||||
// block pauses help the drive cool down on Pass N (gentle
|
||||
// recovery on bad ranges) but on Pass 1 they just turn a 30 s
|
||||
// damage zone into 30 minutes of dead-air sleep. Pass N
|
||||
// (`bisect_on_marginal=true`) keeps the original cooldown logic.
|
||||
let pause_secs = if !ctx.bisect_on_marginal {
|
||||
0
|
||||
} else if ctx.consecutive_failures >= CONSECUTIVE_FAIL_LONG_PAUSE_THRESHOLD {
|
||||
CONSECUTIVE_FAIL_LONG_PAUSE_SECS
|
||||
} else {
|
||||
POST_FAILURE_PAUSE_SECS
|
||||
};
|
||||
|
||||
// 7. Damage-jump: too many failures → skip ahead by an escalating
|
||||
// gap. Multiplier capped so we can't accidentally skip the
|
||||
// entire rest of the disc (observed 2026-05-07: a saturated
|
||||
// multiplier produced a 56 GB jump). Saturating arithmetic on
|
||||
// the sector calc as defence in depth.
|
||||
//
|
||||
// Two triggers, evaluated in order:
|
||||
//
|
||||
// a. **Fast-entry** — `consecutive_outer_failures >= fast_jump_threshold`.
|
||||
// Fires on Pass 1 (threshold=4) so we don't spend ~40 min
|
||||
// grinding to fill a 16-block damage window before the
|
||||
// first jump on a damage zone we entered cleanly. Doesn't
|
||||
// fire on Pass N (threshold=u64::MAX).
|
||||
//
|
||||
// b. **Window-based** — original behaviour: 12% bad in a
|
||||
// sliding window of 16 outer reads. Pass N's only path,
|
||||
// and Pass 1's fallback if the failures are scattered
|
||||
// enough that we don't hit the consecutive threshold.
|
||||
const MAX_JUMP_MULTIPLIER: u64 = 64; // 64 × 256 × batch sectors
|
||||
let fast_trigger = !ctx.bisecting && ctx.consecutive_outer_failures >= ctx.fast_jump_threshold;
|
||||
let window_trigger =
|
||||
ctx.damage_window.len() >= ctx.damage_window_max && bad_pct >= ctx.damage_threshold_pct;
|
||||
if fast_trigger || window_trigger {
|
||||
let mult = ctx.jump_multiplier.min(MAX_JUMP_MULTIPLIER);
|
||||
let sectors = 256u64.saturating_mul(ctx.batch as u64).saturating_mul(mult);
|
||||
ctx.jump_multiplier = (ctx.jump_multiplier.saturating_mul(2)).min(MAX_JUMP_MULTIPLIER);
|
||||
// Reset the outer-failure counter so a long damaged region
|
||||
// doesn't keep firing fast-jump every read after the initial
|
||||
// jump fired. The window-based trigger handles further jumps.
|
||||
ctx.consecutive_outer_failures = 0;
|
||||
return ReadAction::JumpAhead {
|
||||
sectors,
|
||||
pause_secs: pause_secs + POST_JUMP_EXTRA_PAUSE_SECS,
|
||||
};
|
||||
}
|
||||
|
||||
// 8. Default: zero-fill the failed batch as NonTrimmed and pause
|
||||
// before the next read.
|
||||
ReadAction::SkipBlock { pause_secs }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::error::Error;
|
||||
use crate::scsi::ScsiSense;
|
||||
|
||||
fn medium_err() -> Error {
|
||||
Error::DiscRead {
|
||||
sector: 100,
|
||||
status: Some(2),
|
||||
sense: Some(ScsiSense {
|
||||
sense_key: scsi::SENSE_KEY_MEDIUM_ERROR,
|
||||
asc: 0x11,
|
||||
ascq: 0x05,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn hardware_err() -> Error {
|
||||
Error::DiscRead {
|
||||
sector: 100,
|
||||
status: Some(2),
|
||||
sense: Some(ScsiSense {
|
||||
sense_key: scsi::SENSE_KEY_HARDWARE_ERROR,
|
||||
asc: 0x44,
|
||||
ascq: 0x00,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pass_n_marginal_with_batch_gt_1_bisects() {
|
||||
let mut ctx = ReadCtx::for_patch(32);
|
||||
let action = handle_read_error(&medium_err(), &mut ctx);
|
||||
assert_eq!(action, ReadAction::Bisect);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pass_1_marginal_skips_instead_of_bisecting() {
|
||||
// Pass 1's job is "fast and accurate" — leave bisection to
|
||||
// Pass N. A failed batch becomes SkipBlock (whole 32-sector
|
||||
// block marked NonTrimmed for Pass N to revisit).
|
||||
let mut ctx = ReadCtx::for_sweep(32);
|
||||
let action = handle_read_error(&medium_err(), &mut ctx);
|
||||
match action {
|
||||
ReadAction::SkipBlock { .. } => {}
|
||||
other => panic!("expected SkipBlock for Pass 1, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn medium_error_with_batch_1_skips() {
|
||||
let mut ctx = ReadCtx::for_patch(1);
|
||||
let action = handle_read_error(&medium_err(), &mut ctx);
|
||||
match action {
|
||||
ReadAction::SkipBlock { pause_secs } => assert!(pause_secs >= 1),
|
||||
other => panic!("expected SkipBlock, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn medium_error_while_bisecting_does_not_recurse() {
|
||||
let mut ctx = ReadCtx::for_patch(32);
|
||||
ctx.bisecting = true;
|
||||
let action = handle_read_error(&medium_err(), &mut ctx);
|
||||
match action {
|
||||
ReadAction::SkipBlock { .. } => {}
|
||||
other => panic!("expected SkipBlock, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pass_1_jumps_after_4_consecutive_outer_failures() {
|
||||
// The fast-entry trigger: Pass 1 should JumpAhead after 4
|
||||
// consecutive outer-batch failures, BEFORE the 16-block
|
||||
// damage window has filled. Otherwise we spend ~40 minutes
|
||||
// of bisecting/grinding to fill the window before the first
|
||||
// jump on a damage zone we entered cleanly.
|
||||
let mut ctx = ReadCtx::for_sweep(32);
|
||||
// First three should NOT jump (still under threshold of 4).
|
||||
for _ in 0..3 {
|
||||
let a = handle_read_error(&medium_err(), &mut ctx);
|
||||
assert!(
|
||||
!matches!(a, ReadAction::JumpAhead { .. }),
|
||||
"should not jump until 4 consecutive outer failures"
|
||||
);
|
||||
}
|
||||
// Fourth should jump.
|
||||
let a = handle_read_error(&medium_err(), &mut ctx);
|
||||
assert!(
|
||||
matches!(a, ReadAction::JumpAhead { .. }),
|
||||
"expected JumpAhead at 4th consecutive outer failure, got {a:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pass_n_does_not_fast_jump() {
|
||||
// Pass N's whole reason to exist is to grind on bad ranges.
|
||||
// It should NOT bail after 4 consecutive failures the way
|
||||
// Pass 1 does — it bisects and skips with proper recovery.
|
||||
let mut ctx = ReadCtx::for_patch(32);
|
||||
for _ in 0..4 {
|
||||
let a = handle_read_error(&medium_err(), &mut ctx);
|
||||
assert!(
|
||||
!matches!(a, ReadAction::JumpAhead { .. }),
|
||||
"Pass N must not fast-jump; got {a:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outer_success_resets_consecutive_outer_failures() {
|
||||
let mut ctx = ReadCtx::for_sweep(32);
|
||||
for _ in 0..3 {
|
||||
handle_read_error(&medium_err(), &mut ctx);
|
||||
}
|
||||
assert_eq!(ctx.consecutive_outer_failures, 3);
|
||||
// An outer-success (bisecting=false) should reset the counter.
|
||||
ctx.bisecting = false;
|
||||
ctx.on_success();
|
||||
assert_eq!(ctx.consecutive_outer_failures, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bisect_inner_success_does_not_reset_outer_counter() {
|
||||
let mut ctx = ReadCtx::for_patch(32);
|
||||
for _ in 0..3 {
|
||||
handle_read_error(&medium_err(), &mut ctx);
|
||||
}
|
||||
assert_eq!(ctx.consecutive_outer_failures, 3);
|
||||
// A successful inner-sector read during bisect is not the
|
||||
// same as escaping the bad outer batch.
|
||||
ctx.bisecting = true;
|
||||
ctx.on_success();
|
||||
assert_eq!(
|
||||
ctx.consecutive_outer_failures, 3,
|
||||
"bisect inner success must not reset outer-failure counter"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hardware_error_aborts() {
|
||||
let mut ctx = ReadCtx::for_sweep(32);
|
||||
let action = handle_read_error(&hardware_err(), &mut ctx);
|
||||
assert_eq!(action, ReadAction::AbortPass);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_failure_streak_extends_pause_on_pass_n() {
|
||||
// Pass N keeps the cooldown behaviour: after many consecutive
|
||||
// failures, pauses extend to give the drive time to recover.
|
||||
// Pass 1 explicitly does NOT pause — see
|
||||
// `pass_1_does_not_pause_on_skip` below.
|
||||
let mut ctx = ReadCtx::for_patch(1);
|
||||
for _ in 0..15 {
|
||||
handle_read_error(&medium_err(), &mut ctx);
|
||||
}
|
||||
let final_action = handle_read_error(&medium_err(), &mut ctx);
|
||||
match final_action {
|
||||
ReadAction::SkipBlock { pause_secs } => {
|
||||
assert!(pause_secs >= CONSECUTIVE_FAIL_LONG_PAUSE_SECS);
|
||||
}
|
||||
ReadAction::JumpAhead { pause_secs, .. } => {
|
||||
assert!(pause_secs >= CONSECUTIVE_FAIL_LONG_PAUSE_SECS);
|
||||
}
|
||||
other => panic!("expected long-pause action, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pass_1_does_not_pause_on_skip() {
|
||||
// Pass 1 must zoom — a damage zone is Pass N's problem to
|
||||
// recover from. Sleeping between failed batches turned a 30s
|
||||
// damaged region into a 30-minute Pass-1 grind on real discs.
|
||||
let mut ctx = ReadCtx::for_sweep(32);
|
||||
let action = handle_read_error(&medium_err(), &mut ctx);
|
||||
match action {
|
||||
ReadAction::SkipBlock { pause_secs } => assert_eq!(pause_secs, 0),
|
||||
other => panic!("expected SkipBlock, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn damage_window_fills_then_jumps() {
|
||||
let mut ctx = ReadCtx::for_sweep(1);
|
||||
ctx.damage_window_max = 4;
|
||||
ctx.damage_threshold_pct = 50;
|
||||
let mut saw_jump = false;
|
||||
for _ in 0..6 {
|
||||
let a = handle_read_error(&medium_err(), &mut ctx);
|
||||
if matches!(a, ReadAction::JumpAhead { .. }) {
|
||||
saw_jump = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
saw_jump,
|
||||
"expected at least one JumpAhead in 6 failures with 50% threshold"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn on_success_resets_failure_counters_and_pushes_window() {
|
||||
let mut ctx = ReadCtx::for_sweep(32);
|
||||
for _ in 0..3 {
|
||||
handle_read_error(&medium_err(), &mut ctx);
|
||||
}
|
||||
assert!(ctx.consecutive_failures > 0);
|
||||
ctx.bisecting = false;
|
||||
ctx.on_success();
|
||||
assert_eq!(ctx.consecutive_good, 1);
|
||||
assert_eq!(ctx.consecutive_failures, 0);
|
||||
assert!(*ctx.damage_window.last().unwrap());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
//! `Disc::sweep`'s consumer-side `Sink<WorkItem>`.
|
||||
//!
|
||||
//! Background: the original sweep loop runs strictly serialised —
|
||||
//! SCSI read → decrypt → seek + write → mapfile.record → next iter.
|
||||
//! On a healthy disc the SCSI read costs ~5-12 ms per 64 KB batch and
|
||||
//! the post-read work (decrypt 1-3 ms + file write + mapfile fsync
|
||||
//! 5-15 ms) adds another batch's worth of latency. The drive idles
|
||||
//! during the post-read work; throughput tops out at the *sum* of
|
||||
//! both costs.
|
||||
//!
|
||||
//! 0.17.11 introduced a bespoke producer/consumer split (the now-
|
||||
//! removed `disc/sweep_pipeline.rs`) to overlap the two stages. 0.18
|
||||
//! collapses that split — together with the analogous splits patch
|
||||
//! and mux need — onto the generic [`crate::io::Pipeline`] +
|
||||
//! [`crate::io::Sink`] primitive. This module is the sweep-specific
|
||||
//! `Sink` impl; the producer-side state machine (read_error context,
|
||||
//! decrypt, set_speed, halt) stays in `Disc::sweep` in `disc/mod.rs`.
|
||||
//!
|
||||
//! Correctness invariants preserved (same as 0.17.11):
|
||||
//! - Mapfile is single-writer (consumer-only). No locking.
|
||||
//! - All `read_error::ReadCtx` state stays on the producer thread.
|
||||
//! - `set_speed` calls happen on the producer thread (same thread that
|
||||
//! owns the `SectorReader`). No new SCSI concurrency.
|
||||
//! - Per-iteration ordering of file-write → mapfile-record is kept
|
||||
//! intact in the consumer (write before record), so the on-disk
|
||||
//! invariant "mapfile only marks Finished what the file has
|
||||
//! received" survives a crash mid-pass.
|
||||
//! - The BU40N+Initio bridge wedge concern is unchanged: only one
|
||||
//! SCSI command in flight at a time, error-path timing identical,
|
||||
//! no new retry logic.
|
||||
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
use std::sync::mpsc::{Receiver, SyncSender, sync_channel};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::io::{Flow, Sink};
|
||||
|
||||
use super::mapfile::{MapStats, Mapfile, SectorStatus};
|
||||
|
||||
/// Reusable zero buffer for SkipFill / GapFill / BisectBad. 64 KB
|
||||
/// matches the existing zero_gap chunk size used by the pre-split
|
||||
/// sweep loop.
|
||||
const ZERO_CHUNK: usize = 65 * 1024;
|
||||
|
||||
/// Producer → Consumer messages. The consumer applies these in FIFO
|
||||
/// order; ordering of file writes and mapfile records across items is
|
||||
/// preserved.
|
||||
pub(super) enum WorkItem {
|
||||
/// Successful batch read. Producer has already decrypted `buf` if
|
||||
/// `opts.decrypt` was set. Consumer writes `buf` at `pos` and
|
||||
/// records the range as `Finished`.
|
||||
Good { pos: u64, buf: Vec<u8> },
|
||||
|
||||
/// Bisect inner-loop good single sector (already decrypted by the
|
||||
/// producer). 2048 bytes.
|
||||
BisectGood { pos: u64, buf: Box<[u8; 2048]> },
|
||||
|
||||
/// Bisect inner-loop bad single sector. Consumer writes 2048
|
||||
/// zeros at `pos` and records the sector as `NonTrimmed`.
|
||||
BisectBad { pos: u64 },
|
||||
|
||||
/// Whole-batch zero-fill (failed batch on `SkipBlock`, or the
|
||||
/// failed batch portion of `JumpAhead`). Consumer streams zeros
|
||||
/// across `[pos, pos+len)` and records the range as `NonTrimmed`.
|
||||
SkipFill { pos: u64, len: u64 },
|
||||
|
||||
/// Gap fill following a `JumpAhead`. Same effect as `SkipFill`;
|
||||
/// distinguished only so future logging / instrumentation can
|
||||
/// tell them apart without parsing a flag.
|
||||
GapFill { pos: u64, len: u64 },
|
||||
|
||||
/// Producer wants the latest mapfile stats for the progress
|
||||
/// callback. Consumer responds on `prog_tx` with a fresh
|
||||
/// [`ProgressSnapshot`]. Best-effort: if the producer hasn't
|
||||
/// drained the previous snapshot, the new one is silently
|
||||
/// dropped — the producer's local cache stays current enough.
|
||||
StatsRequest,
|
||||
}
|
||||
|
||||
/// Snapshot the consumer sends back to the producer for the progress
|
||||
/// callback.
|
||||
pub(super) struct ProgressSnapshot {
|
||||
pub stats: MapStats,
|
||||
pub bad_ranges: Vec<(u64, u64)>,
|
||||
}
|
||||
|
||||
/// Final summary returned by the consumer thread on shutdown — what
|
||||
/// `SweepSink::close` produces, surfaced to the producer via
|
||||
/// `Pipeline::finish`.
|
||||
pub(super) struct ConsumerSummary {
|
||||
pub stats: MapStats,
|
||||
}
|
||||
|
||||
/// Drain any pending progress snapshots from the consumer. Returns
|
||||
/// the most recent one, if any. The producer caches it and uses it
|
||||
/// for subsequent progress callbacks until a fresh one arrives.
|
||||
pub(super) fn try_recv_progress(rx: &Receiver<ProgressSnapshot>) -> Option<ProgressSnapshot> {
|
||||
let mut latest = None;
|
||||
while let Ok(snap) = rx.try_recv() {
|
||||
latest = Some(snap);
|
||||
}
|
||||
latest
|
||||
}
|
||||
|
||||
/// `Sink<WorkItem>` for sweep. Owns the writeback file + mapfile +
|
||||
/// progress back-channel. `apply` carries the file-write +
|
||||
/// mapfile.record per item; `close` drains the writeback pipeline,
|
||||
/// fsyncs the ISO, and flushes the mapfile.
|
||||
pub(super) struct SweepSink {
|
||||
file: crate::io::WritebackFile,
|
||||
map: Mapfile,
|
||||
/// `sync_all`-on-failure-is-an-error iff the output is a regular
|
||||
/// file. `/dev/null` and pipes always fail `sync_all`; that's not
|
||||
/// a real error.
|
||||
is_regular: bool,
|
||||
/// Back-channel for `StatsRequest` responses. The producer caches
|
||||
/// the latest snapshot and uses it for the progress callback;
|
||||
/// dropped sends on a full channel are by design.
|
||||
prog_tx: SyncSender<ProgressSnapshot>,
|
||||
/// Reusable zero buffer for SkipFill / GapFill / BisectBad. Held
|
||||
/// in the sink so each apply call doesn't reallocate.
|
||||
zero: Box<[u8; ZERO_CHUNK]>,
|
||||
}
|
||||
|
||||
impl SweepSink {
|
||||
/// Construct a new `SweepSink` plus the matching progress
|
||||
/// receiver. Channel depth on the back-channel is `1` — the
|
||||
/// producer's cache is the source of truth between snapshots.
|
||||
pub(super) fn new(
|
||||
file: crate::io::WritebackFile,
|
||||
map: Mapfile,
|
||||
is_regular: bool,
|
||||
) -> (Self, Receiver<ProgressSnapshot>) {
|
||||
let (prog_tx, prog_rx) = sync_channel::<ProgressSnapshot>(1);
|
||||
let sink = SweepSink {
|
||||
file,
|
||||
map,
|
||||
is_regular,
|
||||
prog_tx,
|
||||
zero: Box::new([0u8; ZERO_CHUNK]),
|
||||
};
|
||||
(sink, prog_rx)
|
||||
}
|
||||
}
|
||||
|
||||
impl Sink<WorkItem> for SweepSink {
|
||||
type Output = ConsumerSummary;
|
||||
|
||||
fn apply(&mut self, item: WorkItem) -> Result<Flow, Error> {
|
||||
match item {
|
||||
WorkItem::Good { pos, buf } => {
|
||||
// Decrypt is on the producer; consumer assumes plaintext.
|
||||
let len = buf.len() as u64;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&buf)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, len, SectorStatus::Finished)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::BisectGood { pos, buf } => {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&buf[..])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, 2048, SectorStatus::Finished)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::BisectBad { pos } => {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&self.zero[..2048])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, 2048, SectorStatus::NonTrimmed)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::SkipFill { pos, len } | WorkItem::GapFill { pos, len } => {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
// Subsequent writes are sequential; `WritebackFile`'s
|
||||
// seek-elision keeps them on the writeback pipeline path.
|
||||
let mut filled = 0u64;
|
||||
while filled < len {
|
||||
let chunk = (len - filled).min(self.zero.len() as u64) as usize;
|
||||
self.file
|
||||
.write_all(&self.zero[..chunk])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
filled += chunk as u64;
|
||||
}
|
||||
self.map
|
||||
.record(pos, len, SectorStatus::NonTrimmed)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::StatsRequest => {
|
||||
let stats = self.map.stats();
|
||||
let bad_ranges = self.map.ranges_with(&[
|
||||
SectorStatus::NonTrimmed,
|
||||
SectorStatus::Unreadable,
|
||||
SectorStatus::NonScraped,
|
||||
SectorStatus::NonTried,
|
||||
]);
|
||||
// Best-effort: drop on backpressure; producer's cache
|
||||
// stays current enough.
|
||||
let _ = self
|
||||
.prog_tx
|
||||
.try_send(ProgressSnapshot { stats, bad_ranges });
|
||||
}
|
||||
}
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(mut self) -> Result<Self::Output, Error> {
|
||||
// Drain the writeback pipeline + fsync the ISO, then persist
|
||||
// any pending mapfile state. Same finalisation order as the
|
||||
// pre-Pipeline consumer loop.
|
||||
if let Err(e) = self.file.sync_all() {
|
||||
if self.is_regular {
|
||||
return Err(Error::IoError { source: e });
|
||||
}
|
||||
// Non-regular outputs (/dev/null, pipes) always fail
|
||||
// sync_all; that's not a real error.
|
||||
}
|
||||
self.map.flush().map_err(|e| Error::IoError { source: e })?;
|
||||
|
||||
Ok(ConsumerSummary {
|
||||
stats: self.map.stats(),
|
||||
})
|
||||
}
|
||||
}
|
||||
+10
-11
@@ -1,26 +1,25 @@
|
||||
//! macOS drive discovery and device resolution.
|
||||
//!
|
||||
//! `find_drives` uses IOKit registry enumeration (via `scsi::list_drives`)
|
||||
//! to discover optical drives without exclusive access or unmounts. Only
|
||||
//! the returned paths are then opened for INQUIRY to build full `DriveId`.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::identity::DriveId;
|
||||
|
||||
pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
let mut drives = Vec::new();
|
||||
for i in 0..16 {
|
||||
let path = format!("/dev/disk{}", i);
|
||||
if !std::path::Path::new(&path).exists() {
|
||||
continue;
|
||||
}
|
||||
match crate::scsi::open(std::path::Path::new(&path)) {
|
||||
let discovered = crate::scsi::list_drives();
|
||||
for info in discovered {
|
||||
let path = std::path::Path::new(&info.path);
|
||||
match crate::scsi::open(path) {
|
||||
Ok(mut transport) => {
|
||||
if let Ok(id) = DriveId::from_drive(transport.as_mut()) {
|
||||
if !id.raw_inquiry.is_empty() && (id.raw_inquiry[0] & 0x1F) == 0x05 {
|
||||
drives.push((path, id));
|
||||
}
|
||||
drives.push((info.path.clone(), id));
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
// Device exists but can't be opened (likely mounted).
|
||||
// Use `diskutil unmountDisk /dev/diskN` to unmount before accessing.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+139
-1
@@ -69,6 +69,16 @@ pub struct Drive {
|
||||
halt: Arc<AtomicBool>,
|
||||
/// Event handler — fires for read errors and library-level state changes.
|
||||
event_fn: Option<Box<dyn Fn(Event) + Send>>,
|
||||
/// Linux only: raw fd for the corresponding block device (`/dev/sr*`)
|
||||
/// used as a recovery fallback when SCSI READ via `/dev/sg*` returns
|
||||
/// an error. The kernel `sr_mod` driver auto-retries failed reads
|
||||
/// (~5× per command) — historically the reason `dd if=/dev/sr0`
|
||||
/// recovers ~50% of bad sectors that single-shot `SG_IO` READ
|
||||
/// misses on the same drive. `None` when the block device couldn't
|
||||
/// be resolved or opened (no fallback in that case; SCSI read
|
||||
/// errors propagate as before).
|
||||
#[cfg(target_os = "linux")]
|
||||
block_dev_fd: Option<std::os::unix::io::RawFd>,
|
||||
}
|
||||
|
||||
impl Drive {
|
||||
@@ -87,6 +97,9 @@ impl Drive {
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let block_dev_fd = open_block_device_for_sg(device);
|
||||
|
||||
Ok(Drive {
|
||||
scsi: transport,
|
||||
driver,
|
||||
@@ -96,6 +109,8 @@ impl Drive {
|
||||
device_path: device.to_string_lossy().to_string(),
|
||||
halt: Arc::new(AtomicBool::new(false)),
|
||||
event_fn: None,
|
||||
#[cfg(target_os = "linux")]
|
||||
block_dev_fd,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -433,7 +448,7 @@ impl Drive {
|
||||
///
|
||||
/// Inline retry phases (5× gentle + reset+reopen + 5× more) were
|
||||
/// removed in 0.13.6. Per
|
||||
/// `(internal)/postmortems/2026-04-25-stop-wedge-and-zero-kbs.md`,
|
||||
/// the stop-wedge postmortem (2026-04-25),
|
||||
/// the inline reset on the LG BU40N (Initio bridge) wedged drive
|
||||
/// firmware without ever recovering a sector. The remaining recovery
|
||||
/// layers (Disc::patch multi-pass, DiscStream batch halving) do not
|
||||
@@ -444,6 +459,14 @@ impl Drive {
|
||||
} else {
|
||||
crate::scsi::READ_TIMEOUT_MS
|
||||
};
|
||||
tracing::debug!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
recovery,
|
||||
timeout_ms,
|
||||
"Drive::read enter"
|
||||
);
|
||||
let cdb = [
|
||||
crate::scsi::SCSI_READ_10,
|
||||
0x00,
|
||||
@@ -467,6 +490,62 @@ impl Drive {
|
||||
Err(Error::Halted) => Err(Error::Halted),
|
||||
Err(e) => {
|
||||
let (status, sense) = extract_scsi_context(&e);
|
||||
tracing::warn!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
inner_error = %e,
|
||||
scsi_status = status,
|
||||
"Drive::read checked_exec failed"
|
||||
);
|
||||
|
||||
// /dev/sr0 pread fallback (Linux only). The kernel
|
||||
// sr_mod driver auto-retries failed reads (~5× per
|
||||
// command). Empirically (BU40N + Dune Part 2 UHD,
|
||||
// 2026-05-08) dd via /dev/sr0 recovers ~50% of bad
|
||||
// sectors that a single-shot SG_IO READ misses.
|
||||
#[cfg(target_os = "linux")]
|
||||
if recovery {
|
||||
if let Some(fd) = self.block_dev_fd {
|
||||
let len = count as usize * 2048;
|
||||
if buf.len() >= len {
|
||||
let offset = lba as i64 * 2048;
|
||||
// Drop kernel cache for this region so we get
|
||||
// a fresh device read, not stale page-cache
|
||||
// data from a prior successful neighbour read.
|
||||
let _ = unsafe {
|
||||
libc::posix_fadvise(
|
||||
fd,
|
||||
offset,
|
||||
len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
)
|
||||
};
|
||||
let n = unsafe {
|
||||
libc::pread(fd, buf.as_mut_ptr() as *mut libc::c_void, len, offset)
|
||||
};
|
||||
if n == len as isize {
|
||||
tracing::info!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
bytes = len,
|
||||
"Drive::read recovered via /dev/sr0 pread fallback"
|
||||
);
|
||||
return Ok(len);
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
pread_ret = n as i64,
|
||||
errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0),
|
||||
"/dev/sr0 pread fallback also failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::DiscRead {
|
||||
sector: lba as u64,
|
||||
status: Some(status),
|
||||
@@ -570,6 +649,61 @@ impl Drop for Drive {
|
||||
fn drop(&mut self) {
|
||||
self.cleanup();
|
||||
// SgIoTransport::drop() runs next, calling libc::close(fd)
|
||||
#[cfg(target_os = "linux")]
|
||||
if let Some(fd) = self.block_dev_fd.take() {
|
||||
unsafe { libc::close(fd) };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a `/dev/sg*` path to the corresponding `/dev/sr*` block
|
||||
/// device by walking sysfs, then open it for read (no `O_DIRECT` —
|
||||
/// `posix_fadvise(POSIX_FADV_DONTNEED)` flushes the cache before each
|
||||
/// pread, which avoids buffer-alignment requirements while still
|
||||
/// forcing fresh device reads).
|
||||
///
|
||||
/// Returns `None` on any error (sysfs not present, no matching block
|
||||
/// device, open failed). Callers treat that as "no fallback available"
|
||||
/// and propagate the original SCSI READ error.
|
||||
#[cfg(target_os = "linux")]
|
||||
fn open_block_device_for_sg(sg_path: &Path) -> Option<std::os::unix::io::RawFd> {
|
||||
let basename = sg_path.file_name()?.to_str()?;
|
||||
if !basename.starts_with("sg") {
|
||||
return None;
|
||||
}
|
||||
let sysfs_dir = format!("/sys/class/scsi_generic/{}/device/block", basename);
|
||||
let entries = std::fs::read_dir(&sysfs_dir).ok()?;
|
||||
let block_name = entries
|
||||
.flatten()
|
||||
.find_map(|e| e.file_name().into_string().ok())?;
|
||||
let block_path = format!("/dev/{}", block_name);
|
||||
|
||||
let mut bytes = block_path.as_bytes().to_vec();
|
||||
bytes.push(0);
|
||||
let fd = unsafe {
|
||||
libc::open(
|
||||
bytes.as_ptr() as *const libc::c_char,
|
||||
libc::O_RDONLY | libc::O_CLOEXEC,
|
||||
)
|
||||
};
|
||||
if fd < 0 {
|
||||
tracing::debug!(
|
||||
target: "freemkv::drive",
|
||||
sg = basename,
|
||||
block_path,
|
||||
errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0),
|
||||
"Failed to open block device for fallback; sr0 fallback disabled"
|
||||
);
|
||||
None
|
||||
} else {
|
||||
tracing::info!(
|
||||
target: "freemkv::drive",
|
||||
sg = basename,
|
||||
block_path,
|
||||
fd,
|
||||
"Opened /dev/sr* as recovery fallback for failed SCSI reads"
|
||||
);
|
||||
Some(fd)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -583,6 +717,10 @@ impl SectorReader for Drive {
|
||||
) -> Result<usize> {
|
||||
self.read(lba, count, buf, recovery)
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
Drive::set_speed(self, kbs);
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the first optical drive on this system and open it.
|
||||
|
||||
+24
-1
@@ -507,9 +507,32 @@ impl Error {
|
||||
status: crate::scsi::SCSI_STATUS_TRANSPORT_FAILURE,
|
||||
..
|
||||
}
|
||||
) || matches!(
|
||||
self,
|
||||
Error::DiscRead {
|
||||
status: Some(crate::scsi::SCSI_STATUS_TRANSPORT_FAILURE),
|
||||
..
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
/// True if this error indicates bridge degradation — the SCSI status
|
||||
/// is neither GOOD (0x00), CHECK CONDITION (0x02), nor transport failure
|
||||
/// (0xFF). Observed on the Initio INIC-1618L USB bridge preceding a full
|
||||
/// crash: the bridge firmware returns non-standard status bytes (e.g.
|
||||
/// 0x04, 0x05) with empty sense data. The caller should cool down
|
||||
/// (10 s pause) and retry rather than hammering the bridge.
|
||||
pub fn is_bridge_degradation(&self) -> bool {
|
||||
let status = match self {
|
||||
Error::ScsiError { status, .. } => *status,
|
||||
Error::DiscRead { status, .. } => status.unwrap_or(0),
|
||||
_ => return false,
|
||||
};
|
||||
status != crate::scsi::SCSI_STATUS_GOOD
|
||||
&& status != crate::scsi::SCSI_STATUS_CHECK_CONDITION
|
||||
&& status != crate::scsi::SCSI_STATUS_TRANSPORT_FAILURE
|
||||
}
|
||||
|
||||
/// True if the underlying SCSI failure is a *marginal read* — the
|
||||
/// drive returned an error category in which smaller-granularity
|
||||
/// retries can sometimes recover the data:
|
||||
@@ -520,7 +543,7 @@ impl Error {
|
||||
/// classified as fatal; treat as recoverable
|
||||
///
|
||||
/// Returns `false` for transport failures (no sense data delivered),
|
||||
/// HARDWARE ERROR, DATA PROTECT, UNIT ATTENTION, NOT READY, ILLEGAL
|
||||
/// HARDWARE ERROR, DATA PROTECT, UNIT ATTENTION, ILLEGAL
|
||||
/// REQUEST, BLANK CHECK, kernel `IoError`, and any non-SCSI variant.
|
||||
/// Caller-agnostic predicate — describes a property of the *error*,
|
||||
/// not what one specific call site should do with it. Used by
|
||||
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
//! One-bit cooperative cancellation flag.
|
||||
//!
|
||||
//! `Halt` is a clonable token wrapping `Arc<AtomicBool>`. Pass clones into
|
||||
//! every long-running loop; the loop polls `is_cancelled()` and bails out
|
||||
//! cleanly. Calling `cancel()` from any clone flips the shared flag, and
|
||||
//! every other clone observes it on its next poll.
|
||||
//!
|
||||
//! Why: `Ordering::Relaxed` is sufficient on both load and store because
|
||||
//! this flag is purely advisory — no other memory operations piggyback on
|
||||
//! it for happens-before ordering. Callers that need to publish data
|
||||
//! across threads do so via channels or other synchronization, not via
|
||||
//! this bit.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
/// Clonable, infallible cooperative-cancellation token.
|
||||
///
|
||||
/// Clones share the same underlying flag. `cancel()` is one-way; there is
|
||||
/// no `reset()` by design — construct a fresh `Halt` for a fresh
|
||||
/// operation.
|
||||
///
|
||||
/// Construct with [`Halt::new`] (or [`Halt::default`]). The `Default`
|
||||
/// impl forwards to `new()` — both produce a fresh, uncancelled token.
|
||||
/// The pair exists because clippy's `new_without_default` lint requires
|
||||
/// `Default` whenever a public `new()` is present, even when the two
|
||||
/// would do exactly the same thing.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Halt(Arc<AtomicBool>);
|
||||
|
||||
impl Halt {
|
||||
/// Construct a fresh, uncancelled token.
|
||||
pub fn new() -> Self {
|
||||
Self(Arc::new(AtomicBool::new(false)))
|
||||
}
|
||||
|
||||
/// Wrap an existing `Arc<AtomicBool>` as a `Halt`. Useful as a
|
||||
/// bridge during the 0.18 deprecation window: callers that already
|
||||
/// hold an `Arc<AtomicBool>` (e.g. `Drive::halt_flag()`, the
|
||||
/// deprecated `DiscStream::set_halt`) can adopt the new token API
|
||||
/// without changing the underlying flag.
|
||||
///
|
||||
/// Cancelling either side flips the same bit — the wrapping `Halt`
|
||||
/// and the original `Arc` are two views over one shared flag.
|
||||
pub fn from_arc(flag: Arc<AtomicBool>) -> Self {
|
||||
Self(flag)
|
||||
}
|
||||
|
||||
/// Borrow the underlying `Arc<AtomicBool>`. Used at boundaries with
|
||||
/// pre-`Halt` APIs that still take an `Arc<AtomicBool>` directly
|
||||
/// (`CopyOptions::halt`, the deprecated `DiscStream::set_halt`).
|
||||
/// Round 3 deletes those boundaries and this accessor with them.
|
||||
pub fn as_arc(&self) -> &Arc<AtomicBool> {
|
||||
&self.0
|
||||
}
|
||||
|
||||
/// Flip the shared flag to cancelled. Idempotent.
|
||||
pub fn cancel(&self) {
|
||||
self.0.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Read the shared flag.
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.0.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Halt {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fresh_is_not_cancelled() {
|
||||
let h = Halt::new();
|
||||
assert!(!h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_flips_state() {
|
||||
let h = Halt::new();
|
||||
assert!(!h.is_cancelled());
|
||||
h.cancel();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_is_idempotent() {
|
||||
let h = Halt::new();
|
||||
h.cancel();
|
||||
h.cancel();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state() {
|
||||
let original = Halt::new();
|
||||
let cloned = original.clone();
|
||||
assert!(!original.is_cancelled());
|
||||
assert!(!cloned.is_cancelled());
|
||||
|
||||
// Cancel via the clone; the original observes it.
|
||||
cloned.cancel();
|
||||
assert!(original.is_cancelled());
|
||||
assert!(cloned.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state_reverse_direction() {
|
||||
let original = Halt::new();
|
||||
let cloned = original.clone();
|
||||
|
||||
// Cancel via the original; the clone observes it.
|
||||
original.cancel();
|
||||
assert!(cloned.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state_across_threads() {
|
||||
let h = Halt::new();
|
||||
let h2 = h.clone();
|
||||
let handle = std::thread::spawn(move || {
|
||||
h2.cancel();
|
||||
});
|
||||
handle.join().unwrap();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_arc_shares_state() {
|
||||
// The 0.18 deprecation-window bridge: a Halt built from an
|
||||
// existing Arc<AtomicBool> must be a *view* over the same bit,
|
||||
// not a fresh copy. Cancelling either side flips both.
|
||||
let arc = Arc::new(AtomicBool::new(false));
|
||||
let halt = Halt::from_arc(arc.clone());
|
||||
assert!(!halt.is_cancelled());
|
||||
assert!(!arc.load(Ordering::Relaxed));
|
||||
|
||||
// Cancel via the wrapping Halt; the original Arc observes it.
|
||||
halt.cancel();
|
||||
assert!(arc.load(Ordering::Relaxed));
|
||||
|
||||
// Conversely: flip the Arc directly; the Halt view observes it.
|
||||
let arc2 = Arc::new(AtomicBool::new(false));
|
||||
let halt2 = Halt::from_arc(arc2.clone());
|
||||
arc2.store(true, Ordering::Relaxed);
|
||||
assert!(halt2.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn as_arc_returns_backing_flag() {
|
||||
// `as_arc()` must hand back the *same* Arc, not a clone of a
|
||||
// different bit. Verified by writing through the borrowed Arc
|
||||
// and observing through the Halt.
|
||||
let halt = Halt::new();
|
||||
let arc = halt.as_arc().clone();
|
||||
assert!(!halt.is_cancelled());
|
||||
arc.store(true, Ordering::Relaxed);
|
||||
assert!(halt.is_cancelled());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
//! File I/O helpers that bound kernel cache pressure on big writes.
|
||||
//!
|
||||
//! `WritebackFile` is a drop-in wrapper around `std::fs::File` for any
|
||||
//! call site that performs large sequential writes (sweep, patch, mux,
|
||||
//! etc.). It implements `Write` and `Seek` so existing code paths can
|
||||
//! swap `File` for `WritebackFile` with no body changes. Internally it
|
||||
//! drives a `WritebackPipeline` that, on Linux, drains dirty pages
|
||||
//! continuously at 32 MB granularity to avoid the kernel's
|
||||
//! accumulate-then-burst flush behaviour. macOS and Windows use a
|
||||
//! no-op pipeline — their default cache policies have not been shown
|
||||
//! to exhibit the same pathology for this access pattern.
|
||||
//!
|
||||
//! `Pipeline` + `Sink` (0.18) is the generic producer/consumer primitive
|
||||
//! used by sweep, patch, and mux to overlap reads with writes via a
|
||||
//! bounded channel + dedicated consumer thread.
|
||||
|
||||
mod writeback;
|
||||
mod writeback_file;
|
||||
|
||||
pub mod pipeline;
|
||||
|
||||
pub(crate) use writeback_file::WritebackFile;
|
||||
|
||||
// Re-exports for the 0.18 redesign. Sweep + patch are both wired up
|
||||
// (disc/sweep.rs, disc/patch.rs); mux migrates separately in autorip.
|
||||
// `WRITE_THROUGH_DEPTH` is patch-specific and has no other in-tree
|
||||
// caller — the targeted `#[allow]` keeps the re-export visible without
|
||||
// dragging the rest of the module under `dead_code`.
|
||||
#[allow(unused_imports)]
|
||||
pub use pipeline::WRITE_THROUGH_DEPTH;
|
||||
pub use pipeline::{DEFAULT_PIPELINE_DEPTH, Flow, Pipeline, Sink};
|
||||
@@ -0,0 +1,496 @@
|
||||
//! Generic bounded producer/consumer pipeline.
|
||||
//!
|
||||
//! `Pipeline<I, R>` spawns a single consumer thread, hands it items
|
||||
//! through a bounded `mpsc::sync_channel`, and joins it on `finish()`.
|
||||
//! The consumer's behaviour is supplied by a [`Sink`] implementation:
|
||||
//! `apply` is called once per item, `close` is called once at the end.
|
||||
//!
|
||||
//! Three call sites in libfreemkv want a producer/consumer split —
|
||||
//! sweep (migrated to `disc/sweep.rs::SweepSink`), patch, and mux.
|
||||
//! 0.18 collapses all three onto this primitive; sweep is in,
|
||||
//! patch and mux migrate in later 0.18 slices.
|
||||
//! See `(internal)/memory/0_18_redesign.md` for the full picture.
|
||||
//!
|
||||
//! ## Cancellation and error semantics
|
||||
//!
|
||||
//! - Producer dropping the channel (via `Pipeline::finish` dropping
|
||||
//! `tx`) signals end-of-stream; consumer flushes via `close()` and
|
||||
//! returns its `Output`.
|
||||
//! - Consumer returning [`Flow::Stop`] also calls `close()` and
|
||||
//! returns its `Output`. `send()` from the producer will then either
|
||||
//! succeed (if the item already fit in the channel buffer) or fail
|
||||
//! with `Err(item)` once the consumer has dropped its receiver.
|
||||
//! - Consumer returning `Err` from `apply` skips `close()` entirely;
|
||||
//! the consumer keeps draining the channel so the producer never
|
||||
//! blocks on a dead receiver, and the first error is propagated as
|
||||
//! the `JoinHandle` result.
|
||||
//! - Consumer panic is converted into
|
||||
//! `Error::IoError { source: io::Error::other(...) }`.
|
||||
//!
|
||||
//! ## Dead-code suppression
|
||||
//!
|
||||
//! Both `Disc::sweep` (`DEFAULT_PIPELINE_DEPTH`, `spawn_named`,
|
||||
//! `try_send`) and `Disc::patch` (`WRITE_THROUGH_DEPTH`, `Flow::Stop`)
|
||||
//! are now in-tree callers (0.18 round 2). The targeted `#[allow]`
|
||||
//! markers below cover the corners of the API not exercised by either
|
||||
//! call site (e.g. `Pipeline::spawn` without a name; `Flow::Stop`
|
||||
//! return path from sweep's sink — sweep always consumes the producer's
|
||||
//! full work-list). Drop the allows once mux migrates.
|
||||
|
||||
use std::io;
|
||||
use std::sync::mpsc::{SyncSender, sync_channel};
|
||||
use std::thread::{self, JoinHandle};
|
||||
|
||||
use crate::error::Error;
|
||||
|
||||
/// Default channel depth for callers without a specific reason to
|
||||
/// pick another value.
|
||||
///
|
||||
/// Empirically tuned for sweep and mux — both want enough slack that
|
||||
/// short consumer stalls don't immediately back up onto the producer,
|
||||
/// but not so much that a producer outpacing the consumer accumulates
|
||||
/// arbitrary buffered work. `4` matches the depth sweep has used
|
||||
/// since 0.17.11 (originally in `disc/sweep_pipeline.rs`, now in
|
||||
/// `disc/sweep.rs::SweepSink`). Patch should usually use
|
||||
/// [`WRITE_THROUGH_DEPTH`] (`1`) instead — write-through gives clean
|
||||
/// back-pressure between every read attempt and the matching write,
|
||||
/// which matters when the consumer is updating the mapfile in lockstep.
|
||||
pub const DEFAULT_PIPELINE_DEPTH: usize = 4;
|
||||
|
||||
/// Channel depth for write-through pipelines. Each `send` fully
|
||||
/// drains before the next can enqueue. Use this when the producer
|
||||
/// must observe consumer side-effects (e.g. mapfile state) before
|
||||
/// emitting the next item.
|
||||
#[allow(dead_code)]
|
||||
pub const WRITE_THROUGH_DEPTH: usize = 1;
|
||||
|
||||
/// Outcome of [`Sink::apply`]: either keep feeding items
|
||||
/// ([`Flow::Continue`]), or stop the pipeline early and run `close()`
|
||||
/// ([`Flow::Stop`]).
|
||||
///
|
||||
/// `Stop` has no in-tree caller in this slice — sweep never returns
|
||||
/// it (it always processes the producer's full work-list before the
|
||||
/// channel is dropped). Patch and mux are the intended consumers and
|
||||
/// migrate in later 0.18 slices. The variant ships now so the contract
|
||||
/// is fixed; the targeted `#[allow]` is removed when patch lands.
|
||||
pub enum Flow {
|
||||
Continue,
|
||||
#[allow(dead_code)]
|
||||
Stop,
|
||||
}
|
||||
|
||||
/// Consumer-side of a [`Pipeline`]. The pipeline owns one of these on
|
||||
/// its consumer thread and calls `apply` once per received item, then
|
||||
/// `close` once at end-of-stream.
|
||||
pub trait Sink<I>: Send + 'static {
|
||||
/// Type returned from `close()` and surfaced via
|
||||
/// [`Pipeline::finish`].
|
||||
type Output: Send + 'static;
|
||||
|
||||
/// Apply one item. Returning [`Flow::Continue`] keeps the
|
||||
/// pipeline running; [`Flow::Stop`] ends it cleanly (still calls
|
||||
/// `close()`). An error short-circuits: `close()` is *not* called
|
||||
/// and the error is what `finish()` will return, but the consumer
|
||||
/// keeps draining the channel so the producer never blocks on a
|
||||
/// dead receiver.
|
||||
fn apply(&mut self, item: I) -> Result<Flow, Error>;
|
||||
|
||||
/// Called once at end-of-stream — either because the producer
|
||||
/// dropped `tx` or because `apply` returned [`Flow::Stop`]. Use
|
||||
/// this to flush, fsync, finalise. Skipped if any prior `apply`
|
||||
/// returned `Err`.
|
||||
fn close(self) -> Result<Self::Output, Error>;
|
||||
}
|
||||
|
||||
/// Bounded producer/consumer pipeline. Holds the producer-side
|
||||
/// channel and the consumer thread's join handle.
|
||||
pub struct Pipeline<I: Send + 'static, R: Send + 'static> {
|
||||
tx: SyncSender<I>,
|
||||
handle: JoinHandle<Result<R, Error>>,
|
||||
}
|
||||
|
||||
impl<I: Send + 'static, R: Send + 'static> Pipeline<I, R> {
|
||||
/// Spawn the consumer thread with the given channel depth and
|
||||
/// [`Sink`].
|
||||
///
|
||||
/// The thread is named `freemkv-pipeline-consumer` so it shows up
|
||||
/// distinctly in stack traces and `top -H`. Callers that want a
|
||||
/// more specific name (e.g. `freemkv-sweep-consumer`) should use
|
||||
/// [`Pipeline::spawn_named`] instead. Returns an `Error::IoError`
|
||||
/// if the OS refuses the thread spawn (resource exhaustion);
|
||||
/// callers already operate in fallible context, so this is
|
||||
/// propagated rather than panicked.
|
||||
///
|
||||
/// Sweep uses [`Pipeline::spawn_named`] directly so the consumer
|
||||
/// thread shows up as `freemkv-sweep-consumer`; this function has
|
||||
/// no in-tree caller yet. Patch and mux migrate in later 0.18
|
||||
/// slices. The targeted `#[allow]` is removed when one of them
|
||||
/// lands on the default name.
|
||||
#[allow(dead_code)]
|
||||
pub fn spawn<S: Sink<I, Output = R>>(depth: usize, sink: S) -> Result<Self, Error> {
|
||||
Self::spawn_named("freemkv-pipeline-consumer", depth, sink)
|
||||
}
|
||||
|
||||
/// Like [`Pipeline::spawn`] but lets the caller supply the
|
||||
/// consumer thread's name. Useful when several pipelines run in
|
||||
/// the same process and stack traces / `top -H` need to tell them
|
||||
/// apart (e.g. `freemkv-sweep-consumer`, `freemkv-mux-consumer`).
|
||||
pub fn spawn_named<S: Sink<I, Output = R>>(
|
||||
name: &str,
|
||||
depth: usize,
|
||||
sink: S,
|
||||
) -> Result<Self, Error> {
|
||||
let (tx, rx) = sync_channel::<I>(depth);
|
||||
let handle = thread::Builder::new()
|
||||
.name(name.into())
|
||||
.spawn(move || -> Result<R, Error> {
|
||||
let mut sink = sink;
|
||||
let mut first_err: Option<Error> = None;
|
||||
let mut stopped = false;
|
||||
|
||||
while let Ok(item) = rx.recv() {
|
||||
if first_err.is_some() || stopped {
|
||||
// Drain remaining items so the producer never
|
||||
// blocks on a dead receiver. `apply` is not
|
||||
// called once we've decided to stop.
|
||||
continue;
|
||||
}
|
||||
match sink.apply(item) {
|
||||
Ok(Flow::Continue) => {}
|
||||
Ok(Flow::Stop) => {
|
||||
stopped = true;
|
||||
}
|
||||
Err(e) => {
|
||||
first_err = Some(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match first_err {
|
||||
Some(e) => Err(e),
|
||||
None => sink.close(),
|
||||
}
|
||||
})
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
|
||||
Ok(Pipeline { tx, handle })
|
||||
}
|
||||
|
||||
/// Push one item. Blocks if the channel is full — that's the
|
||||
/// back-pressure the whole primitive exists to provide. Returns
|
||||
/// the item back if the consumer thread is gone (panicked or
|
||||
/// already returned).
|
||||
///
|
||||
/// After the consumer returns [`Flow::Stop`], `send` will silently
|
||||
/// buffer items into the channel until the channel fills, then
|
||||
/// return `Err(item)` once the consumer has dropped its receiver.
|
||||
/// Producers that need to stop pushing on `Stop` should track an
|
||||
/// independent signal (e.g. `Halt`) — `send` alone is not the
|
||||
/// notification edge.
|
||||
pub fn send(&self, item: I) -> Result<(), I> {
|
||||
self.tx.send(item).map_err(|e| e.0)
|
||||
}
|
||||
|
||||
/// Non-blocking variant of [`Pipeline::send`]. If the channel is
|
||||
/// full or the consumer has hung up, the item is returned in
|
||||
/// `Err`. Useful for best-effort signalling (e.g. sweep's
|
||||
/// throttled `StatsRequest`) where dropping the message is
|
||||
/// preferable to blocking the producer.
|
||||
pub fn try_send(&self, item: I) -> Result<(), std::sync::mpsc::TrySendError<I>> {
|
||||
self.tx.try_send(item)
|
||||
}
|
||||
|
||||
/// Drop the producer-side channel and wait for the consumer
|
||||
/// thread to finish. Returns whatever the consumer's `close()`
|
||||
/// produced, or the first `apply` error, or — on consumer panic —
|
||||
/// an `Error::IoError` whose source is `io::Error::other(...)`
|
||||
/// with a "pipeline consumer panicked: <payload>" message
|
||||
/// (callers can match on the constant prefix).
|
||||
pub fn finish(self) -> Result<R, Error> {
|
||||
let Pipeline { tx, handle } = self;
|
||||
// Explicit drop, although the destructure already drops `tx`
|
||||
// at end-of-scope. Being explicit keeps the intent obvious.
|
||||
drop(tx);
|
||||
match handle.join() {
|
||||
Ok(result) => result,
|
||||
Err(payload) => {
|
||||
// Preserve the original panic message when the
|
||||
// consumer's panic payload was a `&str` or `String`
|
||||
// (the two stdlib formats that `panic!` produces).
|
||||
// Anything else falls back to "(no message)".
|
||||
let msg = payload
|
||||
.downcast_ref::<&'static str>()
|
||||
.copied()
|
||||
.or_else(|| payload.downcast_ref::<String>().map(|s| s.as_str()))
|
||||
.unwrap_or("(no message)");
|
||||
Err(Error::IoError {
|
||||
source: io::Error::other(format!("pipeline consumer panicked: {msg}")),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Sums u64s; returns the total from `close`.
|
||||
struct SumSink {
|
||||
total: u64,
|
||||
}
|
||||
|
||||
impl Sink<u64> for SumSink {
|
||||
type Output = u64;
|
||||
|
||||
fn apply(&mut self, item: u64) -> Result<Flow, Error> {
|
||||
self.total += item;
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(self) -> Result<u64, Error> {
|
||||
Ok(self.total)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn happy_path_sums_items() {
|
||||
let pipe = Pipeline::spawn(DEFAULT_PIPELINE_DEPTH, SumSink { total: 0 })
|
||||
.expect("spawn should succeed");
|
||||
let mut expected = 0u64;
|
||||
for i in 0..100u64 {
|
||||
expected += i;
|
||||
pipe.send(i).expect("send should succeed");
|
||||
}
|
||||
let total = pipe.finish().expect("finish should succeed");
|
||||
assert_eq!(total, expected);
|
||||
assert_eq!(total, (0..100u64).sum::<u64>());
|
||||
}
|
||||
|
||||
/// Sleeps `delay` per apply; counts how many it received.
|
||||
struct SlowSink {
|
||||
delay: Duration,
|
||||
count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Sink<()> for SlowSink {
|
||||
type Output = usize;
|
||||
|
||||
fn apply(&mut self, _item: ()) -> Result<Flow, Error> {
|
||||
std::thread::sleep(self.delay);
|
||||
self.count.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(self) -> Result<usize, Error> {
|
||||
Ok(self.count.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn back_pressure_blocks_sender() {
|
||||
// depth=2 + 5 sends + 50ms/apply: with the consumer pinned at
|
||||
// 50 ms per item, the producer can buffer 2 (channel cap) +
|
||||
// 1 (consumer in flight) = 3 items before sends 4 and 5 must
|
||||
// block on consumer progress. Wall-clock floor across all 5
|
||||
// sends is therefore ~2 * 50ms = 100ms (sends 4 and 5 each
|
||||
// wait roughly one apply-cycle). Use 80 ms as the assertion
|
||||
// floor to stay above the 50ms-per-item progress floor while
|
||||
// tolerating CI jitter — it still proves blocking is real.
|
||||
let count = Arc::new(AtomicUsize::new(0));
|
||||
let sink = SlowSink {
|
||||
delay: Duration::from_millis(50),
|
||||
count: count.clone(),
|
||||
};
|
||||
let pipe = Pipeline::spawn(2, sink).expect("spawn should succeed");
|
||||
|
||||
let start = Instant::now();
|
||||
for _ in 0..5 {
|
||||
pipe.send(()).expect("send should succeed");
|
||||
}
|
||||
let elapsed_send = start.elapsed();
|
||||
|
||||
let total = pipe.finish().expect("finish should succeed");
|
||||
assert_eq!(total, 5);
|
||||
assert!(
|
||||
elapsed_send >= Duration::from_millis(80),
|
||||
"back-pressure not observed: 5 sends with depth=2 and 50ms/apply \
|
||||
took {elapsed_send:?}, expected ≥ ~100ms (one or more sends \
|
||||
should have blocked behind the consumer)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns `Err` on the Nth apply (1-indexed). Tracks all calls.
|
||||
struct FailOnNthSink {
|
||||
n: usize,
|
||||
seen: Arc<AtomicUsize>,
|
||||
close_called: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Sink<u64> for FailOnNthSink {
|
||||
type Output = ();
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
let i = self.seen.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if i == self.n {
|
||||
Err(Error::DecryptFailed)
|
||||
} else {
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
fn close(self) -> Result<(), Error> {
|
||||
self.close_called.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_error_drains_then_propagates() {
|
||||
let seen = Arc::new(AtomicUsize::new(0));
|
||||
let close_called = Arc::new(AtomicUsize::new(0));
|
||||
let pipe = Pipeline::spawn(
|
||||
DEFAULT_PIPELINE_DEPTH,
|
||||
FailOnNthSink {
|
||||
n: 3,
|
||||
seen: seen.clone(),
|
||||
close_called: close_called.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
|
||||
// Send 10 items. Subsequent sends after the 3rd error must
|
||||
// still succeed (the consumer is draining).
|
||||
for i in 0..10u64 {
|
||||
pipe.send(i).expect("send should succeed even after error");
|
||||
}
|
||||
|
||||
let res = pipe.finish();
|
||||
assert!(matches!(res, Err(Error::DecryptFailed)));
|
||||
assert_eq!(
|
||||
close_called.load(Ordering::SeqCst),
|
||||
0,
|
||||
"close() must not be called when apply returned Err"
|
||||
);
|
||||
// The consumer kept calling `recv` to drain after the error;
|
||||
// it just stopped invoking `apply`. So `seen` is exactly 3
|
||||
// (apply was called for items 1, 2, 3).
|
||||
assert_eq!(seen.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
/// Returns `Flow::Stop` on the Nth apply.
|
||||
struct StopOnNthSink {
|
||||
n: usize,
|
||||
seen: Arc<AtomicUsize>,
|
||||
close_called: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Sink<u64> for StopOnNthSink {
|
||||
type Output = usize;
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
let i = self.seen.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if i >= self.n {
|
||||
Ok(Flow::Stop)
|
||||
} else {
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
fn close(self) -> Result<usize, Error> {
|
||||
self.close_called.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(self.seen.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_stop_calls_close_and_returns_output() {
|
||||
let seen = Arc::new(AtomicUsize::new(0));
|
||||
let close_called = Arc::new(AtomicUsize::new(0));
|
||||
let pipe = Pipeline::spawn(
|
||||
DEFAULT_PIPELINE_DEPTH,
|
||||
StopOnNthSink {
|
||||
n: 3,
|
||||
seen: seen.clone(),
|
||||
close_called: close_called.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
|
||||
// Send 10 items. After Stop, subsequent sends may either
|
||||
// succeed (already buffered) or fail with Err(I) (channel
|
||||
// closed). Both are valid — we don't assert on the send
|
||||
// results.
|
||||
for i in 0..10u64 {
|
||||
let _ = pipe.send(i);
|
||||
}
|
||||
|
||||
let out = pipe.finish().expect("finish should succeed after Stop");
|
||||
assert_eq!(close_called.load(Ordering::SeqCst), 1);
|
||||
// At least 3 items processed (the one that returned Stop).
|
||||
assert!(
|
||||
out >= 3,
|
||||
"expected ≥ 3 applies before Stop took effect, got {out}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Panics on the first apply call.
|
||||
struct PanickingSink;
|
||||
|
||||
impl Sink<u64> for PanickingSink {
|
||||
type Output = ();
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
panic!("synthetic test panic");
|
||||
}
|
||||
|
||||
fn close(self) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consumer_panic_becomes_io_error() {
|
||||
// Silence the panic message that would otherwise pollute the
|
||||
// test output — we expect this panic.
|
||||
let prev = std::panic::take_hook();
|
||||
std::panic::set_hook(Box::new(|_| {}));
|
||||
|
||||
let pipe =
|
||||
Pipeline::spawn(DEFAULT_PIPELINE_DEPTH, PanickingSink).expect("spawn should succeed");
|
||||
// First send may succeed (item buffered before panic) or fail
|
||||
// (channel closed after panic) — either is fine.
|
||||
let _ = pipe.send(1);
|
||||
// Drain a few more sends; once the channel is closed they'll
|
||||
// return Err(I), which we just discard.
|
||||
for i in 0..5u64 {
|
||||
let _ = pipe.send(i);
|
||||
}
|
||||
let res = pipe.finish();
|
||||
|
||||
std::panic::set_hook(prev);
|
||||
|
||||
match res {
|
||||
Err(Error::IoError { source }) => {
|
||||
let msg = source.to_string();
|
||||
// Constant prefix lets callers match without parsing
|
||||
// the variable payload tail.
|
||||
assert!(
|
||||
msg.contains("pipeline consumer panicked"),
|
||||
"expected constant panic prefix, got: {msg}"
|
||||
);
|
||||
// The original `panic!` payload (a `&'static str`) must
|
||||
// be preserved — without the downcast the message
|
||||
// would just be the prefix.
|
||||
assert!(
|
||||
msg.contains("synthetic test panic"),
|
||||
"expected original panic payload, got: {msg}"
|
||||
);
|
||||
}
|
||||
other => panic!("expected Err(IoError), got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
//! Per-platform writeback pipeline. On Linux, drains dirty pages
|
||||
//! continuously at chunk granularity to keep the kernel's writeback
|
||||
//! queue bounded. On macOS and Windows, a no-op stub.
|
||||
//!
|
||||
//! The platform decision lives entirely in this file (the cfg-gated
|
||||
//! `pub use` below). Callers — and `DiskWriter` itself — are
|
||||
//! platform-independent.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
mod noop;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) use linux::WritebackPipeline;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub(super) use noop::WritebackPipeline;
|
||||
@@ -0,0 +1,104 @@
|
||||
//! Linux writeback pipeline using `sync_file_range` + `posix_fadvise`.
|
||||
//!
|
||||
//! Pathology this fixes: the kernel's default `vm.dirty_ratio` (~20 %
|
||||
//! of RAM) lets dirty pages accumulate to hundreds of MB during a
|
||||
//! big sequential write, then bursts a flush at 99 % disk utilisation.
|
||||
//! While the burst runs, app writes block on the writeback queue —
|
||||
//! observed empirically as instantaneous speed dropping from ~15 MB/s
|
||||
//! to ~1 MB/s every ~30 s during a Pass 1 sweep.
|
||||
//!
|
||||
//! Strategy: every `chunk_bytes` of new sequential output, kick async
|
||||
//! writeback (`SYNC_FILE_RANGE_WRITE`) on the just-completed chunk and
|
||||
//! finalise the *previous* chunk via `WAIT_AFTER` + `posix_fadvise
|
||||
//! (DONTNEED)`. By the time we finalise, that previous chunk has had
|
||||
//! a full chunk's worth of work to flush — the wait is near-instant.
|
||||
//! Dirty cache stays bounded at ~2 × `chunk_bytes` and writes drain
|
||||
//! continuously instead of in bursts.
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::{AsRawFd, RawFd};
|
||||
|
||||
pub(crate) struct WritebackPipeline {
|
||||
/// Aliases the wrapping `WritebackFile::file`. Only valid for the
|
||||
/// lifetime of that struct — moving the `File` independently
|
||||
/// would silently UAF this fd. The pipeline is a private field of
|
||||
/// `WritebackFile` and never exposed outside that wrapper, which
|
||||
/// is what keeps the alias sound.
|
||||
fd: RawFd,
|
||||
chunk_bytes: u64,
|
||||
last_flush_pos: u64,
|
||||
pending: Option<(u64, u64)>,
|
||||
}
|
||||
|
||||
impl WritebackPipeline {
|
||||
/// Construct a pipeline aliasing `file`'s file descriptor. The
|
||||
/// returned `WritebackPipeline` MUST be dropped before `file`
|
||||
/// itself, or kept inside the same struct that owns `file` — the
|
||||
/// alias is unchecked.
|
||||
pub(crate) fn new(file: &File, start_pos: u64, chunk_bytes: u64) -> Self {
|
||||
Self {
|
||||
fd: file.as_raw_fd(),
|
||||
chunk_bytes,
|
||||
last_flush_pos: start_pos,
|
||||
pending: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Caller advanced the file position to `pos`. If a chunk boundary
|
||||
/// was crossed, kick async writeback for the just-completed chunk
|
||||
/// and finalise the previous one.
|
||||
pub(crate) fn note_progress(&mut self, pos: u64) {
|
||||
if pos < self.last_flush_pos.saturating_add(self.chunk_bytes) {
|
||||
return;
|
||||
}
|
||||
unsafe {
|
||||
let chunk_off = self.last_flush_pos as i64;
|
||||
let chunk_len = (pos - self.last_flush_pos) as i64;
|
||||
libc::sync_file_range(self.fd, chunk_off, chunk_len, libc::SYNC_FILE_RANGE_WRITE);
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
libc::sync_file_range(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::SYNC_FILE_RANGE_WAIT_AFTER,
|
||||
);
|
||||
libc::posix_fadvise(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
}
|
||||
self.pending = Some((chunk_off as u64, chunk_len as u64));
|
||||
}
|
||||
self.last_flush_pos = pos;
|
||||
}
|
||||
|
||||
/// Caller is about to seek away from the current write region.
|
||||
/// Drain any in-flight chunk and reset tracking.
|
||||
pub(crate) fn handle_seek(&mut self, new_pos: u64) {
|
||||
self.finalize();
|
||||
self.last_flush_pos = new_pos;
|
||||
}
|
||||
|
||||
/// Drain any in-flight chunk. Idempotent. Call before `sync_all()`
|
||||
/// or when discarding the pipeline.
|
||||
pub(crate) fn finalize(&mut self) {
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
unsafe {
|
||||
libc::sync_file_range(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::SYNC_FILE_RANGE_WAIT_AFTER,
|
||||
);
|
||||
libc::posix_fadvise(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//! No-op writeback pipeline for non-Linux targets. macOS and Windows
|
||||
//! page cache policies have not been shown to exhibit the Linux
|
||||
//! accumulate-then-burst flush pathology for our access pattern.
|
||||
//! If that changes, replace this stub with a real implementation
|
||||
//! (e.g. `F_NOCACHE` on macOS, `FILE_FLAG_WRITE_THROUGH` on Windows).
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(crate) struct WritebackPipeline;
|
||||
|
||||
impl WritebackPipeline {
|
||||
pub(crate) fn new(_file: &File, _start_pos: u64, _chunk_bytes: u64) -> Self {
|
||||
Self
|
||||
}
|
||||
pub(crate) fn note_progress(&mut self, _pos: u64) {}
|
||||
pub(crate) fn handle_seek(&mut self, _new_pos: u64) {}
|
||||
pub(crate) fn finalize(&mut self) {}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
//! `WritebackFile` — a `File` wrapper whose reason for existing is the
|
||||
//! bounded-cache writeback pipeline.
|
||||
//!
|
||||
//! Why: large sequential writes (sweep, patch, mux on UHD-scale output)
|
||||
//! left to the kernel's default writeback policy accumulate hundreds of
|
||||
//! megabytes of dirty pages and then burst-flush, stalling subsequent
|
||||
//! writes for seconds at a time. `WritebackFile` drives a continuous
|
||||
//! [`super::writeback::WritebackPipeline`] that on Linux issues
|
||||
//! incremental `sync_file_range` + `posix_fadvise(DONTNEED)` calls at
|
||||
//! 32 MB granularity so dirty pages drain at the same rate they're
|
||||
//! produced. macOS and Windows fall through to a no-op pipeline — their
|
||||
//! default cache policies have not been shown to exhibit the same
|
||||
//! pathology for this access pattern.
|
||||
//!
|
||||
//! It implements `Write` and `Seek` so any call site that wrote to a
|
||||
//! plain `File` through those traits (sweep, patch, mux) can swap in
|
||||
//! `WritebackFile` without touching the body of the loop. The wrapper
|
||||
//! also tracks the current file position to feed the pipeline with
|
||||
//! progress + seek boundaries.
|
||||
//!
|
||||
//! See `super::writeback::linux` for the underlying pathology and the
|
||||
//! strategy.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use super::writeback::WritebackPipeline;
|
||||
|
||||
const CHUNK_BYTES: u64 = 32 * 1024 * 1024;
|
||||
|
||||
pub(crate) struct WritebackFile {
|
||||
file: File,
|
||||
pipeline: WritebackPipeline,
|
||||
pos: u64,
|
||||
}
|
||||
|
||||
impl WritebackFile {
|
||||
/// Wrap an open `File`. The current OS file position is queried
|
||||
/// once so the pipeline starts tracking from wherever the file
|
||||
/// already is (typically 0 for fresh files; non-zero for resumed
|
||||
/// or appended files).
|
||||
pub(crate) fn new(mut file: File) -> io::Result<Self> {
|
||||
let pos = file.stream_position()?;
|
||||
let pipeline = WritebackPipeline::new(&file, pos, CHUNK_BYTES);
|
||||
Ok(Self {
|
||||
file,
|
||||
pipeline,
|
||||
pos,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new file at `path` (truncating any existing contents)
|
||||
/// and wrap it. Convenience for the common
|
||||
/// `File::create(path)` + `WritebackFile::new(file)` pair so callers
|
||||
/// don't have to assemble a `File` first.
|
||||
pub(crate) fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = File::create(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Open an existing file at `path` for writing (no truncation) and
|
||||
/// wrap it. Mirrors `File::open` semantics for the writable case
|
||||
/// — used by patch / resume paths that mutate an existing ISO in
|
||||
/// place.
|
||||
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new().write(true).open(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Drain in-flight writeback then issue a full fsync. Use this in
|
||||
/// place of `File::sync_all`.
|
||||
pub(crate) fn sync_all(&mut self) -> io::Result<()> {
|
||||
self.pipeline.finalize();
|
||||
self.file.sync_all()
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for WritebackFile {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let n = self.file.write(buf)?;
|
||||
self.pos += n as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.file.write_all(buf)?;
|
||||
self.pos += buf.len() as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for WritebackFile {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
let p = self.file.seek(from)?;
|
||||
// Only treat seeks that actually move the position as
|
||||
// boundaries — sweep does a redundant `seek(Current(pos))`
|
||||
// before every write, and we don't want that to drain the
|
||||
// pipeline on every iteration.
|
||||
if p != self.pos {
|
||||
self.pipeline.handle_seek(p);
|
||||
self.pos = p;
|
||||
}
|
||||
Ok(p)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WritebackFile {
|
||||
fn drop(&mut self) {
|
||||
// Run the pipeline's tail finalize so the last in-flight chunk
|
||||
// gets its `WAIT_AFTER` + `posix_fadvise(DONTNEED)`. Without
|
||||
// this, callers that drop a `WritebackFile` without calling
|
||||
// `sync_all` (panic, early-return, idiomatic `let _ = w;`)
|
||||
// leave the trailing chunk in cache; the kernel still flushes
|
||||
// on close, but the bounded-cache invariant fails at the tail.
|
||||
// We deliberately do *not* call `self.file.sync_all()` here —
|
||||
// close already triggers a flush, and an `fsync` from `Drop`
|
||||
// would silently swallow its `io::Error` anyway. `finalize` is
|
||||
// idempotent so an explicit `sync_all` followed by drop is
|
||||
// still safe.
|
||||
self.pipeline.finalize();
|
||||
}
|
||||
}
|
||||
+94
-1
@@ -268,16 +268,109 @@ fn generate_audio_label(
|
||||
}
|
||||
|
||||
fn extract(reader: &mut dyn SectorReader, udf: &UdfFs) -> Vec<StreamLabel> {
|
||||
for (_name, detect, parse) in PARSERS {
|
||||
for (name, detect, parse) in PARSERS {
|
||||
if detect(udf) {
|
||||
tracing::info!(parser = name, "label parser matched");
|
||||
if let Some(labels) = parse(reader, udf) {
|
||||
return labels;
|
||||
}
|
||||
}
|
||||
}
|
||||
tracing::info!("no label parser matched");
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// Diagnostic introspection — returns the parser that matched, the
|
||||
/// labels it emitted, and the inventory of files under `/BDMV/JAR/*/`
|
||||
/// that the discriminators looked at. Intended for `freemkv-tools
|
||||
/// labels-analyze` and corpus regression tooling, not production code
|
||||
/// paths. The matching/parsing logic is identical to [`extract`]; only
|
||||
/// the return shape is richer.
|
||||
#[doc(hidden)]
|
||||
pub fn analyze(reader: &mut dyn SectorReader, udf: &UdfFs) -> LabelAnalysis {
|
||||
let inventory = jar_inventory(udf);
|
||||
// Record every parser whose discriminator matched — even if its
|
||||
// parse step then returned None — so the analyzer can distinguish
|
||||
// "no parser recognized this disc" from "parser recognized it but
|
||||
// couldn't read the file" (e.g. content past a truncated capture)
|
||||
// or "parser ran but produced no labels."
|
||||
let mut parsers_detected: Vec<&'static str> = Vec::new();
|
||||
for (name, detect, parse) in PARSERS {
|
||||
if detect(udf) {
|
||||
tracing::info!(parser = name, "label parser matched");
|
||||
parsers_detected.push(name);
|
||||
if let Some(labels) = parse(reader, udf) {
|
||||
return LabelAnalysis {
|
||||
parser: Some(name),
|
||||
parsers_detected,
|
||||
jar_inventory: inventory,
|
||||
labels,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
if parsers_detected.is_empty() {
|
||||
tracing::info!("no label parser matched");
|
||||
} else {
|
||||
tracing::info!(
|
||||
detected = ?parsers_detected,
|
||||
"label parsers detected but produced no labels"
|
||||
);
|
||||
}
|
||||
LabelAnalysis {
|
||||
parser: None,
|
||||
parsers_detected,
|
||||
jar_inventory: inventory,
|
||||
labels: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of [`analyze`].
|
||||
#[doc(hidden)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LabelAnalysis {
|
||||
/// Which parser matched ("paramount" / "criterion" / "pixelogic" /
|
||||
/// "ctrm") AND emitted labels. `None` means either no parser
|
||||
/// recognized the disc, OR a parser recognized it but its parse
|
||||
/// step returned None (file unreadable, no parseable tokens). Use
|
||||
/// `parsers_detected` to disambiguate.
|
||||
pub parser: Option<&'static str>,
|
||||
/// Every parser whose discriminator matched, in priority order.
|
||||
/// Distinguishes "we recognized this disc but couldn't extract
|
||||
/// labels" from "we don't recognize this disc at all" — the
|
||||
/// former points at a parser bug or a truncated capture, the
|
||||
/// latter points at a missing parser.
|
||||
pub parsers_detected: Vec<&'static str>,
|
||||
/// Filenames found under any `/BDMV/JAR/*/` subdirectory, deduped
|
||||
/// and sorted. Helps spot unknown authoring formats when no
|
||||
/// parser detected.
|
||||
pub jar_inventory: Vec<String>,
|
||||
/// Raw labels emitted by the matched parser (empty if `parser` is
|
||||
/// `None`).
|
||||
pub labels: Vec<StreamLabel>,
|
||||
}
|
||||
|
||||
/// List filenames found under any `/BDMV/JAR/<x>/` subdirectory of
|
||||
/// the disc. Deduped, sorted. Returns an empty vec if no JAR dir is
|
||||
/// present.
|
||||
fn jar_inventory(udf: &UdfFs) -> Vec<String> {
|
||||
let Some(jar_dir) = udf.find_dir("/BDMV/JAR") else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for entry in &jar_dir.entries {
|
||||
if entry.is_dir {
|
||||
for child in &entry.entries {
|
||||
if !child.is_dir && !out.contains(&child.name) {
|
||||
out.push(child.name.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
|
||||
// ── Shared helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Check if a file exists in any BDMV/JAR subdirectory.
|
||||
|
||||
+55
-7
@@ -79,10 +79,12 @@ pub mod disc;
|
||||
pub mod drive;
|
||||
pub mod error;
|
||||
pub mod event;
|
||||
pub mod halt;
|
||||
pub(crate) mod identity;
|
||||
pub(crate) mod ifo;
|
||||
pub(crate) mod io;
|
||||
pub mod keydb;
|
||||
pub(crate) mod labels;
|
||||
pub mod labels;
|
||||
pub(crate) mod mpls;
|
||||
pub mod mux;
|
||||
pub mod pes;
|
||||
@@ -95,6 +97,9 @@ pub(crate) mod speed;
|
||||
pub(crate) mod udf;
|
||||
pub mod verify;
|
||||
|
||||
// Re-export verify types at the crate root for ergonomic imports.
|
||||
pub use verify::{SectorRange, SectorStatus, VerifyResult, verify_title};
|
||||
|
||||
// ─── Drive lifecycle ────────────────────────────────────────────────────────
|
||||
//
|
||||
// `Drive::open(path)` → `wait_ready()` → `init()` → `Disc::scan()`. `Drive`
|
||||
@@ -112,6 +117,28 @@ pub use drive::{Drive, DriveStatus, find_drive};
|
||||
// codes to localized messages. See `error.rs` for the full taxonomy.
|
||||
pub use error::{Error, Result};
|
||||
|
||||
// ─── 0.18 primitives ────────────────────────────────────────────────────────
|
||||
//
|
||||
// One-bit cooperative cancellation token, shared by every long-running loop
|
||||
// in libfreemkv (sweep, patch, mux). Replaces the ad-hoc `Arc<AtomicBool>`
|
||||
// flags scattered across 0.17 (`DiscStream::set_halt`, autorip's
|
||||
// `HALT_FLAGS` registry). Clone it cheaply; pass it by value into each
|
||||
// component; poll `is_cancelled()` inside the loop body.
|
||||
pub use halt::Halt;
|
||||
|
||||
// Generic bounded producer/consumer primitive used by sweep, patch, and
|
||||
// mux to overlap reads with writes via a dedicated consumer thread.
|
||||
// `Pipeline::spawn(depth, sink)` spawns the consumer; `pipe.send(item)`
|
||||
// pushes one item with back-pressure; `pipe.finish()` joins the
|
||||
// consumer and surfaces its `close()` output. Callers implement `Sink`
|
||||
// to define per-item behaviour and end-of-stream finalisation.
|
||||
//
|
||||
// `DEFAULT_PIPELINE_DEPTH` (=4) is the depth sweep + mux use; patch
|
||||
// uses `WRITE_THROUGH_DEPTH` (=1) so each read fully drains before the
|
||||
// next can enqueue. Returning `Flow::Stop` from `apply` ends the
|
||||
// consumer cleanly (still calls `close()`).
|
||||
pub use io::pipeline::{DEFAULT_PIPELINE_DEPTH, Flow, Pipeline, Sink, WRITE_THROUGH_DEPTH};
|
||||
|
||||
// ─── Drive events (low-level callbacks) ─────────────────────────────────────
|
||||
pub use event::{Event, EventKind};
|
||||
pub use identity::DriveId;
|
||||
@@ -133,11 +160,15 @@ pub use decrypt::{DecryptKeys, decrypt_sectors};
|
||||
// for displaying disc name + format quickly while a full scan runs in the
|
||||
// background. The codec / channel / resolution enums are the canonical
|
||||
// structured representation; never compare against display strings.
|
||||
// Note: `disc::Stream` here is the codec enum (audio / video / sub kind)
|
||||
// — not the `pes::Stream` trait re-exported below as `PesStream`. Two
|
||||
// different concepts, the same short name; both stay because both are
|
||||
// load-bearing in their respective domains.
|
||||
pub use disc::{
|
||||
AacsState, AudioChannels, AudioStream, Clip, Codec, ColorSpace, ContentFormat, DamageSeverity,
|
||||
Disc, DiscFormat, DiscId, DiscTitle, Extent, FrameRate, HdrFormat, KeySource, LabelPurpose,
|
||||
LabelQualifier, Resolution, SampleRate, ScanOptions, Stream, SubtitleStream, VideoStream,
|
||||
classify_damage,
|
||||
LabelQualifier, PatchOptions, PatchOutcome, Resolution, SampleRate, ScanOptions, Stream,
|
||||
SubtitleStream, SweepOptions, VideoStream, classify_damage,
|
||||
};
|
||||
|
||||
// ─── Streams ────────────────────────────────────────────────────────────────
|
||||
@@ -155,6 +186,15 @@ pub use disc::{
|
||||
// Most consumers use the URL resolvers (`input()` / `output()`) which pick
|
||||
// the right type from a scheme:// URL. Direct construction is for callers
|
||||
// that need to wire custom readers (e.g. autorip's drive-session reuse).
|
||||
// 0.18 trait split: `FrameSource` (read-only) and `FrameSink` (write-only)
|
||||
// supersede the unified `pes::Stream`. The old `Stream` re-export below
|
||||
// stays available for the deprecation window — re-exported as
|
||||
// `PesStream` to disambiguate from `disc::Stream` (the codec-kind enum
|
||||
// re-exported above), which would otherwise collide at the crate root.
|
||||
#[allow(deprecated)]
|
||||
pub use pes::Stream as PesStream;
|
||||
pub use pes::{FrameSink, FrameSource, PesFrame};
|
||||
|
||||
pub use mux::DiscStream;
|
||||
pub use mux::M2tsStream;
|
||||
pub use mux::MkvStream;
|
||||
@@ -166,10 +206,18 @@ pub use mux::{InputOptions, StreamUrl, input, output, parse_url};
|
||||
// ─── Lower-level surfaces ───────────────────────────────────────────────────
|
||||
//
|
||||
// `ScsiTransport` is the platform-abstraction trait Drive uses; expose for
|
||||
// out-of-tree platform backends. `SectorReader` lets callers feed any byte
|
||||
// source (test harness, network image, SMB share) into the disc scan
|
||||
// pipeline; `FileSectorReader` is the standard ISO-on-disk implementation.
|
||||
// out-of-tree platform backends. `SectorSource` / `SectorSink` are the 0.18
|
||||
// direction-typed read/write traits; `FileSectorSource` and `FileSectorSink`
|
||||
// are the ISO-on-disk implementations. [`DecryptingSectorSource`] is the
|
||||
// single decrypt-on-read decorator (AACS / CSS / none) — wrap any
|
||||
// `SectorSource` to get plaintext sectors out. The legacy `SectorReader` /
|
||||
// `FileSectorReader` names stay re-exported through the 0.18 migration
|
||||
// window so existing call sites compile unchanged; a blanket impl makes
|
||||
// every `SectorReader` automatically usable as a `SectorSource`.
|
||||
pub use scsi::{DriveInfo, ScsiSense, ScsiTransport, drive_has_disc, list_drives};
|
||||
pub use sector::{FileSectorReader, SectorReader};
|
||||
pub use sector::{
|
||||
DecryptingSectorSource, FileSectorReader, FileSectorSink, FileSectorSource, SectorReader,
|
||||
SectorSink, SectorSource,
|
||||
};
|
||||
pub use speed::DriveSpeed;
|
||||
pub use udf::{UdfFs, read_filesystem};
|
||||
|
||||
+209
-22
@@ -8,10 +8,11 @@
|
||||
use crate::disc::{Disc, DiscTitle, Extent};
|
||||
use crate::drive::extract_scsi_context;
|
||||
use crate::event::{BatchSizeReason, Event, EventKind};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::halt::Halt;
|
||||
use crate::sector::{DecryptingSectorSource, SectorReader, SectorSource};
|
||||
use std::io;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
/// Ramp back up to the preferred batch size after this many sectors
|
||||
/// of clean reading at the current (reduced) size. 100 MiB = 51,200 sectors.
|
||||
@@ -99,9 +100,19 @@ impl AdaptiveBatch {
|
||||
/// Sources: physical drive, ISO file, or any SectorReader.
|
||||
/// Decrypt, demux, and codec parsing happen internally.
|
||||
pub struct DiscStream {
|
||||
reader: Box<dyn SectorReader>,
|
||||
/// Underlying sector source wrapped in the 0.18
|
||||
/// [`DecryptingSectorSource`] decorator. Every `read_sectors`
|
||||
/// call yields plaintext, so `fill_extents` no longer needs an
|
||||
/// inline `decrypt::decrypt_sectors` step. `DecryptKeys::None`
|
||||
/// (raw / unencrypted disc) makes the decorator a pass-through.
|
||||
reader: DecryptingSectorSource<Box<dyn SectorReader>>,
|
||||
title: DiscTitle,
|
||||
disc: Option<Disc>,
|
||||
/// Mirror of the keys handed in at construction. The decorator
|
||||
/// owns the cryptographic state; this field is kept for
|
||||
/// metadata-side callers (`info()` and friends) that want to
|
||||
/// know whether the disc was encrypted, without reaching through
|
||||
/// the wrapper.
|
||||
decrypt_keys: crate::decrypt::DecryptKeys,
|
||||
|
||||
// Extents to read
|
||||
@@ -120,11 +131,13 @@ pub struct DiscStream {
|
||||
adaptive: AdaptiveBatch,
|
||||
pub errors: u64,
|
||||
pub skip_errors: bool,
|
||||
/// When set and the flag is raised, fill_extents returns Err(Halted) at the
|
||||
/// next retry boundary. Unlike skip_errors, this propagates the error up so
|
||||
/// the rip terminates cleanly. Share the Arc with Drive::halt_flag() to get
|
||||
/// unified Stop behavior across drive reads and sector processing.
|
||||
halt: Option<Arc<AtomicBool>>,
|
||||
/// When set and the token is cancelled, fill_extents returns Err(Halted)
|
||||
/// at the next retry boundary. Unlike skip_errors, this propagates the
|
||||
/// error up so the rip terminates cleanly. Construct with
|
||||
/// [`DiscStream::with_halt`] (preferred) or set post-hoc via the
|
||||
/// deprecated [`DiscStream::set_halt`] bridge — both populate this same
|
||||
/// field and either entry point yields one source of truth.
|
||||
halt: Option<Halt>,
|
||||
event_fn: Option<Box<dyn Fn(Event) + Send>>,
|
||||
eof: bool,
|
||||
|
||||
@@ -189,7 +202,11 @@ impl DiscStream {
|
||||
}
|
||||
|
||||
Self {
|
||||
reader,
|
||||
// Wrap the input reader in DecryptingSectorSource so the
|
||||
// internal fill_extents path sees plaintext bytes. For
|
||||
// DecryptKeys::None (unencrypted / raw / test fixtures)
|
||||
// the decorator is a pass-through.
|
||||
reader: DecryptingSectorSource::new(reader, decrypt_keys.clone()),
|
||||
title,
|
||||
disc: None,
|
||||
decrypt_keys,
|
||||
@@ -219,19 +236,39 @@ impl DiscStream {
|
||||
self.event_fn = Some(Box::new(f));
|
||||
}
|
||||
|
||||
/// Share a halt flag — typically from `Drive::halt_flag()`. When raised,
|
||||
/// the next read-retry boundary inside fill_extents returns Err(Halted)
|
||||
/// instead of continuing. Required for Stop to work during dense bad-sector
|
||||
/// regions (where read() loops internally waiting for enough clean data to
|
||||
/// emit a frame and would otherwise never check an external stop signal).
|
||||
/// Constructor-time builder: attach a [`Halt`] token so that when
|
||||
/// any clone is cancelled, the next read-retry boundary inside
|
||||
/// `fill_extents` returns `Err(Halted)`. Required for Stop to work
|
||||
/// during dense bad-sector regions (where the outer PES read() loop
|
||||
/// can spend minutes inside fill_extents before emitting a frame).
|
||||
///
|
||||
/// Preferred over the post-hoc [`DiscStream::set_halt`] bridge —
|
||||
/// pass the same `Halt` clone you hand to sweep / patch / mux so
|
||||
/// every phase observes a single Stop signal.
|
||||
pub fn with_halt(mut self, halt: Halt) -> Self {
|
||||
self.halt = Some(halt);
|
||||
self
|
||||
}
|
||||
|
||||
/// Bridge for callers that haven't migrated to the
|
||||
/// [`DiscStream::with_halt`] constructor-time path yet. Wraps the
|
||||
/// supplied `Arc<AtomicBool>` as a [`Halt`] (`Halt::from_arc`) and
|
||||
/// stores it in the same internal slot, so a halt installed via
|
||||
/// either entry point goes through one halt-check inside
|
||||
/// `fill_extents`. Calling `set_halt` after `with_halt` (or vice
|
||||
/// versa) replaces the previous token with the new one.
|
||||
#[deprecated(
|
||||
since = "0.18.0",
|
||||
note = "use `DiscStream::with_halt(Halt)` at construction instead"
|
||||
)]
|
||||
pub fn set_halt(&mut self, flag: Arc<AtomicBool>) {
|
||||
self.halt = Some(flag);
|
||||
self.halt = Some(Halt::from_arc(flag));
|
||||
}
|
||||
|
||||
fn is_halted(&self) -> bool {
|
||||
self.halt
|
||||
.as_ref()
|
||||
.map(|h| h.load(Ordering::Relaxed))
|
||||
.map(|h| h.is_cancelled())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
@@ -241,9 +278,12 @@ impl DiscStream {
|
||||
}
|
||||
}
|
||||
|
||||
/// Skip decryption — return raw encrypted bytes.
|
||||
/// Skip decryption — return raw encrypted bytes. Updates both
|
||||
/// the metadata-side key field and the wrapped reader's keys so
|
||||
/// subsequent `read_sectors` calls become a pass-through.
|
||||
pub fn set_raw(&mut self) {
|
||||
self.decrypt_keys = crate::decrypt::DecryptKeys::None;
|
||||
self.reader.set_keys(crate::decrypt::DecryptKeys::None);
|
||||
}
|
||||
|
||||
/// Get the scanned Disc (for listing all titles).
|
||||
@@ -347,6 +387,7 @@ impl DiscStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource/FrameSink in follow-up commit.
|
||||
impl crate::pes::Stream for DiscStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
if let Some(frame) = self.pending_frames.pop_front() {
|
||||
@@ -419,11 +460,10 @@ impl crate::pes::Stream for DiscStream {
|
||||
}
|
||||
|
||||
let bytes = self.buf_valid;
|
||||
if let Err(e) =
|
||||
crate::decrypt::decrypt_sectors(&mut self.read_buf[..bytes], &self.decrypt_keys, 0)
|
||||
{
|
||||
return Err(e.into());
|
||||
}
|
||||
// Plaintext: the wrapped reader (DecryptingSectorSource)
|
||||
// applied AACS / CSS in-place during fill_extents'
|
||||
// read_sectors call. The pre-0.18 inline decrypt step
|
||||
// lived here.
|
||||
|
||||
if let Some(ref mut demuxer) = self.ts_demuxer {
|
||||
let packets = demuxer.feed(&self.read_buf[..bytes]);
|
||||
@@ -525,3 +565,150 @@ impl crate::pes::Stream for DiscStream {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
//! `DiscStream` is the only meaningful `FrameSource` impl in tree (every
|
||||
//! other concrete `pes::Stream` impl in `mux/*` is a sink). The 0.18
|
||||
//! round-1 blanket `impl<T: pes::Stream + Send> pes::FrameSource for T`
|
||||
//! covers `DiscStream` for free as long as it is `Send`. These tests
|
||||
//! lock that down: a static `Send` assertion plus a `Box<dyn FrameSource>`
|
||||
//! round trip exercising every `FrameSource` method through the trait
|
||||
//! object, so future Send-breaking edits to `DiscStream`'s interior
|
||||
//! types fail at compile time and the trait-bridge dispatch is verified
|
||||
//! at runtime.
|
||||
#![allow(deprecated)] // exercising the 0.18 deprecation-window blanket bridge.
|
||||
use super::*;
|
||||
use crate::disc::{ContentFormat, DiscTitle};
|
||||
use crate::pes::FrameSource;
|
||||
|
||||
/// Static-assert `DiscStream: Send`. The blanket
|
||||
/// `impl<T: pes::Stream + Send> pes::FrameSource for T` only fires for
|
||||
/// `Send` types — if a future field on `DiscStream` is non-`Send` (e.g.
|
||||
/// a `Box<dyn Read>` instead of `Box<dyn SectorReader>`), this fails
|
||||
/// at compile time, before the runtime trait-object test below.
|
||||
fn _assert_disc_stream_is_send() {
|
||||
fn requires_send<T: Send>() {}
|
||||
requires_send::<DiscStream>();
|
||||
}
|
||||
|
||||
/// Trivial `SectorReader` that yields zeroed sectors. Empty title means
|
||||
/// the demuxer produces no PES frames, so `read()` walks the extents to
|
||||
/// EOF and returns `Ok(None)`. That's enough to exercise the trait-object
|
||||
/// dispatch — the goal here is the bridge, not the demuxer.
|
||||
struct ZeroReader {
|
||||
capacity: u32,
|
||||
}
|
||||
|
||||
impl crate::sector::SectorReader for ZeroReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
_lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> crate::error::Result<usize> {
|
||||
let bytes = count as usize * 2048;
|
||||
buf[..bytes].fill(0);
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn capacity(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
|
||||
fn synthetic_title(sector_count: u32) -> DiscTitle {
|
||||
DiscTitle {
|
||||
extents: vec![crate::disc::Extent {
|
||||
start_lba: 0,
|
||||
sector_count,
|
||||
}],
|
||||
..DiscTitle::empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Smallest credible witness that `DiscStream` flows through the
|
||||
/// `FrameSource` blanket impl: build a `Box<dyn FrameSource>`, drive
|
||||
/// `read()` to EOF, exercise `info()` / `headers_ready()` /
|
||||
/// `codec_private()` through the trait object. The trait-bridge
|
||||
/// correctness is what's being verified — not demuxer behaviour.
|
||||
#[test]
|
||||
fn frame_source_via_dyn_object() {
|
||||
let reader = ZeroReader { capacity: 8 };
|
||||
let title = synthetic_title(8);
|
||||
let stream = DiscStream::new(
|
||||
Box::new(reader),
|
||||
title,
|
||||
crate::decrypt::DecryptKeys::None,
|
||||
8,
|
||||
ContentFormat::BdTs,
|
||||
);
|
||||
|
||||
let mut src: Box<dyn FrameSource> = Box::new(stream);
|
||||
|
||||
// Empty-title fixture has no streams configured, so headers are
|
||||
// trivially ready and codec_private() yields nothing on track 0.
|
||||
assert!(src.headers_ready());
|
||||
assert!(src.codec_private(0).is_none());
|
||||
let _ = src.info();
|
||||
|
||||
// Drive read() to EOF through the trait object — empty-title fixture
|
||||
// produces no frames, but the call still routes through the blanket
|
||||
// dispatch into Stream::read.
|
||||
let mut frames = 0usize;
|
||||
loop {
|
||||
match src.read().expect("read") {
|
||||
Some(_) => frames += 1,
|
||||
None => break,
|
||||
}
|
||||
if frames > 1024 {
|
||||
panic!("unexpected unbounded frame stream from empty title");
|
||||
}
|
||||
}
|
||||
assert_eq!(frames, 0);
|
||||
}
|
||||
|
||||
/// `is_halted()` must observe a cancellation signal regardless of
|
||||
/// which entry point installed the token. The deprecated
|
||||
/// `set_halt(Arc<AtomicBool>)` and the new `with_halt(Halt)` are
|
||||
/// two views over one slot — flipping either bit must cause the
|
||||
/// next `fill_extents` retry boundary to bail.
|
||||
#[test]
|
||||
fn halt_via_with_halt_observed_by_is_halted() {
|
||||
let halt = Halt::new();
|
||||
let stream = DiscStream::new(
|
||||
Box::new(ZeroReader { capacity: 8 }),
|
||||
synthetic_title(8),
|
||||
crate::decrypt::DecryptKeys::None,
|
||||
8,
|
||||
crate::disc::ContentFormat::BdTs,
|
||||
)
|
||||
.with_halt(halt.clone());
|
||||
assert!(!stream.is_halted());
|
||||
halt.cancel();
|
||||
assert!(
|
||||
stream.is_halted(),
|
||||
"with_halt token cancellation must be observed by is_halted()"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_via_set_halt_bridge_observed_by_is_halted() {
|
||||
let arc = Arc::new(AtomicBool::new(false));
|
||||
let mut stream = DiscStream::new(
|
||||
Box::new(ZeroReader { capacity: 8 }),
|
||||
synthetic_title(8),
|
||||
crate::decrypt::DecryptKeys::None,
|
||||
8,
|
||||
crate::disc::ContentFormat::BdTs,
|
||||
);
|
||||
stream.set_halt(arc.clone());
|
||||
assert!(!stream.is_halted());
|
||||
arc.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(
|
||||
stream.is_halted(),
|
||||
"set_halt(Arc<AtomicBool>) bridge must observe Arc-side flips"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+32
-7
@@ -18,10 +18,10 @@ const SCAN_SIZE: usize = 1024 * 1024;
|
||||
|
||||
enum Mode {
|
||||
Write {
|
||||
muxer: super::tsmux::TsMuxer<Box<dyn Write>>,
|
||||
muxer: super::tsmux::TsMuxer<Box<dyn Write + Send>>,
|
||||
},
|
||||
Read {
|
||||
reader: Box<dyn Read>,
|
||||
reader: Box<dyn Read + Send>,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -57,7 +57,7 @@ pub struct M2tsStream {
|
||||
impl M2tsStream {
|
||||
/// Create for writing PES frames → BD-TS output.
|
||||
/// Writes FMKV metadata header, then muxes PES frames into BD transport stream.
|
||||
pub fn create(mut writer: impl Write + 'static, title: &DiscTitle) -> io::Result<Self> {
|
||||
pub fn create(mut writer: impl Write + Send + 'static, title: &DiscTitle) -> io::Result<Self> {
|
||||
// Write FMKV metadata header
|
||||
if !title.streams.is_empty() {
|
||||
let m = meta::M2tsMeta::from_title(title);
|
||||
@@ -72,7 +72,7 @@ impl M2tsStream {
|
||||
DiscStream::Subtitle(s) => s.pid,
|
||||
})
|
||||
.collect();
|
||||
let boxed: Box<dyn Write> = Box::new(writer);
|
||||
let boxed: Box<dyn Write + Send> = Box::new(writer);
|
||||
let mut muxer = super::tsmux::TsMuxer::new(boxed, &pids);
|
||||
for (i, cp) in title.codec_privates.iter().enumerate() {
|
||||
if let Some(data) = cp {
|
||||
@@ -111,7 +111,7 @@ impl M2tsStream {
|
||||
/// Open an M2TS stream for reading. Takes any Read source — file, pipe, socket.
|
||||
///
|
||||
/// Tries FMKV metadata header first. Falls back to PMT scan of first 1 MB.
|
||||
pub fn open(mut reader: impl Read + 'static) -> io::Result<Self> {
|
||||
pub fn open(mut reader: impl Read + Send + 'static) -> io::Result<Self> {
|
||||
// Read first chunk — enough for FMKV header or PMT scan
|
||||
let mut head = vec![0u8; SCAN_SIZE];
|
||||
let head_len = read_fill(&mut reader, &mut head)?;
|
||||
@@ -125,7 +125,7 @@ impl M2tsStream {
|
||||
let (pids, parsers, pid_to_track) = Self::setup_pes(&title.streams);
|
||||
// Chain: remaining head bytes + rest of reader
|
||||
let remaining_head = &head[header_end..];
|
||||
let chain: Box<dyn Read> =
|
||||
let chain: Box<dyn Read + Send> =
|
||||
Box::new(io::Cursor::new(remaining_head.to_vec()).chain(reader));
|
||||
return Ok(Self {
|
||||
disc_title: title.clone(),
|
||||
@@ -150,7 +150,7 @@ impl M2tsStream {
|
||||
let (pids, parsers, pid_to_track) = Self::setup_pes(&streams);
|
||||
|
||||
// Chain: full head (it's all TS data) + rest of reader
|
||||
let chain: Box<dyn Read> = Box::new(io::Cursor::new(head).chain(reader));
|
||||
let chain: Box<dyn Read + Send> = Box::new(io::Cursor::new(head).chain(reader));
|
||||
|
||||
Ok(Self {
|
||||
disc_title: DiscTitle {
|
||||
@@ -173,6 +173,7 @@ impl M2tsStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource/FrameSink in follow-up commit.
|
||||
impl crate::pes::Stream for M2tsStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
if let Some(frame) = self.pending_frames.pop_front() {
|
||||
@@ -283,3 +284,27 @@ impl crate::pes::Stream for M2tsStream {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// FrameSink sibling to the deprecated Stream impl; both coexist during the
|
||||
/// 0.18 deprecation window. Caller may pick either at the trait-object
|
||||
/// boundary — `Box<dyn Stream>` (deprecated) or `Box<dyn FrameSink>` (new).
|
||||
/// Use `M2tsStream::create(writer, title)` to construct the write half;
|
||||
/// calling `FrameSink::write` on an `M2tsStream::open(reader)` instance
|
||||
/// returns `StreamReadOnly`.
|
||||
#[allow(deprecated)] // delegating to deprecated Stream during the 0.18 deprecation window so callers don't see the deprecation twice.
|
||||
impl crate::pes::FrameSink for M2tsStream {
|
||||
fn write(&mut self, frame: &crate::pes::PesFrame) -> io::Result<()> {
|
||||
<Self as crate::pes::Stream>::write(self, frame)
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> io::Result<()> {
|
||||
// Why: Stream::finish takes &mut self, FrameSink::finish takes Box<Self>.
|
||||
// Re-borrow inside the box, call Stream::finish, drop the box.
|
||||
let mut s: Self = *self;
|
||||
<Self as crate::pes::Stream>::finish(&mut s)
|
||||
}
|
||||
|
||||
fn info(&self) -> &crate::disc::DiscTitle {
|
||||
<Self as crate::pes::Stream>::info(self)
|
||||
}
|
||||
}
|
||||
|
||||
+34
-4
@@ -18,7 +18,7 @@ use crate::disc::*;
|
||||
use std::io::{self, Read};
|
||||
|
||||
struct ReadState {
|
||||
reader: Box<dyn Read>,
|
||||
reader: Box<dyn Read + Send>,
|
||||
cluster_ts_ms: i64,
|
||||
/// Codec private data per track (track_number, hvcC/avcC bytes).
|
||||
codec_privates: Vec<(u16, Vec<u8>)>,
|
||||
@@ -26,7 +26,7 @@ struct ReadState {
|
||||
|
||||
enum Mode {
|
||||
Write {
|
||||
muxer: Option<MkvMuxer<Box<dyn WriteSeek>>>,
|
||||
muxer: Option<MkvMuxer<Box<dyn WriteSeek + Send>>>,
|
||||
},
|
||||
Read(ReadState),
|
||||
}
|
||||
@@ -40,7 +40,7 @@ pub struct MkvStream {
|
||||
impl MkvStream {
|
||||
/// Create for writing PES frames → MKV container.
|
||||
/// Codec privates come from title.codec_privates (populated by input stream).
|
||||
pub fn create(writer: Box<dyn WriteSeek>, title: &DiscTitle) -> io::Result<Self> {
|
||||
pub fn create(writer: Box<dyn WriteSeek + Send>, title: &DiscTitle) -> io::Result<Self> {
|
||||
let mut tracks = Vec::new();
|
||||
let mut has_default_video = false;
|
||||
let mut has_default_audio = false;
|
||||
@@ -79,7 +79,7 @@ impl MkvStream {
|
||||
}
|
||||
|
||||
/// Open an MKV file for reading → PES frames.
|
||||
pub fn open(mut reader: impl Read + 'static) -> io::Result<Self> {
|
||||
pub fn open(mut reader: impl Read + Send + 'static) -> io::Result<Self> {
|
||||
let (disc_title, codec_privates) = parse_mkv_header(&mut reader)?;
|
||||
Ok(Self {
|
||||
disc_title,
|
||||
@@ -92,6 +92,7 @@ impl MkvStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource/FrameSink in follow-up commit.
|
||||
impl crate::pes::Stream for MkvStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
let rs = match self.mode {
|
||||
@@ -188,6 +189,35 @@ impl crate::pes::Stream for MkvStream {
|
||||
}
|
||||
}
|
||||
|
||||
/// FrameSink sibling to the deprecated Stream impl; both coexist during the
|
||||
/// 0.18 deprecation window. Caller may pick either at the trait-object
|
||||
/// boundary — `Box<dyn Stream>` (deprecated) or `Box<dyn FrameSink>` (new).
|
||||
/// Use `MkvStream::create(writer, title)` to construct the write half;
|
||||
/// calling `FrameSink::write` on a `MkvStream::open(reader)` instance returns
|
||||
/// `StreamReadOnly`. `finish` is where the Cues index is written, so it must
|
||||
/// be called for the resulting MKV to be seekable.
|
||||
#[allow(deprecated)] // delegating to deprecated Stream during the 0.18 deprecation window so callers don't see the deprecation twice.
|
||||
impl crate::pes::FrameSink for MkvStream {
|
||||
fn write(&mut self, frame: &crate::pes::PesFrame) -> io::Result<()> {
|
||||
<Self as crate::pes::Stream>::write(self, frame)
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> io::Result<()> {
|
||||
// Why: Stream::finish takes &mut self, FrameSink::finish takes Box<Self>.
|
||||
// Re-borrow inside the box, call Stream::finish, drop the box.
|
||||
// The inner `MkvMuxer` is owned via `Option`, and `Stream::finish`
|
||||
// already takes it via `Option::take()` — moving `*self` out of the
|
||||
// box hands it the same field by-value, so the muxer's own
|
||||
// by-value `finish()` runs correctly.
|
||||
let mut s: Self = *self;
|
||||
<Self as crate::pes::Stream>::finish(&mut s)
|
||||
}
|
||||
|
||||
fn info(&self) -> &crate::disc::DiscTitle {
|
||||
<Self as crate::pes::Stream>::info(self)
|
||||
}
|
||||
}
|
||||
|
||||
// ── MKV header parsing (read side) ────────────────────────────
|
||||
|
||||
/// Returns (DiscTitle, codec_privates: Vec<(track_number, codec_private_bytes)>)
|
||||
|
||||
@@ -70,6 +70,7 @@ impl NetworkStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource/FrameSink in follow-up commit.
|
||||
impl crate::pes::Stream for NetworkStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
match &mut self.mode {
|
||||
@@ -108,6 +109,30 @@ impl crate::pes::Stream for NetworkStream {
|
||||
}
|
||||
}
|
||||
|
||||
/// FrameSink sibling to the deprecated Stream impl; both coexist during the
|
||||
/// 0.18 deprecation window. Caller may pick either at the trait-object
|
||||
/// boundary — `Box<dyn Stream>` (deprecated) or `Box<dyn FrameSink>` (new).
|
||||
/// Use `NetworkStream::connect(addr).meta(title)` to construct the write
|
||||
/// half; calling `FrameSink::write` on `NetworkStream::listen(addr)` returns
|
||||
/// `StreamReadOnly`.
|
||||
#[allow(deprecated)] // delegating to deprecated Stream during the 0.18 deprecation window so callers don't see the deprecation twice.
|
||||
impl crate::pes::FrameSink for NetworkStream {
|
||||
fn write(&mut self, frame: &crate::pes::PesFrame) -> io::Result<()> {
|
||||
<Self as crate::pes::Stream>::write(self, frame)
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> io::Result<()> {
|
||||
// Why: Stream::finish takes &mut self, FrameSink::finish takes Box<Self>.
|
||||
// Re-borrow inside the box, call Stream::finish, drop the box.
|
||||
let mut s: Self = *self;
|
||||
<Self as crate::pes::Stream>::finish(&mut s)
|
||||
}
|
||||
|
||||
fn info(&self) -> &DiscTitle {
|
||||
<Self as crate::pes::Stream>::info(self)
|
||||
}
|
||||
}
|
||||
|
||||
// NetworkStream is PES-only — no IOStream/Read/Write byte interface.
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -157,6 +182,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
#[ignore] // Requires TCP; may be flaky in CI environments
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource/FrameSink in follow-up commit.
|
||||
fn network_pes_roundtrip() {
|
||||
use crate::pes;
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ impl NullStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSink in follow-up commit.
|
||||
impl crate::pes::Stream for NullStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
Ok(None)
|
||||
@@ -30,3 +31,55 @@ impl crate::pes::Stream for NullStream {
|
||||
&self.disc_title
|
||||
}
|
||||
}
|
||||
|
||||
/// FrameSink sibling to the deprecated Stream impl; both coexist during the
|
||||
/// 0.18 deprecation window. Caller may pick either at the trait-object
|
||||
/// boundary — `Box<dyn Stream>` (deprecated) or `Box<dyn FrameSink>` (new).
|
||||
/// The deprecation-window callers eventually migrate; this impl exists so
|
||||
/// new callers can target `FrameSink` without waiting for the rest of the
|
||||
/// migration to complete.
|
||||
#[allow(deprecated)] // delegating to deprecated Stream during the 0.18 deprecation window so callers don't see the deprecation twice.
|
||||
impl crate::pes::FrameSink for NullStream {
|
||||
fn write(&mut self, frame: &crate::pes::PesFrame) -> io::Result<()> {
|
||||
<Self as crate::pes::Stream>::write(self, frame)
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> io::Result<()> {
|
||||
// Why: Stream::finish takes &mut self, FrameSink::finish takes Box<Self>.
|
||||
// Re-borrow inside the box, call Stream::finish, drop the box.
|
||||
let mut s: Self = *self;
|
||||
<Self as crate::pes::Stream>::finish(&mut s)
|
||||
}
|
||||
|
||||
fn info(&self) -> &DiscTitle {
|
||||
<Self as crate::pes::Stream>::info(self)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::pes::FrameSink;
|
||||
|
||||
/// Smallest credible witness that the new FrameSink impl on a concrete
|
||||
/// `mux/*` sink works through the trait object: build a boxed
|
||||
/// `dyn FrameSink`, write a frame, finish it. The trait-bridge correctness
|
||||
/// is what's being verified — not NullStream-specific behaviour.
|
||||
#[test]
|
||||
fn frame_sink_via_dyn_object_writes_and_finishes() {
|
||||
let title = DiscTitle::empty();
|
||||
let mut sink: Box<dyn FrameSink> = Box::new(NullStream::new(&title));
|
||||
|
||||
let frame = crate::pes::PesFrame {
|
||||
track: 0,
|
||||
pts: 0,
|
||||
keyframe: true,
|
||||
data: vec![0x01, 0x02, 0x03],
|
||||
};
|
||||
sink.write(&frame).unwrap();
|
||||
// info() routes through the trait object.
|
||||
let _ = sink.info();
|
||||
// finish() consumes the Box<Self> — must compile and run.
|
||||
sink.finish().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
+16
-5
@@ -167,6 +167,7 @@ pub struct InputOptions {
|
||||
}
|
||||
|
||||
/// Open a PES input stream (produces PES frames).
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource in follow-up commit.
|
||||
pub fn input(url: &str, opts: &InputOptions) -> io::Result<Box<dyn crate::pes::Stream>> {
|
||||
let parsed = parse_url(url);
|
||||
match parsed {
|
||||
@@ -234,6 +235,7 @@ pub fn input(url: &str, opts: &InputOptions) -> io::Result<Box<dyn crate::pes::S
|
||||
}
|
||||
|
||||
/// Open a PES output stream (consumes PES frames).
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSink in follow-up commit.
|
||||
pub fn output(
|
||||
url: &str,
|
||||
title: &crate::disc::DiscTitle,
|
||||
@@ -242,15 +244,24 @@ pub fn output(
|
||||
match parsed {
|
||||
StreamUrl::Mkv { ref path } => {
|
||||
validate_file_path(path, "mkv")?;
|
||||
let file = std::fs::File::create(path)?;
|
||||
let writer: Box<dyn super::WriteSeek> =
|
||||
Box::new(std::io::BufWriter::with_capacity(IO_BUF_SIZE, file));
|
||||
// Wrap the output in `crate::io::WritebackFile` (bounded-cache
|
||||
// writeback) so a UHD-scale MKV mux to slow / network-attached
|
||||
// staging doesn't hit the dirty-page burst pathology that
|
||||
// sweep already side-steps. BufWriter sits on top to coalesce
|
||||
// mux's many small EBML element writes.
|
||||
let writer: Box<dyn super::WriteSeek + Send> =
|
||||
Box::new(std::io::BufWriter::with_capacity(
|
||||
IO_BUF_SIZE,
|
||||
crate::io::WritebackFile::create(path)?,
|
||||
));
|
||||
Ok(Box::new(MkvStream::create(writer, title)?))
|
||||
}
|
||||
StreamUrl::M2ts { ref path } => {
|
||||
validate_file_path(path, "m2ts")?;
|
||||
let file = std::fs::File::create(path)?;
|
||||
let writer = std::io::BufWriter::with_capacity(IO_BUF_SIZE, file);
|
||||
let writer = std::io::BufWriter::with_capacity(
|
||||
IO_BUF_SIZE,
|
||||
crate::io::WritebackFile::create(path)?,
|
||||
);
|
||||
Ok(Box::new(M2tsStream::create(writer, title)?))
|
||||
}
|
||||
StreamUrl::Network { ref addr } => {
|
||||
|
||||
@@ -60,6 +60,7 @@ impl StdioStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource/FrameSink in follow-up commit.
|
||||
impl crate::pes::Stream for StdioStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
self.ensure_header_read()?;
|
||||
@@ -104,3 +105,26 @@ impl crate::pes::Stream for StdioStream {
|
||||
self.header_read || self.writer.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
/// FrameSink sibling to the deprecated Stream impl; both coexist during the
|
||||
/// 0.18 deprecation window. Caller may pick either at the trait-object
|
||||
/// boundary — `Box<dyn Stream>` (deprecated) or `Box<dyn FrameSink>` (new).
|
||||
/// Use `StdioStream::output(title)` to construct the write half; calling
|
||||
/// `FrameSink::write` on a `StdioStream::input()` returns `StreamReadOnly`.
|
||||
#[allow(deprecated)] // delegating to deprecated Stream during the 0.18 deprecation window so callers don't see the deprecation twice.
|
||||
impl crate::pes::FrameSink for StdioStream {
|
||||
fn write(&mut self, frame: &crate::pes::PesFrame) -> io::Result<()> {
|
||||
<Self as crate::pes::Stream>::write(self, frame)
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> io::Result<()> {
|
||||
// Why: Stream::finish takes &mut self, FrameSink::finish takes Box<Self>.
|
||||
// Re-borrow inside the box, call Stream::finish, drop the box.
|
||||
let mut s: Self = *self;
|
||||
<Self as crate::pes::Stream>::finish(&mut s)
|
||||
}
|
||||
|
||||
fn info(&self) -> &DiscTitle {
|
||||
<Self as crate::pes::Stream>::info(self)
|
||||
}
|
||||
}
|
||||
|
||||
+293
-34
@@ -78,43 +78,19 @@ impl PesFrame {
|
||||
}
|
||||
}
|
||||
|
||||
/// A PES frame source or sink. Each implementor is **either** read-only or
|
||||
/// write-only — never both.
|
||||
///
|
||||
/// Implementors fall into two camps:
|
||||
///
|
||||
/// - **Read sources**: `DiscStream` (drive or ISO), `M2tsStream` (when
|
||||
/// constructed from an existing file), `MkvStream` (demux), `NetworkStream`
|
||||
/// (TCP listener), `StdioStream::input()`. These return frames from
|
||||
/// `read()` and surface `StreamWriteOnly` (E9001) from `write()`.
|
||||
/// - **Write sinks**: `MkvStream::create`, `M2tsStream::create`,
|
||||
/// `NetworkStream::connect`, `StdioStream::output()`, `NullStream`.
|
||||
/// These accept frames in `write()` and surface `StreamReadOnly` (E9000)
|
||||
/// from `read()`. Always call `finish()` when done — that's where MKV
|
||||
/// writes its `Cues` index and `M2tsStream` flushes the TS muxer.
|
||||
///
|
||||
/// Direction is established at construction; mixing produces an error code,
|
||||
/// not a panic. Most consumers don't construct streams directly — call
|
||||
/// `mux::input(url, opts)` / `mux::output(url, title)` and let URL parsing
|
||||
/// pick the right type.
|
||||
///
|
||||
/// `info()` returns the stream's `DiscTitle` metadata (track list, codec
|
||||
/// info, duration). For sources it's parsed from the input; for sinks it's
|
||||
/// the metadata supplied at creation. Stable across all reads.
|
||||
///
|
||||
/// `codec_private(track)` exposes per-track initialization data
|
||||
/// (H.264 SPS/PPS, HEVC VPS/SPS/PPS, AC-3 fscod, etc.) that some output
|
||||
/// formats need before any frame can be written. `headers_ready()` returns
|
||||
/// false until enough input frames have been seen to populate every video
|
||||
/// track's codec-private blob — callers buffer frames they read until
|
||||
/// `headers_ready()` returns true.
|
||||
/// Deprecated; use [`FrameSource`] for read-only sources or [`FrameSink`]
|
||||
/// for write-only sinks. The runtime direction-error semantics
|
||||
/// (`StreamReadOnly` / `StreamWriteOnly` from a wrong-direction call) are
|
||||
/// removed in 0.18 — direction is type-checked.
|
||||
#[deprecated(
|
||||
since = "0.18.0",
|
||||
note = "use FrameSource (read-only) or FrameSink (write-only) instead"
|
||||
)]
|
||||
pub trait Stream {
|
||||
/// Read the next frame, or `Ok(None)` at end of stream. Returns
|
||||
/// `StreamWriteOnly` (E9001) on a write-only sink.
|
||||
/// Read the next frame, or `Ok(None)` at end of stream.
|
||||
fn read(&mut self) -> std::io::Result<Option<PesFrame>>;
|
||||
|
||||
/// Write a frame to the sink. Returns `StreamReadOnly` (E9000) on a
|
||||
/// read-only source.
|
||||
/// Write a frame to the sink.
|
||||
fn write(&mut self, frame: &PesFrame) -> std::io::Result<()>;
|
||||
|
||||
/// Finalize the stream: flush buffered frames, write any container
|
||||
@@ -140,6 +116,111 @@ pub trait Stream {
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-only source of PES frames.
|
||||
///
|
||||
/// Replaces the read half of the deprecated [`Stream`] trait. Implementors
|
||||
/// produce frames via [`read`](FrameSource::read) and never accept writes —
|
||||
/// passing a `FrameSource` where a sink is expected is a compile error,
|
||||
/// not a runtime `E9001`.
|
||||
///
|
||||
/// `info()` returns the source's `DiscTitle` metadata (track list, codec
|
||||
/// info, duration). It must be stable across the lifetime of the source.
|
||||
///
|
||||
/// `codec_private(track)` exposes per-track codec initialization data
|
||||
/// (H.264 SPS/PPS, HEVC VPS/SPS/PPS, AC-3 fscod, etc.) that downstream
|
||||
/// muxers may need before any frame is written. `headers_ready()` returns
|
||||
/// false until enough input frames have been seen to populate every video
|
||||
/// track's codec-private blob — callers buffer frames they read until
|
||||
/// `headers_ready()` returns true.
|
||||
pub trait FrameSource: Send {
|
||||
/// Read the next frame, or `Ok(None)` at end of stream.
|
||||
fn read(&mut self) -> std::io::Result<Option<PesFrame>>;
|
||||
|
||||
/// Source metadata. Stable across reads — implementors must return a
|
||||
/// consistent reference for the lifetime of the source.
|
||||
fn info(&self) -> &crate::disc::DiscTitle;
|
||||
|
||||
/// Codec initialization data for a track (SPS/PPS, AC-3 fscod, etc.).
|
||||
/// `None` for tracks that don't need codec_private (raw passthrough).
|
||||
fn codec_private(&self, _track: usize) -> Option<Vec<u8>> {
|
||||
None
|
||||
}
|
||||
|
||||
/// True when `codec_private` is available for every video track —
|
||||
/// callers buffer input frames until this flips, since some output
|
||||
/// formats (MKV) can't write frames without codec init data.
|
||||
fn headers_ready(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// Write-only sink of PES frames.
|
||||
///
|
||||
/// Replaces the write half of the deprecated [`Stream`] trait. Implementors
|
||||
/// accept frames via [`write`](FrameSink::write) and finalize via
|
||||
/// [`finish`](FrameSink::finish) — passing a `FrameSink` where a source is
|
||||
/// expected is a compile error, not a runtime `E9000`.
|
||||
///
|
||||
/// `finish` takes `Box<Self>` (rather than `&mut self` like `Stream::finish`)
|
||||
/// so that finalization is a one-shot terminal operation: callers cannot use
|
||||
/// the sink after `finish` returns. This is the standard idiom for terminal
|
||||
/// methods on `dyn Trait` objects.
|
||||
pub trait FrameSink: Send {
|
||||
/// Write a frame to the sink.
|
||||
fn write(&mut self, frame: &PesFrame) -> std::io::Result<()>;
|
||||
|
||||
/// Finalize the sink: flush buffered frames, write any container index
|
||||
/// (e.g. MKV `Cues`), close the underlying file/socket. Consumes the
|
||||
/// sink — callers cannot use it afterwards.
|
||||
fn finish(self: Box<Self>) -> std::io::Result<()>;
|
||||
|
||||
/// Sink metadata. Stable across writes — implementors must return a
|
||||
/// consistent reference for the lifetime of the sink.
|
||||
fn info(&self) -> &crate::disc::DiscTitle;
|
||||
}
|
||||
|
||||
// Bridge: any **`Send`** type implementing the deprecated `Stream` trait
|
||||
// is also a `FrameSource`. This lets existing concrete `Stream` impls in
|
||||
// `mux/*` satisfy `FrameSource` bounds without per-type migration during
|
||||
// the 0.18 deprecation window.
|
||||
//
|
||||
// **Send caveat (read me before tightening `Stream` itself).** This
|
||||
// blanket carries a `T: Send` bound rather than promoting `Send` to a
|
||||
// supertrait of `Stream`, because not every concrete in-tree `Stream`
|
||||
// impl is `Send`: `MkvStream` and `M2tsStream` carry `Box<dyn Read>`
|
||||
// and `Box<dyn Write>` fields whose trait objects don't include `Send`.
|
||||
// Adding `Stream: Send` would force a wider audit (every `Box<dyn Read>`
|
||||
// becomes `Box<dyn Read + Send>`) than this commit is taking on, and
|
||||
// the type-level migration target is `FrameSource` / `FrameSink`
|
||||
// directly anyway. Consequence: coercing a non-Send `Box<dyn Stream>`
|
||||
// (the return shape of `crate::mux::input` / `output`) to
|
||||
// `Box<dyn FrameSource>` will fail with a `T: Send` trait-bound error.
|
||||
// The fix on the consumer side is to construct a Send-compliant
|
||||
// `FrameSource` / `FrameSink` directly rather than relying on this
|
||||
// bridge for non-Send streams.
|
||||
//
|
||||
// Note: `FrameSink` cannot be blanket-impl'd from `Stream` because
|
||||
// `Stream::finish` takes `&mut self` while `FrameSink::finish` takes
|
||||
// `Box<Self>`; concrete types will be migrated in a follow-up commit.
|
||||
#[allow(deprecated)]
|
||||
impl<T: Stream + Send + ?Sized> FrameSource for T {
|
||||
fn read(&mut self) -> std::io::Result<Option<PesFrame>> {
|
||||
<Self as Stream>::read(self)
|
||||
}
|
||||
|
||||
fn info(&self) -> &crate::disc::DiscTitle {
|
||||
<Self as Stream>::info(self)
|
||||
}
|
||||
|
||||
fn codec_private(&self, track: usize) -> Option<Vec<u8>> {
|
||||
<Self as Stream>::codec_private(self, track)
|
||||
}
|
||||
|
||||
fn headers_ready(&self) -> bool {
|
||||
<Self as Stream>::headers_ready(self)
|
||||
}
|
||||
}
|
||||
|
||||
/// Wraps any output stream and counts bytes written.
|
||||
///
|
||||
/// Progress tracking is a CLI concern — streams don't know their size.
|
||||
@@ -152,11 +233,13 @@ pub trait Stream {
|
||||
/// let pct = output.bytes_written() as f64 / total as f64;
|
||||
/// }
|
||||
/// ```
|
||||
#[allow(deprecated)]
|
||||
pub struct CountingStream {
|
||||
inner: Box<dyn Stream>,
|
||||
written: u64,
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
impl CountingStream {
|
||||
pub fn new(inner: Box<dyn Stream>) -> Self {
|
||||
Self { inner, written: 0 }
|
||||
@@ -168,6 +251,7 @@ impl CountingStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
impl Stream for CountingStream {
|
||||
fn read(&mut self) -> std::io::Result<Option<PesFrame>> {
|
||||
self.inner.read()
|
||||
@@ -194,3 +278,178 @@ impl Stream for CountingStream {
|
||||
self.inner.headers_ready()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::disc::DiscTitle;
|
||||
|
||||
fn make_frame(track: usize, pts: i64) -> PesFrame {
|
||||
PesFrame {
|
||||
track,
|
||||
pts,
|
||||
keyframe: track == 0 && pts == 0,
|
||||
data: vec![track as u8, (pts & 0xff) as u8, 0xAA],
|
||||
}
|
||||
}
|
||||
|
||||
/// Direct `FrameSource` impl (not via the deprecated `Stream` blanket).
|
||||
struct MockSource {
|
||||
frames: std::vec::IntoIter<PesFrame>,
|
||||
title: DiscTitle,
|
||||
}
|
||||
|
||||
impl MockSource {
|
||||
fn new(frames: Vec<PesFrame>) -> Self {
|
||||
Self {
|
||||
frames: frames.into_iter(),
|
||||
title: DiscTitle::empty(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FrameSource for MockSource {
|
||||
fn read(&mut self) -> std::io::Result<Option<PesFrame>> {
|
||||
Ok(self.frames.next())
|
||||
}
|
||||
|
||||
fn info(&self) -> &DiscTitle {
|
||||
&self.title
|
||||
}
|
||||
}
|
||||
|
||||
/// Direct `FrameSink` impl (not via the deprecated `Stream` blanket).
|
||||
struct MockSink {
|
||||
collected: Vec<PesFrame>,
|
||||
title: DiscTitle,
|
||||
}
|
||||
|
||||
impl MockSink {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
collected: Vec::new(),
|
||||
title: DiscTitle::empty(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FrameSink for MockSink {
|
||||
fn write(&mut self, frame: &PesFrame) -> std::io::Result<()> {
|
||||
self.collected.push(frame.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> std::io::Result<()> {
|
||||
// Drop self; in real sinks this is where flush/fsync/close happens.
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn info(&self) -> &DiscTitle {
|
||||
&self.title
|
||||
}
|
||||
}
|
||||
|
||||
/// Variant of `MockSink` whose `finish` returns the collected frames so
|
||||
/// the test can assert on them after the consuming `Box<Self>` call.
|
||||
struct CollectingSink {
|
||||
collected: Vec<PesFrame>,
|
||||
title: DiscTitle,
|
||||
}
|
||||
|
||||
impl CollectingSink {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
collected: Vec::new(),
|
||||
title: DiscTitle::empty(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FrameSink for CollectingSink {
|
||||
fn write(&mut self, frame: &PesFrame) -> std::io::Result<()> {
|
||||
self.collected.push(frame.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> std::io::Result<()> {
|
||||
// Real CollectingSink consumers would expose `take()` before
|
||||
// finish; this trait method just confirms the boxed signature
|
||||
// compiles and runs.
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn info(&self) -> &DiscTitle {
|
||||
&self.title
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_source_yields_frames_then_eof() {
|
||||
let frames = vec![make_frame(0, 0), make_frame(1, 1_000), make_frame(0, 2_000)];
|
||||
let mut src = MockSource::new(frames.clone());
|
||||
|
||||
let f0 = src.read().unwrap().expect("first frame");
|
||||
assert_eq!(f0.track, frames[0].track);
|
||||
assert_eq!(f0.pts, frames[0].pts);
|
||||
assert!(f0.keyframe);
|
||||
|
||||
let f1 = src.read().unwrap().expect("second frame");
|
||||
assert_eq!(f1.track, frames[1].track);
|
||||
assert_eq!(f1.pts, frames[1].pts);
|
||||
|
||||
let f2 = src.read().unwrap().expect("third frame");
|
||||
assert_eq!(f2.track, frames[2].track);
|
||||
assert_eq!(f2.pts, frames[2].pts);
|
||||
|
||||
assert!(src.read().unwrap().is_none());
|
||||
assert!(src.read().unwrap().is_none()); // idempotent at EOF
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_sink_collects_then_finishes() {
|
||||
let mut sink = MockSink::new();
|
||||
let frames = [make_frame(0, 0), make_frame(1, 100), make_frame(2, 200)];
|
||||
|
||||
for f in &frames {
|
||||
sink.write(f).unwrap();
|
||||
}
|
||||
assert_eq!(sink.collected.len(), 3);
|
||||
assert_eq!(sink.collected[0].pts, 0);
|
||||
assert_eq!(sink.collected[1].pts, 100);
|
||||
assert_eq!(sink.collected[2].pts, 200);
|
||||
|
||||
// Box-and-finish — the `self: Box<Self>` shape must compile and run.
|
||||
Box::new(sink).finish().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_sink_via_dyn_object() {
|
||||
let frames = [make_frame(0, 0), make_frame(0, 33)];
|
||||
let mut sink: Box<dyn FrameSink> = Box::new(CollectingSink::new());
|
||||
|
||||
for f in &frames {
|
||||
sink.write(f).unwrap();
|
||||
}
|
||||
// info() routes through the trait object.
|
||||
let _ = sink.info();
|
||||
sink.finish().unwrap();
|
||||
}
|
||||
|
||||
/// The deprecated blanket impl: any concrete `Stream` should also act as
|
||||
/// a `FrameSource`. `NullStream` has the simplest constructor of every
|
||||
/// concrete `Stream` impl in `mux/*`, so it's the smallest credible
|
||||
/// witness that the bridge compiles and dispatches correctly.
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn deprecated_stream_satisfies_frame_source() {
|
||||
let title = DiscTitle::empty();
|
||||
let mut null = crate::mux::NullStream::new(&title);
|
||||
|
||||
let src: &mut dyn FrameSource = &mut null;
|
||||
// NullStream::read returns Ok(None) — it's a write-only sink.
|
||||
assert!(src.read().unwrap().is_none());
|
||||
// info() forwards through the blanket impl.
|
||||
let _ = src.info();
|
||||
assert!(src.headers_ready());
|
||||
}
|
||||
}
|
||||
|
||||
+66
-11
@@ -1,7 +1,7 @@
|
||||
//! Pipeline-progress reporting for the rip pipeline.
|
||||
//!
|
||||
//! v0.13.16 architecture rule: ONE progress signal type. Every long-running
|
||||
//! pipeline operation (`Disc::copy`, `Disc::patch`, mux) emits the same
|
||||
//! pipeline operation (`Disc::copy`, `Disc::patch`, `verify_title`) emits the same
|
||||
//! `PassProgress` shape via the `Progress` trait. Consumers (autorip) compute
|
||||
//! a single `PipelineStats` derived view and never reach into per-pass
|
||||
//! internals.
|
||||
@@ -28,6 +28,8 @@ pub enum PassKind {
|
||||
/// Demux ISO → output (MKV / M2TS / network). Single phase that runs
|
||||
/// after all rip passes complete.
|
||||
Mux,
|
||||
/// Sector verification — reads every sector and classifies health.
|
||||
Verify,
|
||||
}
|
||||
|
||||
/// One progress sample from a pipeline phase.
|
||||
@@ -36,35 +38,88 @@ pub enum PassKind {
|
||||
/// regardless of which kind of pass is running. `bytes_good_total` is the
|
||||
/// cumulative count of confirmed-clean bytes across the whole rip; useful
|
||||
/// for the "data recovered" stat the user sees.
|
||||
///
|
||||
/// For `PassKind::Verify`, the fields map as follows:
|
||||
/// - `work_done` = sectors read so far
|
||||
/// - `work_total` = total sectors in title
|
||||
/// - `bytes_good_total` = good + slow + recovered sectors × 2048
|
||||
/// - `bytes_unreadable_total` = bad sectors × 2048
|
||||
/// - `bytes_pending_total` = 0 (verify processes sequentially, nothing pending)
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct PassProgress {
|
||||
pub kind: PassKind,
|
||||
/// Bytes processed in this pass so far. Monotonically non-decreasing.
|
||||
pub work_done: u64,
|
||||
/// Total bytes this pass will process. Constant for the duration of
|
||||
/// the pass.
|
||||
pub work_total: u64,
|
||||
/// Cumulative bytes confirmed clean (`Finished` mapfile state) across
|
||||
/// every pass run on this rip. Doesn't change across pass boundaries.
|
||||
pub bytes_good_total: u64,
|
||||
/// Total disc capacity in bytes. Constant.
|
||||
pub bytes_unreadable_total: u64,
|
||||
pub bytes_pending_total: u64,
|
||||
pub bytes_total_disc: u64,
|
||||
pub disc_duration_secs: Option<f64>,
|
||||
/// How many bytes of the worst-case damage (unreadable + pending) fall
|
||||
/// within the main title's extents. Zero means none of the damage
|
||||
/// affects the main movie — it's all in extras/menus.
|
||||
pub bytes_bad_in_main_title: u64,
|
||||
/// Main title duration in seconds. Same as disc_duration_secs when the
|
||||
/// disc has one dominant title, but separate so consumers can show both.
|
||||
pub main_title_duration_secs: Option<f64>,
|
||||
/// Main title size in bytes (sum of extent sizes).
|
||||
pub main_title_size_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
impl PassProgress {
|
||||
/// Percentage of work completed for this pass (0..=100).
|
||||
///
|
||||
/// Returns `100.0` if `work_total` is zero to avoid division by zero.
|
||||
pub fn work_pct(&self) -> f64 {
|
||||
if self.work_total == 0 {
|
||||
return 100.0;
|
||||
}
|
||||
self.work_done as f64 / self.work_total as f64 * 100.0
|
||||
}
|
||||
|
||||
/// Percentage of the disc that is confirmed clean (0..=100).
|
||||
///
|
||||
/// Computed from `bytes_good_total / bytes_total_disc`.
|
||||
pub fn good_pct(&self) -> f64 {
|
||||
if self.bytes_total_disc == 0 {
|
||||
return 100.0;
|
||||
}
|
||||
self.bytes_good_total as f64 / self.bytes_total_disc as f64 * 100.0
|
||||
}
|
||||
|
||||
/// Percentage of the disc that is unreadable (0..=100).
|
||||
pub fn bad_pct(&self) -> f64 {
|
||||
if self.bytes_total_disc == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
self.bytes_unreadable_total as f64 / self.bytes_total_disc as f64 * 100.0
|
||||
}
|
||||
|
||||
/// Percentage of the disc that is still pending (not yet attempted or needs retry).
|
||||
pub fn pending_pct(&self) -> f64 {
|
||||
if self.bytes_total_disc == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
self.bytes_pending_total as f64 / self.bytes_total_disc as f64 * 100.0
|
||||
}
|
||||
}
|
||||
|
||||
/// A consumer of pipeline progress events. Library code calls
|
||||
/// `Progress::report` once per inner-loop iteration (throttling is the
|
||||
/// consumer's job — `report` is cheap; the library doesn't gate it).
|
||||
///
|
||||
/// Returns `true` to continue, `false` to request early stop.
|
||||
///
|
||||
/// No `Send`/`Sync` bound — `report` is always called from the same thread
|
||||
/// running the rip pipeline, so closures with non-`Sync` captures (e.g.
|
||||
/// running the pipeline, so closures with non-`Sync` captures (e.g.
|
||||
/// `RefCell<PassProgressState>`) work directly. Blanket impl below lets
|
||||
/// callers pass closures without explicit struct types.
|
||||
pub trait Progress {
|
||||
fn report(&self, p: &PassProgress);
|
||||
fn report(&self, p: &PassProgress) -> bool;
|
||||
}
|
||||
|
||||
impl<F: Fn(&PassProgress)> Progress for F {
|
||||
fn report(&self, p: &PassProgress) {
|
||||
impl<F: Fn(&PassProgress) -> bool> Progress for F {
|
||||
fn report(&self, p: &PassProgress) -> bool {
|
||||
(self)(p)
|
||||
}
|
||||
}
|
||||
|
||||
+5
-46
@@ -11,8 +11,8 @@
|
||||
//! This matches what every reference project does: MakeMKV (8 s sync
|
||||
//! ioctl), sg_dd (60 s sync ioctl), the kernel default for SCSI block
|
||||
//! devices (30 s `/sys/.../timeout`). See
|
||||
//! `(internal)/docs/audits/2026-04-26-scsi-architecture-research.md`
|
||||
//! for the full primary-source audit.
|
||||
//! the SCSI architecture audit (2026-04-26) for the full primary-source
|
||||
//! references.
|
||||
//!
|
||||
//! Pre-0.13.20 we ran an async `write() + poll(1.5s) + close-on-timeout +
|
||||
//! bg reopen` pattern. That abandoned slow-but-alive commands faster than
|
||||
@@ -64,9 +64,9 @@ const _: () = assert!(std::mem::size_of::<sg_io_hdr>() == 88);
|
||||
const _: () = assert!(std::mem::size_of::<sg_io_hdr>() == 64);
|
||||
|
||||
pub struct SgIoTransport {
|
||||
fd: i32,
|
||||
pub fd: i32,
|
||||
device_path: std::path::PathBuf,
|
||||
fd_recovery: std::sync::Arc<std::sync::atomic::AtomicI32>,
|
||||
pub fd_recovery: std::sync::Arc<std::sync::atomic::AtomicI32>,
|
||||
}
|
||||
|
||||
impl SgIoTransport {
|
||||
@@ -106,48 +106,7 @@ impl SgIoTransport {
|
||||
/// STOP+START UNIT. Both escalations were tried in 0.13.0–0.13.5
|
||||
/// against the LG BU40N (Initio USB-SATA bridge); both failed to
|
||||
/// recover wedged drives and made the wedge worse — see
|
||||
/// `(internal)/postmortems/2026-04-25-bu40n-wedge-recovery.md`.
|
||||
/// If the drive is genuinely unresponsive, the next workload command
|
||||
/// fails naturally and the caller surfaces a "physical reconnect
|
||||
/// required" prompt. Software has no path back from a wedged Initio
|
||||
/// bridge — only physical replug clears it.
|
||||
pub fn reset(device: &Path) -> Result<()> {
|
||||
let c_path = Self::to_c_path(device);
|
||||
|
||||
// open + close — make the kernel cancel any SG_IO commands queued
|
||||
// against a previous fd that didn't close cleanly.
|
||||
let probe_fd = unsafe {
|
||||
libc::open(
|
||||
c_path.as_ptr() as *const libc::c_char,
|
||||
libc::O_RDWR | libc::O_NONBLOCK | libc::O_CLOEXEC,
|
||||
)
|
||||
};
|
||||
if probe_fd >= 0 {
|
||||
unsafe { libc::close(probe_fd) };
|
||||
}
|
||||
|
||||
// Let the kernel finish that cancellation before we reopen.
|
||||
std::thread::sleep(std::time::Duration::from_secs(2));
|
||||
|
||||
// Fresh fd just to send the unlock command, then close.
|
||||
let fd = unsafe {
|
||||
libc::open(
|
||||
c_path.as_ptr() as *const libc::c_char,
|
||||
libc::O_RDWR | libc::O_NONBLOCK | libc::O_CLOEXEC,
|
||||
)
|
||||
};
|
||||
if fd < 0 {
|
||||
return Self::open_error(device);
|
||||
}
|
||||
|
||||
// ALLOW MEDIUM REMOVAL — clear any tray lock left by a killed
|
||||
// process whose Drop never ran. Best-effort; ignore result.
|
||||
let _ = Self::raw_command(fd, &[0x1E, 0, 0, 0, 0, 0], 3_000);
|
||||
|
||||
unsafe { libc::close(fd) };
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// the BU40N wedge recovery postmortem (2026-04-25).
|
||||
fn open_error<T>(device: &Path) -> Result<T> {
|
||||
let err = std::io::Error::last_os_error();
|
||||
Err(if err.kind() == std::io::ErrorKind::PermissionDenied {
|
||||
|
||||
+130
-510
@@ -1,183 +1,55 @@
|
||||
//! macOS SCSI transport via IOKit SCSITaskDeviceInterface.
|
||||
//! macOS SCSI transport: IOKit SCSITaskDeviceInterface with exclusive access.
|
||||
//!
|
||||
//! Sends SCSI commands to optical drives through IOKit's SCSI Architecture
|
||||
//! Model family. Accepts BSD device paths like `/dev/disk2` or `/dev/rdisk2`.
|
||||
//! Single dispatch path: **all** CDBs (INQUIRY, READ, REPORT KEY, etc.) go
|
||||
//! through `SCSITaskDeviceInterface::ExecuteTaskSync` — 1:1 with the Linux
|
||||
//! SG_IO backend. The C shim (`macos_shim.c`) handles:
|
||||
//!
|
||||
//! Requires exclusive access to the device — unmount the disc first:
|
||||
//! `diskutil unmountDisk /dev/disk2`
|
||||
//! 1. `diskutil unmountDisk force` on the target device only
|
||||
//! 2. Find `IOBDServices` matching the requested BSD name (walks IOKit
|
||||
//! registry: IOBDServices → IOBDBlockStorageDriver → IOMedia → BSD Name)
|
||||
//! 3. Create `MMCDeviceInterface` → `SCSITaskDeviceInterface`
|
||||
//! 4. `ObtainExclusiveAccess`
|
||||
//! 5. Raw CDB dispatch via `CreateSCSITask` + `ExecuteTaskSync`
|
||||
//!
|
||||
//! Drive enumeration (`list_drives`) uses the IOKit registry directly via
|
||||
//! `shim_list_drives` — no exclusive access, no SCSI commands, no unmounts.
|
||||
|
||||
use super::{DataDirection, ScsiResult, ScsiTransport};
|
||||
use crate::error::{Error, Result};
|
||||
use std::path::Path;
|
||||
|
||||
// ── IOKit / CoreFoundation type aliases ─────────────────────────────────────
|
||||
|
||||
type CFMutableDictionaryRef = *mut std::ffi::c_void;
|
||||
type IOObject = u32;
|
||||
type IOReturn = i32;
|
||||
type MachPort = u32;
|
||||
|
||||
/// Opaque COM interface pointer — `*mut *mut VTable` (double-indirect).
|
||||
/// IOKit plugins use COM-style vtables: the pointer points to a pointer
|
||||
/// to the function table.
|
||||
type ComRef = *mut *mut std::ffi::c_void;
|
||||
|
||||
const K_IO_RETURN_SUCCESS: IOReturn = 0;
|
||||
|
||||
// SCSI data transfer directions (SCSITaskLib.h)
|
||||
const K_SCSI_DATA_TRANSFER_NO_DATA: u8 = 0;
|
||||
const K_SCSI_DATA_TRANSFER_FROM_TARGET: u8 = 1;
|
||||
const K_SCSI_DATA_TRANSFER_TO_TARGET: u8 = 2;
|
||||
|
||||
// SCSI task status values
|
||||
const K_SCSI_TASK_STATUS_GOOD: u8 = 0x00;
|
||||
|
||||
const K_MAX_CDB_SIZE: usize = 16;
|
||||
const K_SENSE_DATA_SIZE: usize = 32;
|
||||
|
||||
// ── IOKit plugin UUIDs ──────────────────────────────────────────────────────
|
||||
// From IOKit/scsi/SCSITaskLib.h
|
||||
|
||||
/// kIOMMCDeviceUserClientTypeID — plugin type for MMC (optical) devices.
|
||||
const K_IO_MMC_DEVICE_USER_CLIENT_TYPE_ID: [u8; 16] = [
|
||||
0x97, 0xAB, 0xCF, 0x5C, 0x45, 0x71, 0x11, 0xD6, 0xB6, 0xA0, 0x00, 0x30, 0x65, 0xA4, 0x7A, 0xEE,
|
||||
];
|
||||
|
||||
/// kIOCFPlugInInterfaceID — base IOCFPlugin interface.
|
||||
const K_IO_CFPLUGIN_INTERFACE_ID: [u8; 16] = [
|
||||
0xC2, 0x44, 0xE8, 0x58, 0x10, 0x9C, 0x11, 0xD4, 0x91, 0xD4, 0x00, 0x50, 0xE4, 0xC6, 0x42, 0x6F,
|
||||
];
|
||||
|
||||
/// kIOSCSITaskDeviceInterfaceID — the interface we QueryInterface for.
|
||||
const K_IO_SCSI_TASK_DEVICE_INTERFACE_ID: [u8; 16] = [
|
||||
0x61, 0x3E, 0x48, 0xB0, 0x30, 0x01, 0x11, 0xD6, 0xA4, 0xC0, 0x00, 0x0A, 0x27, 0x05, 0x28, 0x61,
|
||||
];
|
||||
|
||||
// ── Scatter/gather element ──────────────────────────────────────────────────
|
||||
|
||||
#[repr(C)]
|
||||
struct SCSITaskSGElement {
|
||||
address: u64,
|
||||
length: u64,
|
||||
#[derive(Copy, Clone)]
|
||||
struct ShimDriveInfo {
|
||||
bsd_name: [u8; 32],
|
||||
vendor: [u8; 32],
|
||||
model: [u8; 48],
|
||||
firmware: [u8; 16],
|
||||
}
|
||||
|
||||
// ── External IOKit / CoreFoundation functions ───────────────────────────────
|
||||
|
||||
// Rust 2024: FFI blocks declaring extern fns must be `unsafe extern`.
|
||||
unsafe extern "C" {
|
||||
fn IOMasterPort(bootstrap: u32, master: *mut MachPort) -> IOReturn;
|
||||
fn IOBSDNameMatching(
|
||||
master: MachPort,
|
||||
options: u32,
|
||||
bsd_name: *const u8,
|
||||
) -> CFMutableDictionaryRef;
|
||||
fn IOServiceGetMatchingService(master: MachPort, matching: CFMutableDictionaryRef) -> IOObject;
|
||||
fn IOObjectRelease(object: IOObject) -> IOReturn;
|
||||
fn IORegistryEntryGetParentEntry(
|
||||
entry: IOObject,
|
||||
plane: *const u8,
|
||||
parent: *mut IOObject,
|
||||
) -> IOReturn;
|
||||
fn IOObjectConformsTo(object: IOObject, class_name: *const u8) -> u8;
|
||||
fn IOCreatePlugInInterfaceForService(
|
||||
service: IOObject,
|
||||
plugin_type: *const [u8; 16],
|
||||
interface_type: *const [u8; 16],
|
||||
the_interface: *mut ComRef,
|
||||
the_score: *mut i32,
|
||||
) -> IOReturn;
|
||||
fn shim_open_exclusive(bsd_name: *const u8) -> i32;
|
||||
fn shim_close();
|
||||
fn shim_execute(
|
||||
cdb: *const u8,
|
||||
cdb_len: u8,
|
||||
buf: *mut u8,
|
||||
buf_len: u32,
|
||||
data_in: i32,
|
||||
sense_out: *mut u8,
|
||||
sense_len: u32,
|
||||
task_status_out: *mut u8,
|
||||
transfer_count: *mut u64,
|
||||
) -> i32;
|
||||
fn shim_list_drives(out: *mut ShimDriveInfo, max_entries: i32) -> i32;
|
||||
}
|
||||
|
||||
// ── COM vtable helpers ──────────────────────────────────────────────────────
|
||||
//
|
||||
// IOKit plugin interfaces use COM-style vtables. A ComRef is **vtable —
|
||||
// dereferencing once gives the vtable pointer, then index into it for
|
||||
// individual function pointers.
|
||||
//
|
||||
// All vtable indices verified against Apple open source:
|
||||
// IOSCSIArchitectureModelFamily/UserClientLib/SCSITaskLib.h
|
||||
|
||||
/// Read a function pointer from a COM vtable at the given index.
|
||||
///
|
||||
/// # Safety
|
||||
/// `iface` must be a valid COM interface pointer (*mut *mut c_void), and
|
||||
/// `index` must be a valid vtable slot for the target type `T`.
|
||||
unsafe fn vtable_fn<T>(iface: ComRef, index: usize) -> T {
|
||||
// Rust 2024: `unsafe fn` bodies are no longer implicitly unsafe.
|
||||
// Each unsafe op needs its own `unsafe { }` block.
|
||||
unsafe {
|
||||
let vtable = *iface as *const *const std::ffi::c_void;
|
||||
let fn_ptr = *vtable.add(index);
|
||||
std::mem::transmute_copy(&fn_ptr)
|
||||
}
|
||||
}
|
||||
|
||||
/// Call Release (vtable index 3) on any COM interface.
|
||||
fn com_release(iface: ComRef) {
|
||||
type Fn = unsafe extern "C" fn(ComRef) -> u32;
|
||||
unsafe {
|
||||
let f: Fn = vtable_fn(iface, 3);
|
||||
f(iface);
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCSITaskDeviceInterface vtable ──────────────────────────────────────────
|
||||
//
|
||||
// Index Method
|
||||
// 0 _reserved
|
||||
// 1 QueryInterface
|
||||
// 2 AddRef
|
||||
// 3 Release
|
||||
// 4 IsExclusiveAccessAvailable
|
||||
// 5 AddCallbackDispatcherToRunLoop
|
||||
// 6 RemoveCallbackDispatcherFromRunLoop
|
||||
// 7 ObtainExclusiveAccess
|
||||
// 8 ReleaseExclusiveAccess
|
||||
// 9 CreateSCSITask
|
||||
|
||||
const VTIDX_OBTAIN_EXCLUSIVE: usize = 7;
|
||||
const VTIDX_RELEASE_EXCLUSIVE: usize = 8;
|
||||
const VTIDX_CREATE_TASK: usize = 9;
|
||||
|
||||
// ── SCSITaskInterface vtable ────────────────────────────────────────────────
|
||||
//
|
||||
// Index Method
|
||||
// 0 _reserved
|
||||
// 1 QueryInterface
|
||||
// 2 AddRef
|
||||
// 3 Release
|
||||
// 4 IsTaskActive
|
||||
// 5 SetTaskAttribute
|
||||
// 6 GetTaskAttribute
|
||||
// 7 GetTaskState
|
||||
// 8 SetCommandDescriptorBlock
|
||||
// 9 GetCommandDescriptorBlockSize
|
||||
// 10 GetCommandDescriptorBlock
|
||||
// 11 SetScatterGatherEntries
|
||||
// 12 SetTimeoutDuration
|
||||
// 13 GetTimeoutDuration
|
||||
// 14 SetTaskCompletionCallback
|
||||
// 15 ExecuteTaskSync
|
||||
// 16 ExecuteTaskAsync
|
||||
// 17 AbortTask
|
||||
// 18 GetSCSIServiceResponse
|
||||
// 19 GetTaskStatus
|
||||
// 20 GetRealizedDataTransferCount
|
||||
// 21 GetAutoSenseData
|
||||
|
||||
const VTIDX_SET_CDB: usize = 8;
|
||||
const VTIDX_SET_SG: usize = 11;
|
||||
const VTIDX_SET_TIMEOUT: usize = 12;
|
||||
const VTIDX_EXECUTE_SYNC: usize = 15;
|
||||
|
||||
// ── Transport implementation ────────────────────────────────────────────────
|
||||
|
||||
pub struct MacScsiTransport {
|
||||
device_iface: ComRef,
|
||||
exclusive: bool,
|
||||
_bsd_name: String,
|
||||
}
|
||||
|
||||
// IOKit COM interface pointers are Mach port references — safe to send between threads.
|
||||
unsafe impl Send for MacScsiTransport {}
|
||||
|
||||
impl MacScsiTransport {
|
||||
@@ -186,7 +58,6 @@ impl MacScsiTransport {
|
||||
path: device.display().to_string(),
|
||||
})?;
|
||||
|
||||
// Strip /dev/ prefix to get BSD name (e.g. "disk2")
|
||||
let bsd_name = if let Some(rest) = dev_str.strip_prefix("/dev/r") {
|
||||
rest
|
||||
} else if let Some(rest) = dev_str.strip_prefix("/dev/") {
|
||||
@@ -195,137 +66,118 @@ impl MacScsiTransport {
|
||||
dev_str
|
||||
};
|
||||
|
||||
let device_iface = Self::acquire_device_iface(bsd_name)?;
|
||||
let mut bsd_c = bsd_name.as_bytes().to_vec();
|
||||
bsd_c.push(0);
|
||||
|
||||
let rc = unsafe { shim_open_exclusive(bsd_c.as_ptr()) };
|
||||
if rc != 0 {
|
||||
return Err(Error::DeviceNotFound {
|
||||
path: bsd_name.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(MacScsiTransport {
|
||||
device_iface,
|
||||
exclusive: true,
|
||||
_bsd_name: bsd_name.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolve the BSD name → IOKit SCSITaskDeviceInterface with exclusive
|
||||
/// access. Returns the COM ref the caller must release.
|
||||
fn acquire_device_iface(bsd_name: &str) -> Result<ComRef> {
|
||||
let service = find_scsi_service(bsd_name)?;
|
||||
|
||||
// Create IOKit plugin for the MMC device
|
||||
let mut plugin: ComRef = std::ptr::null_mut();
|
||||
let mut score: i32 = 0;
|
||||
let kr = unsafe {
|
||||
IOCreatePlugInInterfaceForService(
|
||||
service,
|
||||
&K_IO_MMC_DEVICE_USER_CLIENT_TYPE_ID,
|
||||
&K_IO_CFPLUGIN_INTERFACE_ID,
|
||||
&mut plugin,
|
||||
&mut score,
|
||||
)
|
||||
};
|
||||
unsafe { IOObjectRelease(service) };
|
||||
|
||||
if kr != K_IO_RETURN_SUCCESS || plugin.is_null() {
|
||||
return Err(Error::IoKitPluginFailed {
|
||||
path: bsd_name.to_string(),
|
||||
kr: kr as u32,
|
||||
});
|
||||
}
|
||||
|
||||
// QueryInterface for SCSITaskDeviceInterface
|
||||
let mut device_iface: ComRef = std::ptr::null_mut();
|
||||
let hr = unsafe {
|
||||
type QiFn = unsafe extern "C" fn(ComRef, *const [u8; 16], *mut ComRef) -> i32;
|
||||
let qi: QiFn = vtable_fn(plugin, 1);
|
||||
qi(
|
||||
plugin,
|
||||
&K_IO_SCSI_TASK_DEVICE_INTERFACE_ID,
|
||||
&mut device_iface,
|
||||
)
|
||||
};
|
||||
com_release(plugin);
|
||||
|
||||
if hr != 0 || device_iface.is_null() {
|
||||
return Err(Error::ScsiInterfaceUnavailable {
|
||||
path: bsd_name.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Obtain exclusive access
|
||||
let kr = unsafe {
|
||||
type Fn = unsafe extern "C" fn(ComRef) -> IOReturn;
|
||||
let f: Fn = vtable_fn(device_iface, VTIDX_OBTAIN_EXCLUSIVE);
|
||||
f(device_iface)
|
||||
};
|
||||
if kr != K_IO_RETURN_SUCCESS {
|
||||
com_release(device_iface);
|
||||
return Err(Error::DeviceLocked {
|
||||
path: bsd_name.to_string(),
|
||||
kr: kr as u32,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(device_iface)
|
||||
}
|
||||
|
||||
// `reset()` removed in 0.13.6 — see scsi/mod.rs for rationale.
|
||||
// `try_recover()` removed in 0.13.20 — userspace handle-recovery on
|
||||
// task failure was the same anti-pattern stripped from Linux SG_IO
|
||||
// (see (internal)/docs/audits/2026-04-26-scsi-architecture-research.md).
|
||||
// Errors bubble up; caller decides whether to reopen the Drive.
|
||||
}
|
||||
|
||||
/// Enumerate optical drives on macOS. Mirrors `drive::macos::find_drives`
|
||||
/// (which iterates `/dev/disk0..15` + INQUIRY + filters peripheral
|
||||
/// type 5). Same logic, exposed through the new `DriveInfo` shape so
|
||||
/// callers — `list_drives()` in `scsi::mod` — never reach into
|
||||
/// `crate::drive::macos`.
|
||||
pub(super) fn list_drives() -> Vec<super::DriveInfo> {
|
||||
let mut out = Vec::new();
|
||||
for i in 0..K_DEV_DISK_MAX {
|
||||
let path = format!("/dev/disk{i}");
|
||||
if !std::path::Path::new(&path).exists() {
|
||||
continue;
|
||||
impl Drop for MacScsiTransport {
|
||||
fn drop(&mut self) {
|
||||
unsafe { shim_close() };
|
||||
}
|
||||
}
|
||||
|
||||
impl ScsiTransport for MacScsiTransport {
|
||||
fn execute(
|
||||
&mut self,
|
||||
cdb: &[u8],
|
||||
direction: DataDirection,
|
||||
data: &mut [u8],
|
||||
_timeout_ms: u32,
|
||||
) -> Result<ScsiResult> {
|
||||
let data_in = match direction {
|
||||
DataDirection::FromDevice => 1,
|
||||
DataDirection::ToDevice => 0,
|
||||
DataDirection::None => 0,
|
||||
};
|
||||
|
||||
let mut sense = [0u8; K_SENSE_DATA_SIZE];
|
||||
let mut task_status: u8 = 0xFF;
|
||||
let mut transfer_count: u64 = 0;
|
||||
|
||||
let kr = unsafe {
|
||||
shim_execute(
|
||||
cdb.as_ptr(),
|
||||
cdb.len() as u8,
|
||||
data.as_mut_ptr(),
|
||||
data.len() as u32,
|
||||
data_in,
|
||||
sense.as_mut_ptr(),
|
||||
K_SENSE_DATA_SIZE as u32,
|
||||
&mut task_status,
|
||||
&mut transfer_count,
|
||||
)
|
||||
};
|
||||
|
||||
if kr != 0 {
|
||||
return Err(Error::ScsiError {
|
||||
opcode: cdb[0],
|
||||
status: super::SCSI_STATUS_TRANSPORT_FAILURE,
|
||||
sense: None,
|
||||
});
|
||||
}
|
||||
let mut transport = match MacScsiTransport::open(std::path::Path::new(&path)) {
|
||||
Ok(t) => t,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let inquiry = match super::inquiry(&mut transport) {
|
||||
Ok(r) => r,
|
||||
Err(_) => continue,
|
||||
};
|
||||
// SCSI peripheral type field is the lower 5 bits of byte 0.
|
||||
if inquiry.raw.is_empty()
|
||||
|| (inquiry.raw[K_INQUIRY_TYPE_BYTE] & K_INQUIRY_TYPE_MASK) != K_SCSI_TYPE_OPTICAL
|
||||
{
|
||||
|
||||
if task_status != 0 {
|
||||
let parsed = super::parse_sense(&sense, K_SENSE_DATA_SIZE as u8);
|
||||
return Err(Error::ScsiError {
|
||||
opcode: cdb[0],
|
||||
status: task_status,
|
||||
sense: Some(parsed),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(ScsiResult {
|
||||
status: 0,
|
||||
bytes_transferred: transfer_count as usize,
|
||||
sense,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── Drive enumeration (registry-based, no exclusive access) ──────────────
|
||||
|
||||
pub(super) fn list_drives() -> Vec<super::DriveInfo> {
|
||||
let mut buf = [ShimDriveInfo {
|
||||
bsd_name: [0; 32],
|
||||
vendor: [0; 32],
|
||||
model: [0; 48],
|
||||
firmware: [0; 16],
|
||||
}; 8];
|
||||
|
||||
let count = unsafe { shim_list_drives(buf.as_mut_ptr(), buf.len() as i32) };
|
||||
|
||||
let mut out = Vec::new();
|
||||
for info in buf.iter().take((count as usize).min(buf.len())) {
|
||||
let bsd_name = cstr_to_str(&info.bsd_name);
|
||||
if bsd_name.is_empty() {
|
||||
continue;
|
||||
}
|
||||
out.push(super::DriveInfo {
|
||||
path,
|
||||
vendor: inquiry.vendor_id,
|
||||
model: inquiry.model,
|
||||
firmware: inquiry.firmware,
|
||||
path: format!("/dev/{bsd_name}"),
|
||||
vendor: cstr_to_str(&info.vendor).to_string(),
|
||||
model: cstr_to_str(&info.model).to_string(),
|
||||
firmware: cstr_to_str(&info.firmware).to_string(),
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Maximum BSD disk index probed during enumeration. macOS assigns
|
||||
/// `/dev/diskN` sequentially per attached storage device; 16 covers
|
||||
/// any realistic homelab.
|
||||
const K_DEV_DISK_MAX: u8 = 16;
|
||||
fn cstr_to_str(bytes: &[u8]) -> &str {
|
||||
let end = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len());
|
||||
std::str::from_utf8(&bytes[..end]).unwrap_or("")
|
||||
}
|
||||
|
||||
/// SCSI INQUIRY response: peripheral device type lives in byte 0,
|
||||
/// lower 5 bits.
|
||||
const K_INQUIRY_TYPE_BYTE: usize = 0;
|
||||
const K_INQUIRY_TYPE_MASK: u8 = 0x1F;
|
||||
|
||||
/// SCSI peripheral type 5 = "CD-ROM device" (covers DVD, BD-ROM, BD-RE).
|
||||
const K_SCSI_TYPE_OPTICAL: u8 = 0x05;
|
||||
|
||||
/// TEST UNIT READY probe on macOS. Any non-"not ready" error bubbles up
|
||||
/// to the caller — in-library wedge recovery was rolled back in 0.13.4
|
||||
/// after USB-layer resets failed to recover the LG BU40N on Linux; the
|
||||
/// macOS impl mirrors that choice for symmetry. See the Linux
|
||||
/// `drive_has_disc` in `scsi/linux.rs` for the full rationale.
|
||||
pub(super) fn drive_has_disc(path: &Path) -> Result<bool> {
|
||||
let mut transport = MacScsiTransport::open(path)?;
|
||||
let cdb = [crate::scsi::SCSI_TEST_UNIT_READY, 0, 0, 0, 0, 0];
|
||||
@@ -341,235 +193,3 @@ pub(super) fn drive_has_disc(path: &Path) -> Result<bool> {
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MacScsiTransport {
|
||||
fn drop(&mut self) {
|
||||
if self.device_iface.is_null() {
|
||||
return;
|
||||
}
|
||||
if self.exclusive {
|
||||
unsafe {
|
||||
type Fn = unsafe extern "C" fn(ComRef) -> IOReturn;
|
||||
let f: Fn = vtable_fn(self.device_iface, VTIDX_RELEASE_EXCLUSIVE);
|
||||
f(self.device_iface);
|
||||
}
|
||||
}
|
||||
com_release(self.device_iface);
|
||||
}
|
||||
}
|
||||
|
||||
impl ScsiTransport for MacScsiTransport {
|
||||
fn execute(
|
||||
&mut self,
|
||||
cdb: &[u8],
|
||||
direction: DataDirection,
|
||||
data: &mut [u8],
|
||||
timeout_ms: u32,
|
||||
) -> Result<ScsiResult> {
|
||||
// Create a SCSI task
|
||||
let task: ComRef = unsafe {
|
||||
type Fn = unsafe extern "C" fn(ComRef) -> ComRef;
|
||||
let f: Fn = vtable_fn(self.device_iface, VTIDX_CREATE_TASK);
|
||||
f(self.device_iface)
|
||||
};
|
||||
if task.is_null() {
|
||||
return Err(Error::ScsiError {
|
||||
opcode: cdb[0],
|
||||
status: super::SCSI_STATUS_TRANSPORT_FAILURE,
|
||||
sense: None,
|
||||
});
|
||||
}
|
||||
|
||||
// Set CDB
|
||||
let mut cdb_padded = [0u8; K_MAX_CDB_SIZE];
|
||||
let cdb_len = cdb.len().min(K_MAX_CDB_SIZE);
|
||||
cdb_padded[..cdb_len].copy_from_slice(&cdb[..cdb_len]);
|
||||
unsafe {
|
||||
type Fn = unsafe extern "C" fn(ComRef, *const u8, u8) -> IOReturn;
|
||||
let f: Fn = vtable_fn(task, VTIDX_SET_CDB);
|
||||
f(task, cdb_padded.as_ptr(), cdb_len as u8);
|
||||
}
|
||||
|
||||
// Set scatter/gather and transfer direction
|
||||
let iokit_dir = match direction {
|
||||
DataDirection::None => K_SCSI_DATA_TRANSFER_NO_DATA,
|
||||
DataDirection::FromDevice => K_SCSI_DATA_TRANSFER_FROM_TARGET,
|
||||
DataDirection::ToDevice => K_SCSI_DATA_TRANSFER_TO_TARGET,
|
||||
};
|
||||
|
||||
if direction != DataDirection::None && !data.is_empty() {
|
||||
let sg = SCSITaskSGElement {
|
||||
address: data.as_mut_ptr() as u64,
|
||||
length: data.len() as u64,
|
||||
};
|
||||
unsafe {
|
||||
type Fn =
|
||||
unsafe extern "C" fn(ComRef, *const SCSITaskSGElement, u8, u64, u8) -> IOReturn;
|
||||
let f: Fn = vtable_fn(task, VTIDX_SET_SG);
|
||||
f(task, &sg, 1, data.len() as u64, iokit_dir);
|
||||
}
|
||||
} else {
|
||||
unsafe {
|
||||
type Fn =
|
||||
unsafe extern "C" fn(ComRef, *const SCSITaskSGElement, u8, u64, u8) -> IOReturn;
|
||||
let f: Fn = vtable_fn(task, VTIDX_SET_SG);
|
||||
f(task, std::ptr::null(), 0, 0, K_SCSI_DATA_TRANSFER_NO_DATA);
|
||||
}
|
||||
}
|
||||
|
||||
// Set timeout (IOKit SCSITask takes milliseconds)
|
||||
unsafe {
|
||||
type Fn = unsafe extern "C" fn(ComRef, u32);
|
||||
let f: Fn = vtable_fn(task, VTIDX_SET_TIMEOUT);
|
||||
f(task, timeout_ms);
|
||||
}
|
||||
|
||||
// Execute synchronously
|
||||
let mut sense = [0u8; K_SENSE_DATA_SIZE];
|
||||
let mut task_status: u32 = 0;
|
||||
let mut realized_count: u64 = 0;
|
||||
|
||||
let kr = unsafe {
|
||||
type Fn = unsafe extern "C" fn(ComRef, *mut u8, *mut u32, *mut u64) -> IOReturn;
|
||||
let f: Fn = vtable_fn(task, VTIDX_EXECUTE_SYNC);
|
||||
f(
|
||||
task,
|
||||
sense.as_mut_ptr(),
|
||||
&mut task_status,
|
||||
&mut realized_count,
|
||||
)
|
||||
};
|
||||
|
||||
com_release(task);
|
||||
|
||||
if kr != K_IO_RETURN_SUCCESS {
|
||||
// Task-level failure (timeout / IOKit error). Bubble it up;
|
||||
// the kernel mid-layer has already done what it can.
|
||||
return Err(Error::ScsiError {
|
||||
opcode: cdb[0],
|
||||
status: super::SCSI_STATUS_TRANSPORT_FAILURE,
|
||||
sense: None,
|
||||
});
|
||||
}
|
||||
|
||||
if task_status != K_SCSI_TASK_STATUS_GOOD as u32 {
|
||||
// IOKit doesn't surface a "bytes written into sense buffer"
|
||||
// count the way SG_IO does — pass the buffer's full length
|
||||
// and let parse_sense inspect byte 0's response code to pick
|
||||
// descriptor-vs-fixed format.
|
||||
//
|
||||
// 0.13.23: carry the full SPC-4 sense triple in
|
||||
// `Error::ScsiError::sense` so callers can route on
|
||||
// `ScsiSense::is_medium_error()` etc.
|
||||
let parsed = super::parse_sense(&sense, sense.len() as u8);
|
||||
return Err(Error::ScsiError {
|
||||
opcode: cdb[0],
|
||||
status: task_status as u8,
|
||||
sense: Some(parsed),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(ScsiResult {
|
||||
status: task_status as u8,
|
||||
bytes_transferred: realized_count as usize,
|
||||
sense,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── IOKit service discovery ─────────────────────────────────────────────────
|
||||
|
||||
/// BSD name → IOKit service for the SCSI device.
|
||||
///
|
||||
/// Walk: IOMedia (BSD name match) → parent chain → SCSIPeripheralDeviceNub.
|
||||
///
|
||||
/// All failure paths surface as `Error::DeviceNotFound { path: bsd_name }` —
|
||||
/// the four internal stages (IOMasterPort / IOBSDNameMatching / IOMedia
|
||||
/// lookup / walk_to_authoring_device) collapse into one observable error
|
||||
/// because none of them are user-actionable individually. Pre-0.13 each
|
||||
/// stage stuffed an English description into `path:` ("…IOMasterPort
|
||||
/// failed", "…SCSITaskDeviceInterface not available", etc.) which broke
|
||||
/// the library's "no English text" rule.
|
||||
fn find_scsi_service(bsd_name: &str) -> Result<IOObject> {
|
||||
let not_found = || Error::DeviceNotFound {
|
||||
path: bsd_name.to_string(),
|
||||
};
|
||||
|
||||
let mut master: MachPort = 0;
|
||||
let kr = unsafe { IOMasterPort(0, &mut master) };
|
||||
if kr != K_IO_RETURN_SUCCESS {
|
||||
return Err(not_found());
|
||||
}
|
||||
|
||||
// IOBSDNameMatching creates a dictionary matching { "BSD Name" = bsd_name }
|
||||
let mut bsd_c = bsd_name.as_bytes().to_vec();
|
||||
bsd_c.push(0);
|
||||
let matching = unsafe { IOBSDNameMatching(master, 0, bsd_c.as_ptr()) };
|
||||
if matching.is_null() {
|
||||
return Err(not_found());
|
||||
}
|
||||
|
||||
// Find the single IOMedia service (consumes the matching dict)
|
||||
let media = unsafe { IOServiceGetMatchingService(master, matching) };
|
||||
if media == 0 {
|
||||
return Err(not_found());
|
||||
}
|
||||
|
||||
// Walk up the IOService plane to find the authoring device.
|
||||
// The chain is typically:
|
||||
// IOMedia → IOPartitionScheme → IOMedia → IOBlockStorageDriver
|
||||
// → IOSCSIPeripheralDeviceNub (this is what we want)
|
||||
//
|
||||
// We walk up until we find a service that IOCreatePlugInInterfaceForService
|
||||
// accepts with kIOMMCDeviceUserClientTypeID, or until we hit the root.
|
||||
let service = walk_to_authoring_device(media);
|
||||
unsafe { IOObjectRelease(media) };
|
||||
|
||||
service.ok_or_else(not_found)
|
||||
}
|
||||
|
||||
/// Walk up the IOService plane from an IOMedia to the SCSI authoring device.
|
||||
fn walk_to_authoring_device(start: IOObject) -> Option<IOObject> {
|
||||
let mut current = start;
|
||||
// Retain start so we can release uniformly in the loop
|
||||
// (IORegistryEntryGetParentEntry retains the parent for us)
|
||||
|
||||
// Target class names for authoring devices
|
||||
let target_classes: &[&[u8]] = &[
|
||||
b"IOSCSIPeripheralDeviceNub\0",
|
||||
b"IOBDBlockStorageDevice\0",
|
||||
b"IODVDBlockStorageDevice\0",
|
||||
b"IOCDBlockStorageDevice\0",
|
||||
b"IOBlockStorageDevice\0",
|
||||
];
|
||||
|
||||
// Walk up to 10 levels (more than enough)
|
||||
for _ in 0..10 {
|
||||
let mut parent: IOObject = 0;
|
||||
let kr = unsafe {
|
||||
IORegistryEntryGetParentEntry(current, c"IOService".as_ptr() as *const u8, &mut parent)
|
||||
};
|
||||
|
||||
if current != start {
|
||||
unsafe { IOObjectRelease(current) };
|
||||
}
|
||||
|
||||
if kr != K_IO_RETURN_SUCCESS || parent == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Check if this parent matches any of our target classes
|
||||
for class in target_classes {
|
||||
if unsafe { IOObjectConformsTo(parent, class.as_ptr()) } != 0 {
|
||||
return Some(parent);
|
||||
}
|
||||
}
|
||||
|
||||
current = parent;
|
||||
}
|
||||
|
||||
if current != start {
|
||||
unsafe { IOObjectRelease(current) };
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
@@ -0,0 +1,387 @@
|
||||
#include <IOKit/IOKitLib.h>
|
||||
#include <IOKit/IOCFPlugIn.h>
|
||||
#include <IOKit/scsi/SCSITaskLib.h>
|
||||
#include <CoreFoundation/CoreFoundation.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
typedef struct {
|
||||
IOCFPlugInInterface **plugin;
|
||||
MMCDeviceInterface **mmc;
|
||||
SCSITaskDeviceInterface **scsi;
|
||||
int exclusive;
|
||||
} ShimHandle;
|
||||
|
||||
typedef struct {
|
||||
char bsd_name[32];
|
||||
char vendor[32];
|
||||
char model[48];
|
||||
char firmware[16];
|
||||
} ShimDriveInfo;
|
||||
|
||||
// ── Global handle (single-drive, same as before) ──────────────────────────
|
||||
|
||||
static ShimHandle g_handle = {NULL, NULL, NULL, 0};
|
||||
|
||||
// ── Registry helpers ──────────────────────────────────────────────────────
|
||||
|
||||
static int cfstring_to_cstr(CFStringRef cf, char *buf, size_t buflen) {
|
||||
if (!cf) return 0;
|
||||
if (!CFStringGetCString(cf, buf, buflen, kCFStringEncodingUTF8)) return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int registry_entry_bsd_name(io_registry_entry_t entry, char *buf, size_t buflen) {
|
||||
CFStringRef cf = IORegistryEntryCreateCFProperty(entry, CFSTR("BSD Name"),
|
||||
kCFAllocatorDefault, 0);
|
||||
if (!cf) return 0;
|
||||
int ok = cfstring_to_cstr(cf, buf, buflen);
|
||||
CFRelease(cf);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static io_registry_entry_t find_iomedia_child(io_registry_entry_t parent) {
|
||||
io_iterator_t iter;
|
||||
kern_return_t kr = IORegistryEntryGetChildIterator(parent, kIOServicePlane, &iter);
|
||||
if (kr != KERN_SUCCESS) return 0;
|
||||
|
||||
io_registry_entry_t child;
|
||||
while ((child = IOIteratorNext(iter)) != 0) {
|
||||
char cls[128];
|
||||
kr = IOObjectGetClass(child, cls);
|
||||
if (kr == KERN_SUCCESS) {
|
||||
if (strcmp(cls, "IOMedia") == 0 || strcmp(cls, "IOBDMedia") == 0) {
|
||||
IOObjectRelease(iter);
|
||||
return child;
|
||||
}
|
||||
}
|
||||
IOObjectRelease(child);
|
||||
}
|
||||
IOObjectRelease(iter);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static io_registry_entry_t find_child_of_class(io_registry_entry_t parent, const char *target_class) {
|
||||
io_iterator_t iter;
|
||||
kern_return_t kr = IORegistryEntryGetChildIterator(parent, kIOServicePlane, &iter);
|
||||
if (kr != KERN_SUCCESS) return 0;
|
||||
|
||||
io_registry_entry_t child;
|
||||
while ((child = IOIteratorNext(iter)) != 0) {
|
||||
char cls[128];
|
||||
kr = IOObjectGetClass(child, cls);
|
||||
if (kr == KERN_SUCCESS && strcmp(cls, target_class) == 0) {
|
||||
IOObjectRelease(iter);
|
||||
return child;
|
||||
}
|
||||
IOObjectRelease(child);
|
||||
}
|
||||
IOObjectRelease(iter);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static io_registry_entry_t find_parent_of_class(io_registry_entry_t entry, const char *target_class) {
|
||||
io_registry_entry_t parent;
|
||||
kern_return_t kr = IORegistryEntryGetParentEntry(entry, kIOServicePlane, &parent);
|
||||
if (kr != KERN_SUCCESS) return 0;
|
||||
|
||||
char cls[128];
|
||||
kr = IOObjectGetClass(parent, cls);
|
||||
if (kr == KERN_SUCCESS && strcmp(cls, target_class) == 0) {
|
||||
return parent;
|
||||
}
|
||||
IOObjectRelease(parent);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Given an IOBDServices, find the BSD name of its IOMedia child.
|
||||
// Chain: IOBDServices -> IOBDBlockStorageDriver -> IOMedia (has "BSD Name")
|
||||
static int bdsvc_to_bsd_name(io_registry_entry_t bdsvc, char *buf, size_t buflen) {
|
||||
io_registry_entry_t driver = find_child_of_class(bdsvc, "IOBDBlockStorageDriver");
|
||||
if (!driver) return 0;
|
||||
|
||||
io_registry_entry_t media = find_iomedia_child(driver);
|
||||
IOObjectRelease(driver);
|
||||
if (!media) return 0;
|
||||
|
||||
int ok = registry_entry_bsd_name(media, buf, buflen);
|
||||
IOObjectRelease(media);
|
||||
return ok;
|
||||
}
|
||||
|
||||
// Given an IOBDServices, extract Device Characteristics strings.
|
||||
static void bdsvc_device_info(io_registry_entry_t bdsvc, ShimDriveInfo *info) {
|
||||
CFDictionaryRef dc = IORegistryEntryCreateCFProperty(bdsvc,
|
||||
CFSTR("Device Characteristics"), kCFAllocatorDefault, 0);
|
||||
if (!dc) return;
|
||||
|
||||
CFStringRef val;
|
||||
|
||||
val = CFDictionaryGetValue(dc, CFSTR("Vendor Name"));
|
||||
if (val) cfstring_to_cstr(val, info->vendor, sizeof(info->vendor));
|
||||
|
||||
val = CFDictionaryGetValue(dc, CFSTR("Product Name"));
|
||||
if (val) cfstring_to_cstr(val, info->model, sizeof(info->model));
|
||||
|
||||
val = CFDictionaryGetValue(dc, CFSTR("Product Revision Level"));
|
||||
if (val) cfstring_to_cstr(val, info->firmware, sizeof(info->firmware));
|
||||
|
||||
CFRelease(dc);
|
||||
}
|
||||
|
||||
// Find the IOBDServices that owns the given BSD name.
|
||||
// Returns a retained io_service_t (caller must release), or 0.
|
||||
static io_service_t find_bdsvc_by_bsd_name(mach_port_t mp, const char *bsd_name) {
|
||||
CFMutableDictionaryRef matching = IOServiceMatching("IOBDServices");
|
||||
if (!matching) return 0;
|
||||
|
||||
io_iterator_t iter;
|
||||
kern_return_t kr = IOServiceGetMatchingServices(mp, matching, &iter);
|
||||
if (kr != KERN_SUCCESS) return 0;
|
||||
|
||||
io_service_t result = 0;
|
||||
io_service_t svc;
|
||||
while ((svc = IOIteratorNext(iter)) != 0) {
|
||||
char name[64];
|
||||
if (bdsvc_to_bsd_name(svc, name, sizeof(name))) {
|
||||
if (strcmp(name, bsd_name) == 0) {
|
||||
result = svc;
|
||||
break;
|
||||
}
|
||||
}
|
||||
IOObjectRelease(svc);
|
||||
}
|
||||
|
||||
if (!result) {
|
||||
IOIteratorReset(iter);
|
||||
while ((svc = IOIteratorNext(iter)) != 0) {
|
||||
IOObjectRelease(svc);
|
||||
}
|
||||
}
|
||||
|
||||
IOObjectRelease(iter);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Find the IOBDServices that owns the given BSD name by walking from
|
||||
// IOMedia upward. Used as fallback when bdsvc_to_bsd_name fails
|
||||
// (e.g. disc under exclusive access, no IOMedia child).
|
||||
// Chain: IOMedia -> IOBDBlockStorageDriver -> IOBDServices
|
||||
static io_service_t find_bdsvc_from_iomedia(mach_port_t mp, const char *bsd_name) {
|
||||
CFMutableDictionaryRef matching = IOServiceMatching("IOMedia");
|
||||
if (!matching) return 0;
|
||||
|
||||
io_iterator_t iter;
|
||||
kern_return_t kr = IOServiceGetMatchingServices(mp, matching, &iter);
|
||||
if (kr != KERN_SUCCESS) return 0;
|
||||
|
||||
io_service_t result = 0;
|
||||
io_service_t media;
|
||||
while ((media = IOIteratorNext(iter)) != 0) {
|
||||
char name[64];
|
||||
if (registry_entry_bsd_name(media, name, sizeof(name))
|
||||
&& strcmp(name, bsd_name) == 0)
|
||||
{
|
||||
io_registry_entry_t driver = find_parent_of_class(media, "IOBDBlockStorageDriver");
|
||||
if (driver) {
|
||||
io_registry_entry_t bdsvc = find_parent_of_class(driver, "IOBDServices");
|
||||
IOObjectRelease(driver);
|
||||
if (bdsvc) {
|
||||
result = bdsvc;
|
||||
IOObjectRelease(media);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
IOObjectRelease(media);
|
||||
}
|
||||
|
||||
IOObjectRelease(iter);
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Public API ────────────────────────────────────────────────────────────
|
||||
|
||||
int shim_open_exclusive(const char *bsd_name) {
|
||||
kern_return_t kr;
|
||||
HRESULT hr;
|
||||
SInt32 score = 0;
|
||||
|
||||
if (g_handle.exclusive && g_handle.scsi) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Use a shell wrapper so the device path is not subject to buffer limits.
|
||||
// snprintf into 128 bytes could truncate long BSD names (e.g. disk12s3s1),
|
||||
// producing a broken command. sh -c with $1 passes the arg via argv.
|
||||
const char *shell_fmt = "sh -c 'diskutil unmountDisk force \"$1\" >/dev/null 2>&1' _ %s";
|
||||
char cmd[512];
|
||||
int written = snprintf(cmd, sizeof(cmd), shell_fmt, bsd_name);
|
||||
if (written < 0 || (size_t)written >= sizeof(cmd)) {
|
||||
return -1;
|
||||
}
|
||||
system(cmd);
|
||||
usleep(500000);
|
||||
|
||||
mach_port_t mp;
|
||||
IOMainPort(0, &mp);
|
||||
|
||||
io_service_t svc = find_bdsvc_by_bsd_name(mp, bsd_name);
|
||||
if (!svc) {
|
||||
svc = find_bdsvc_from_iomedia(mp, bsd_name);
|
||||
}
|
||||
if (!svc) {
|
||||
CFMutableDictionaryRef matching = IOServiceMatching("IOBDServices");
|
||||
svc = IOServiceGetMatchingService(mp, matching);
|
||||
}
|
||||
if (!svc) return -1;
|
||||
|
||||
kr = IOCreatePlugInInterfaceForService(svc,
|
||||
kIOMMCDeviceUserClientTypeID, kIOCFPlugInInterfaceID,
|
||||
&g_handle.plugin, &score);
|
||||
IOObjectRelease(svc);
|
||||
|
||||
if (kr != KERN_SUCCESS || !g_handle.plugin) return -2;
|
||||
|
||||
hr = (*g_handle.plugin)->QueryInterface(g_handle.plugin,
|
||||
CFUUIDGetUUIDBytes(kIOMMCDeviceInterfaceID), (LPVOID *)&g_handle.mmc);
|
||||
if (hr != S_OK || !g_handle.mmc) {
|
||||
IODestroyPlugInInterface(g_handle.plugin);
|
||||
g_handle.plugin = NULL;
|
||||
return -3;
|
||||
}
|
||||
|
||||
g_handle.scsi = (*g_handle.mmc)->GetSCSITaskDeviceInterface(g_handle.mmc);
|
||||
if (!g_handle.scsi) {
|
||||
(*g_handle.mmc)->Release(g_handle.mmc);
|
||||
IODestroyPlugInInterface(g_handle.plugin);
|
||||
g_handle.mmc = NULL;
|
||||
g_handle.plugin = NULL;
|
||||
return -4;
|
||||
}
|
||||
|
||||
for (int retry = 0; retry < 10; retry++) {
|
||||
kr = (*g_handle.scsi)->ObtainExclusiveAccess(g_handle.scsi);
|
||||
if (kr == kIOReturnSuccess) break;
|
||||
usleep(500000);
|
||||
}
|
||||
if (kr != kIOReturnSuccess) {
|
||||
(*g_handle.scsi)->Release(g_handle.scsi);
|
||||
(*g_handle.mmc)->Release(g_handle.mmc);
|
||||
IODestroyPlugInInterface(g_handle.plugin);
|
||||
g_handle.scsi = NULL;
|
||||
g_handle.mmc = NULL;
|
||||
g_handle.plugin = NULL;
|
||||
return -5;
|
||||
}
|
||||
|
||||
g_handle.exclusive = 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void shim_close(void) {
|
||||
if (g_handle.exclusive && g_handle.scsi) {
|
||||
(*g_handle.scsi)->ReleaseExclusiveAccess(g_handle.scsi);
|
||||
}
|
||||
if (g_handle.scsi) {
|
||||
(*g_handle.scsi)->Release(g_handle.scsi);
|
||||
g_handle.scsi = NULL;
|
||||
}
|
||||
if (g_handle.mmc) {
|
||||
(*g_handle.mmc)->Release(g_handle.mmc);
|
||||
g_handle.mmc = NULL;
|
||||
}
|
||||
if (g_handle.plugin) {
|
||||
IODestroyPlugInInterface(g_handle.plugin);
|
||||
g_handle.plugin = NULL;
|
||||
}
|
||||
g_handle.exclusive = 0;
|
||||
}
|
||||
|
||||
int shim_execute(const unsigned char *cdb, unsigned char cdb_len,
|
||||
void *buf, unsigned int buf_len, int data_in,
|
||||
unsigned char *sense_out, unsigned int sense_len,
|
||||
unsigned char *task_status_out, unsigned long long *transfer_count) {
|
||||
if (!g_handle.scsi) return -1;
|
||||
|
||||
SCSITaskInterface **task = (*g_handle.scsi)->CreateSCSITask(g_handle.scsi);
|
||||
if (!task) return -2;
|
||||
|
||||
SCSICommandDescriptorBlock cdb_buf;
|
||||
memset(&cdb_buf, 0, sizeof(cdb_buf));
|
||||
memcpy(&cdb_buf, cdb, cdb_len);
|
||||
|
||||
(*task)->SetCommandDescriptorBlock(task, cdb_buf, cdb_len);
|
||||
|
||||
if (buf_len > 0 && buf) {
|
||||
SCSITaskSGElement sg;
|
||||
sg.address = (UInt64)(uintptr_t)buf;
|
||||
sg.length = buf_len;
|
||||
(*task)->SetScatterGatherEntries(task, &sg, 1, buf_len,
|
||||
data_in ? kSCSIDataTransfer_FromTargetToInitiator
|
||||
: kSCSIDataTransfer_FromInitiatorToTarget);
|
||||
} else {
|
||||
(*task)->SetScatterGatherEntries(task, NULL, 0, 0,
|
||||
kSCSIDataTransfer_NoDataTransfer);
|
||||
}
|
||||
|
||||
(*task)->SetTimeoutDuration(task, 30000);
|
||||
|
||||
SCSI_Sense_Data sense;
|
||||
memset(&sense, 0, sizeof(sense));
|
||||
SCSITaskStatus status = 0xFF;
|
||||
UInt64 count = 0;
|
||||
|
||||
IOReturn kr = (*task)->ExecuteTaskSync(task, &sense, &status, &count);
|
||||
|
||||
if (sense_out && sense_len > 0) {
|
||||
size_t copy = sense_len < sizeof(sense) ? sense_len : sizeof(sense);
|
||||
memcpy(sense_out, &sense, copy);
|
||||
}
|
||||
if (task_status_out) *task_status_out = (unsigned char)status;
|
||||
if (transfer_count) *transfer_count = count;
|
||||
|
||||
(*task)->Release(task);
|
||||
|
||||
return (int)kr;
|
||||
}
|
||||
|
||||
// ── Registry-based drive enumeration ──────────────────────────────────────
|
||||
//
|
||||
// Walks IOBDServices entries in the IOKit registry. No exclusive access,
|
||||
// no SCSI commands, no unmounts. Returns up to max_entries drives.
|
||||
|
||||
int shim_list_drives(ShimDriveInfo *out, int max_entries) {
|
||||
mach_port_t mp;
|
||||
IOReturn ret = IOMainPort(0, &mp);
|
||||
if (ret != kIOReturnSuccess) return 0;
|
||||
|
||||
CFMutableDictionaryRef matching = IOServiceMatching("IOBDServices");
|
||||
if (!matching) return 0;
|
||||
|
||||
io_iterator_t iter;
|
||||
kern_return_t kr = IOServiceGetMatchingServices(mp, matching, &iter);
|
||||
if (kr != KERN_SUCCESS) return 0;
|
||||
|
||||
int count = 0;
|
||||
io_service_t svc;
|
||||
while ((svc = IOIteratorNext(iter)) != 0 && count < max_entries) {
|
||||
ShimDriveInfo *info = &out[count];
|
||||
memset(info, 0, sizeof(*info));
|
||||
|
||||
bdsvc_device_info(svc, info);
|
||||
bdsvc_to_bsd_name(svc, info->bsd_name, sizeof(info->bsd_name));
|
||||
|
||||
if (info->bsd_name[0]) {
|
||||
count++;
|
||||
}
|
||||
|
||||
IOObjectRelease(svc);
|
||||
}
|
||||
|
||||
IOObjectRelease(iter);
|
||||
return count;
|
||||
}
|
||||
+28
-11
@@ -2,11 +2,11 @@
|
||||
//!
|
||||
//! Platform backends are in separate files:
|
||||
//! - `linux.rs` — SG_IO ioctl
|
||||
//! - `macos.rs` — IOKit SCSITaskDeviceInterface
|
||||
//! - `macos.rs` — IOKit SCSITaskDeviceInterface (exclusive access)
|
||||
//! - `windows.rs` — SPTI (SCSI Pass-Through Interface)
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
pub mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(target_os = "windows")]
|
||||
@@ -56,9 +56,8 @@ pub(crate) const TUR_TIMEOUT_MS: u32 = 5_000;
|
||||
///
|
||||
/// 10 s catches every legitimate slow read with comfortable margin and
|
||||
/// short-circuits truly bad sectors at ~10 s rather than letting the
|
||||
/// kernel mid-layer escalate for 30 s+. See run log in
|
||||
/// `(internal)/docs/TEST_PLAN.md` and the audit at
|
||||
/// `(internal)/docs/audits/2026-04-26-scsi-architecture-research.md`.
|
||||
/// kernel mid-layer escalate for 30 s+. See the SCSI architecture audit
|
||||
/// (2026-04-26) for primary-source references.
|
||||
///
|
||||
/// Pre-0.13.21 this was 1.5 s, which forced the kernel mid-layer to
|
||||
/// time out *normal* reads (cold-start often takes ~1.5 s) and run its
|
||||
@@ -69,10 +68,25 @@ pub(crate) const TUR_TIMEOUT_MS: u32 = 5_000;
|
||||
pub(crate) const READ_TIMEOUT_MS: u32 = 10_000;
|
||||
|
||||
/// Timeout for content READ commands on the recovery path —
|
||||
/// [`disc::Disc::patch`]'s targeted retries on bad ranges. Doubles
|
||||
/// the fast-path budget so a sector that fails at 30 s gets one more
|
||||
/// honest attempt. Matches sg_dd's default per-command timeout
|
||||
/// (`DEF_TIMEOUT = 60000`).
|
||||
/// [`disc::Disc::patch`]'s targeted retries on bad ranges. Matches
|
||||
/// `sg_dd`'s 60 s ceiling: long enough that any sector the drive can
|
||||
/// recover at all gets the time to do so, short enough that an
|
||||
/// unresponsive bus is detected before the per-range watchdog fires.
|
||||
///
|
||||
/// In practice failed reads return in 1–4 s (the drive itself gives up
|
||||
/// on uncorrectable ECC before the timeout); the 60 s value is a
|
||||
/// safety ceiling, not a steady-state cost.
|
||||
///
|
||||
/// Historical note (2026-05-08): briefly lowered to 2 s with a 5×
|
||||
/// inline retry loop in `Disc::patch` to mimic the kernel `sr_mod`
|
||||
/// driver's auto-retry pattern. The synthetic logic worked but on the
|
||||
/// live drive each "2 s" read paid ~1.5 s of kernel SCSI mid-layer
|
||||
/// error escalation on top, so 5× retries took ~17 s per LBA and
|
||||
/// triggered MAX_RANGE_SECS after 4 sectors — pushing recovery to
|
||||
/// 0/22 ranges (worse than the 0/22 baseline of v0.17.3 single-shot
|
||||
/// at 60 s, since that at least visited every range). Reverted; the
|
||||
/// kernel-auto-retry approach is being pursued via a `/dev/sr0` pread
|
||||
/// fallback instead.
|
||||
pub(crate) const READ_RECOVERY_TIMEOUT_MS: u32 = 60_000;
|
||||
|
||||
// ── SCSI status bytes (SPC-4 §4.5.5) ────────────────────────────────────────
|
||||
@@ -146,12 +160,14 @@ impl ScsiSense {
|
||||
/// (or a brief retry) sometimes succeeds:
|
||||
///
|
||||
/// - `MEDIUM ERROR` (3) — canonical bad-sector signal
|
||||
/// - `NOT READY` (2) — on many drives (notably BU40N), this is the
|
||||
/// dominant response for unreadable sectors (ASC 04/3E, 04/01, etc.)
|
||||
/// - `ABORTED COMMAND` (B) — transient; retry usually works
|
||||
/// - `RECOVERED ERROR` (1) / `NO SENSE` (0) — drive is healthy and
|
||||
/// either recovered the data or has no specific fault to report
|
||||
///
|
||||
/// `false` for HARDWARE ERROR, DATA PROTECT, UNIT ATTENTION, NOT
|
||||
/// READY, ILLEGAL REQUEST, BLANK CHECK, and any unknown key. Used
|
||||
/// `false` for HARDWARE ERROR, DATA PROTECT, UNIT ATTENTION,
|
||||
/// ILLEGAL REQUEST, BLANK CHECK, and any unknown key. Used
|
||||
/// by [`Error::is_marginal_read`] / `Disc::copy`'s hysteresis
|
||||
/// dispatch.
|
||||
pub fn is_marginal(&self) -> bool {
|
||||
@@ -159,6 +175,7 @@ impl ScsiSense {
|
||||
self.sense_key,
|
||||
SENSE_KEY_NO_SENSE
|
||||
| SENSE_KEY_RECOVERED_ERROR
|
||||
| SENSE_KEY_NOT_READY
|
||||
| SENSE_KEY_MEDIUM_ERROR
|
||||
| SENSE_KEY_ABORTED_COMMAND
|
||||
)
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
//! SectorReader — trait for reading 2048-byte disc sectors.
|
||||
//!
|
||||
//! Implemented by Drive (SCSI) and IsoFile (file-backed).
|
||||
//! Used by UDF parser, disc scanner, label parsers — anything that
|
||||
//! reads sectors doesn't need to know where they come from.
|
||||
|
||||
use crate::error::Result;
|
||||
|
||||
/// Read 2048-byte sectors from a disc or disc image.
|
||||
pub trait SectorReader: Send {
|
||||
/// Read `count` sectors starting at `lba` into `buf`.
|
||||
/// `buf` must be at least `count * 2048` bytes.
|
||||
/// `recovery`: true = full retry/reset loop (ripping), false = single attempt (verify).
|
||||
/// File-backed readers ignore the flag.
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize>;
|
||||
|
||||
/// Total capacity in sectors, if known.
|
||||
fn capacity(&self) -> u32 {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/// SectorReader backed by a file (ISO image).
|
||||
/// Seeks to lba * 2048, reads count * 2048 bytes.
|
||||
pub struct FileSectorReader {
|
||||
file: std::io::BufReader<std::fs::File>,
|
||||
capacity: u32,
|
||||
}
|
||||
|
||||
impl FileSectorReader {
|
||||
pub fn open(path: &str) -> std::io::Result<Self> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
let len = file.metadata()?.len();
|
||||
let sectors = len / 2048;
|
||||
if sectors > u32::MAX as u64 {
|
||||
// ~8 TB hard cap (u32::MAX × 2048 bytes). Path lives in the
|
||||
// typed Error variant — no English in the message.
|
||||
return Err(crate::error::Error::IsoTooLarge {
|
||||
path: path.to_string(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let capacity = sectors as u32;
|
||||
Ok(Self {
|
||||
file: std::io::BufReader::with_capacity(4 * 1024 * 1024, file),
|
||||
capacity,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for FileSectorReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
let offset = lba as u64 * 2048;
|
||||
let bytes = count as usize * 2048;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| crate::error::Error::IoError { source: e })?;
|
||||
self.file
|
||||
.read_exact(&mut buf[..bytes])
|
||||
.map_err(|e| crate::error::Error::IoError { source: e })?;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn capacity(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
//! `DecryptingSectorSource` — wrap any [`SectorSource`] to apply
|
||||
//! AACS / CSS in-place decryption on every read.
|
||||
//!
|
||||
//! This is the 0.18 single-source-of-truth for decrypt-on-read. The
|
||||
//! actual cipher code lives in [`crate::aacs`] and [`crate::css`];
|
||||
//! we just call the existing [`crate::decrypt::decrypt_sectors`]
|
||||
//! helper that already drives both of them. In follow-up commits
|
||||
//! `sweep_pipeline` and `DiscStream` migrate onto this decorator
|
||||
//! and delete their duplicate decrypt call sites.
|
||||
//!
|
||||
//! Composition: `Drive` → `DecryptingSectorSource` → caller sees
|
||||
//! plaintext. For `DecryptKeys::None` discs the decorator is a
|
||||
//! pass-through, so callers can wire it unconditionally and keep
|
||||
//! their pipeline shape uniform regardless of encryption state.
|
||||
|
||||
use crate::decrypt::{DecryptKeys, decrypt_sectors};
|
||||
use crate::error::Result;
|
||||
|
||||
use super::SectorSource;
|
||||
|
||||
/// Decorator: read from `inner`, then run the configured
|
||||
/// AACS / CSS decrypt over the bytes that landed in `buf`.
|
||||
///
|
||||
/// `unit_key_idx` selects the AACS unit key for the disc (0 for
|
||||
/// the vast majority of titles; the rare multi-CPS-unit discs pick
|
||||
/// the index that covers the title being read). For
|
||||
/// [`DecryptKeys::None`] and [`DecryptKeys::Css`] the index is
|
||||
/// ignored.
|
||||
pub struct DecryptingSectorSource<S: SectorSource> {
|
||||
inner: S,
|
||||
keys: DecryptKeys,
|
||||
unit_key_idx: usize,
|
||||
}
|
||||
|
||||
impl<S: SectorSource> DecryptingSectorSource<S> {
|
||||
/// Wrap `inner` with the given keys. The default unit-key
|
||||
/// index is 0; use [`with_unit_key_idx`] for the multi-CPS-unit
|
||||
/// case.
|
||||
///
|
||||
/// [`with_unit_key_idx`]: Self::with_unit_key_idx
|
||||
pub fn new(inner: S, keys: DecryptKeys) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
keys,
|
||||
unit_key_idx: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Override the AACS unit-key index. Only meaningful for
|
||||
/// [`DecryptKeys::Aacs`]; other variants ignore it.
|
||||
pub fn with_unit_key_idx(mut self, idx: usize) -> Self {
|
||||
self.unit_key_idx = idx;
|
||||
self
|
||||
}
|
||||
|
||||
/// Replace the configured keys without unwrapping the decorator.
|
||||
/// Used by `DiscStream::set_raw()` to flip from encrypted-disc
|
||||
/// decryption to a pass-through after the inner reader is already
|
||||
/// owned by the wrapper. For new construction prefer [`new`].
|
||||
///
|
||||
/// [`new`]: Self::new
|
||||
pub fn set_keys(&mut self, keys: DecryptKeys) {
|
||||
self.keys = keys;
|
||||
}
|
||||
|
||||
/// Borrow the inner source. Useful for tests and for adapters
|
||||
/// that want to introspect the underlying drive / file without
|
||||
/// unwrapping the decorator.
|
||||
pub fn inner(&self) -> &S {
|
||||
&self.inner
|
||||
}
|
||||
|
||||
/// Mutable borrow of the inner source.
|
||||
pub fn inner_mut(&mut self) -> &mut S {
|
||||
&mut self.inner
|
||||
}
|
||||
|
||||
/// Consume the decorator and return the underlying source.
|
||||
pub fn into_inner(self) -> S {
|
||||
self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: SectorSource> SectorSource for DecryptingSectorSource<S> {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.inner.capacity_sectors()
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize> {
|
||||
let n = self.inner.read_sectors(lba, count, buf, recovery)?;
|
||||
// Reuse the existing crate-wide decrypt entry point — same
|
||||
// path the 0.17 sweep_pipeline and DiscStream call, so we
|
||||
// inherit their AACS / CSS / None semantics verbatim. The
|
||||
// helper is a no-op for DecryptKeys::None.
|
||||
decrypt_sectors(&mut buf[..n], &self.keys, self.unit_key_idx)?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
self.inner.set_speed(kbs)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::error::Result;
|
||||
|
||||
/// Synthetic SectorSource that yields a deterministic byte
|
||||
/// pattern keyed by LBA. Used to verify the decorator's
|
||||
/// pass-through behaviour for `DecryptKeys::None`.
|
||||
struct PatternedSource {
|
||||
capacity: u32,
|
||||
}
|
||||
|
||||
impl PatternedSource {
|
||||
fn fill(lba: u32, count: u16, buf: &mut [u8]) {
|
||||
let bytes = count as usize * 2048;
|
||||
for (i, slot) in buf[..bytes].iter_mut().enumerate() {
|
||||
let abs = lba as u64 * 2048 + i as u64;
|
||||
*slot = ((abs.wrapping_mul(2654435761) >> 16) & 0xff) as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for PatternedSource {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
Self::fill(lba, count, buf);
|
||||
Ok(count as usize * 2048)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passthrough_with_no_keys() {
|
||||
let src = PatternedSource { capacity: 16 };
|
||||
let mut wrapped = DecryptingSectorSource::new(src, DecryptKeys::None);
|
||||
|
||||
// capacity_sectors delegates.
|
||||
assert_eq!(wrapped.capacity_sectors(), 16);
|
||||
|
||||
let mut got = vec![0u8; 4 * 2048];
|
||||
let n = wrapped.read_sectors(3, 4, &mut got, false).unwrap();
|
||||
assert_eq!(n, 4 * 2048);
|
||||
|
||||
let mut expected = vec![0u8; 4 * 2048];
|
||||
PatternedSource::fill(3, 4, &mut expected);
|
||||
assert_eq!(got, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passthrough_set_speed_delegates() {
|
||||
struct SpeedRecorder {
|
||||
last: Option<u16>,
|
||||
}
|
||||
impl SectorSource for SpeedRecorder {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
0
|
||||
}
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
_lba: u32,
|
||||
_count: u16,
|
||||
_buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
Ok(0)
|
||||
}
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
self.last = Some(kbs);
|
||||
}
|
||||
}
|
||||
|
||||
let mut wrapped =
|
||||
DecryptingSectorSource::new(SpeedRecorder { last: None }, DecryptKeys::None);
|
||||
wrapped.set_speed(7200);
|
||||
assert_eq!(wrapped.inner().last, Some(7200));
|
||||
}
|
||||
|
||||
// TODO: AACS round-trip test — needs a fixture-encrypted unit
|
||||
// (6144-byte aligned) plus the matching unit key. The cipher
|
||||
// path itself is exercised by `crate::aacs` unit tests; here
|
||||
// we only assert the decorator wires the existing helper, not
|
||||
// that AES-128 is correct.
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
//! File-backed sector I/O — read and write 2048-byte sectors against
|
||||
//! an ISO image on disk.
|
||||
//!
|
||||
//! [`FileSectorSource`] is the read side (open-only). [`FileSectorSink`]
|
||||
//! is the write side (create or open-rw); writes go through
|
||||
//! [`crate::io::WritebackFile`] so big sequential ISO writes share
|
||||
//! the same bounded-cache writeback pipeline used by sweep / patch /
|
||||
//! mux.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
use super::{SectorReader, SectorSink};
|
||||
|
||||
/// SectorSource backed by a file (ISO image).
|
||||
///
|
||||
/// Seeks to `lba * 2048`, reads `count * 2048` bytes per call. The
|
||||
/// file is held directly: every `read_sectors` call performs an
|
||||
/// absolute seek, so a wrapping `BufReader` would have its buffer
|
||||
/// invalidated on every call (its internal cursor moves with the
|
||||
/// `Seek` impl) — pure overhead. Callers that benefit from buffered
|
||||
/// reads should compose their own `BufReader` at the `read_sectors`
|
||||
/// granularity they care about.
|
||||
pub struct FileSectorSource {
|
||||
file: File,
|
||||
capacity: u32,
|
||||
}
|
||||
|
||||
impl FileSectorSource {
|
||||
/// Open an existing ISO file for reading. Capacity is derived
|
||||
/// from `metadata().len() / 2048`. Returns
|
||||
/// [`Error::IsoTooLarge`] if the file would exceed the 32-bit
|
||||
/// LBA address space (~8 TB).
|
||||
pub fn open(path: &Path) -> std::io::Result<Self> {
|
||||
let file = File::open(path)?;
|
||||
let len = file.metadata()?.len();
|
||||
let sectors = len / 2048;
|
||||
if sectors > u32::MAX as u64 {
|
||||
return Err(Error::IsoTooLarge {
|
||||
path: path.to_string_lossy().into_owned(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let capacity = sectors as u32;
|
||||
Ok(Self { file, capacity })
|
||||
}
|
||||
}
|
||||
|
||||
// Implement the legacy `SectorReader` trait. The blanket impl in
|
||||
// `super` produces the `SectorSource` impl automatically — no need
|
||||
// to write both, and writing both would conflict. This keeps the
|
||||
// 0.17 method-resolution path intact (callers with `SectorReader`
|
||||
// in scope can still write `fsr.read_sectors(..)` against a
|
||||
// `FileSectorSource`).
|
||||
impl SectorReader for FileSectorSource {
|
||||
fn capacity(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
let offset = lba as u64 * 2048;
|
||||
let bytes = count as usize * 2048;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.read_exact(&mut buf[..bytes])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
/// SectorSink backed by a file (ISO image).
|
||||
///
|
||||
/// Writes go through [`crate::io::WritebackFile`], which on Linux drives
|
||||
/// continuous `sync_file_range` + `posix_fadvise(DONTNEED)` to keep
|
||||
/// the kernel dirty page cache bounded during multi-GB sequential
|
||||
/// writes. macOS / Windows fall through to a no-op pipeline.
|
||||
///
|
||||
/// `finish` runs `sync_all` before dropping the underlying file.
|
||||
pub struct FileSectorSink {
|
||||
inner: crate::io::WritebackFile,
|
||||
}
|
||||
|
||||
impl FileSectorSink {
|
||||
/// Create a new ISO file at `path`, truncating any existing
|
||||
/// file. The file is opened read-write so the same handle can
|
||||
/// later be reused for verification reads if needed (sweep
|
||||
/// doesn't, but it costs nothing here).
|
||||
pub fn create(path: &Path) -> std::io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
Ok(Self {
|
||||
inner: crate::io::WritebackFile::new(file)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Open an existing ISO file for in-place updates (e.g. patch
|
||||
/// pass writing recovered sectors over zero-filled holes).
|
||||
/// Does not truncate.
|
||||
pub fn open(path: &Path) -> std::io::Result<Self> {
|
||||
let file = OpenOptions::new().read(true).write(true).open(path)?;
|
||||
Ok(Self {
|
||||
inner: crate::io::WritebackFile::new(file)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSink for FileSectorSink {
|
||||
fn write_sectors(&mut self, lba: u32, buf: &[u8]) -> Result<()> {
|
||||
debug_assert!(
|
||||
buf.len() % 2048 == 0,
|
||||
"FileSectorSink::write_sectors: buf len {} not a multiple of 2048",
|
||||
buf.len()
|
||||
);
|
||||
let offset = lba as u64 * 2048;
|
||||
self.inner
|
||||
.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.inner
|
||||
.write_all(buf)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(mut self: Box<Self>) -> Result<()> {
|
||||
self.inner
|
||||
.sync_all()
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
// Bring the 0.18 trait into scope (not super::*: the super
|
||||
// module also re-exports the legacy `SectorReader`, and
|
||||
// having both `SectorReader::read_sectors` and
|
||||
// `SectorSource::read_sectors` visible would force every
|
||||
// call site to disambiguate). External consumers see the
|
||||
// same surface this test exercises.
|
||||
use super::{FileSectorSink, FileSectorSource};
|
||||
use crate::sector::{SectorSink, SectorSource};
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn round_trip_single_sector() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("rt.iso");
|
||||
|
||||
let mut sink = FileSectorSink::create(&path).unwrap();
|
||||
// Pre-extend the file to 4 sectors of zeros so we can write
|
||||
// sector 2 in place. Easiest way: write zeros first.
|
||||
let zeros = [0u8; 4 * 2048];
|
||||
sink.write_sectors(0, &zeros).unwrap();
|
||||
|
||||
let mut payload = [0u8; 2048];
|
||||
for (i, b) in payload.iter_mut().enumerate() {
|
||||
*b = (i as u8).wrapping_mul(17);
|
||||
}
|
||||
sink.write_sectors(2, &payload).unwrap();
|
||||
Box::new(sink).finish().unwrap();
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), 4);
|
||||
|
||||
let mut got = [0u8; 2048];
|
||||
let n = src.read_sectors(2, 1, &mut got, false).unwrap();
|
||||
assert_eq!(n, 2048);
|
||||
assert_eq!(got, payload);
|
||||
|
||||
// Sectors 0,1,3 still zero.
|
||||
let mut z = [0xffu8; 2048];
|
||||
src.read_sectors(0, 1, &mut z, false).unwrap();
|
||||
assert!(z.iter().all(|b| *b == 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trip_multi_sector() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("multi.iso");
|
||||
|
||||
let mut sink = FileSectorSink::create(&path).unwrap();
|
||||
let mut payload = vec![0u8; 8 * 2048];
|
||||
for (i, b) in payload.iter_mut().enumerate() {
|
||||
*b = ((i * 31) ^ (i >> 7)) as u8;
|
||||
}
|
||||
sink.write_sectors(0, &payload).unwrap();
|
||||
Box::new(sink).finish().unwrap();
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), 8);
|
||||
|
||||
let mut got = vec![0u8; 8 * 2048];
|
||||
let n = src.read_sectors(0, 8, &mut got, false).unwrap();
|
||||
assert_eq!(n, 8 * 2048);
|
||||
assert_eq!(got, payload);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_existing_does_not_truncate() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("open.iso");
|
||||
|
||||
// Create with 4 sectors of pattern A.
|
||||
let mut sink = FileSectorSink::create(&path).unwrap();
|
||||
let pat_a = [0xaau8; 4 * 2048];
|
||||
sink.write_sectors(0, &pat_a).unwrap();
|
||||
Box::new(sink).finish().unwrap();
|
||||
|
||||
// Reopen and overwrite sector 1 only.
|
||||
let mut sink = FileSectorSink::open(&path).unwrap();
|
||||
let pat_b = [0xbbu8; 2048];
|
||||
sink.write_sectors(1, &pat_b).unwrap();
|
||||
Box::new(sink).finish().unwrap();
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), 4);
|
||||
let mut got = [0u8; 2048];
|
||||
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
assert_eq!(got, [0xaau8; 2048]);
|
||||
|
||||
src.read_sectors(1, 1, &mut got, false).unwrap();
|
||||
assert_eq!(got, [0xbbu8; 2048]);
|
||||
|
||||
src.read_sectors(2, 1, &mut got, false).unwrap();
|
||||
assert_eq!(got, [0xaau8; 2048]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
//! Sector-level I/O traits.
|
||||
//!
|
||||
//! 0.18 splits the unidirectional read trait from a write trait at
|
||||
//! the sector layer, so the type system catches "wrong direction"
|
||||
//! mistakes at compile time instead of runtime. See
|
||||
//! `(internal)/memory/0_18_redesign.md`.
|
||||
//!
|
||||
//! - [`SectorSource`] reads 2048-byte sectors. Implemented by
|
||||
//! `Drive` (via the legacy [`SectorReader`] alias) and
|
||||
//! [`FileSectorSource`] (ISO-backed).
|
||||
//! - [`SectorSink`] writes 2048-byte sectors. Implemented by
|
||||
//! [`FileSectorSink`] (ISO-backed) and, in later commits, by
|
||||
//! sweep/patch consumer adapters.
|
||||
//! - [`DecryptingSectorSource`] is a decorator that wraps any
|
||||
//! `SectorSource` and applies the existing AACS / CSS in-place
|
||||
//! decrypt to plaintext-out.
|
||||
//!
|
||||
//! [`SectorReader`] is the 0.17 read trait. It stays on through
|
||||
//! the 0.18 migration window so existing call sites
|
||||
//! (`Drive`, `IsoSectorReader`, `BufferedSectorReader`,
|
||||
//! `DiscStream`, `verify`) compile unchanged. A blanket impl
|
||||
//! forwards every `SectorReader` impl to `SectorSource`, so new
|
||||
//! code should target `SectorSource` / `SectorSink` directly. The
|
||||
//! formal `#[deprecated]` attribute lands once the internal
|
||||
//! callers have migrated; see the comment on `SectorReader` for
|
||||
//! why this commit holds it back.
|
||||
|
||||
pub mod decrypting;
|
||||
pub mod file;
|
||||
|
||||
use crate::error::Result;
|
||||
|
||||
/// Read 2048-byte sectors from a disc, image, or composed source.
|
||||
///
|
||||
/// Direction-typed: a `SectorSource` cannot be written to. Wrap the
|
||||
/// inner source in [`DecryptingSectorSource`] to get plaintext
|
||||
/// sectors out of an encrypted disc.
|
||||
pub trait SectorSource: Send {
|
||||
/// Total capacity in sectors, if known. Returns 0 when unknown
|
||||
/// (e.g. live drives that haven't completed `READ CAPACITY` yet).
|
||||
fn capacity_sectors(&self) -> u32;
|
||||
|
||||
/// Read `count` sectors starting at `lba` into `buf`.
|
||||
/// `buf` must be at least `count * 2048` bytes.
|
||||
/// `recovery`: true = full retry/reset loop (ripping),
|
||||
/// false = single attempt (verify). File-backed sources ignore
|
||||
/// the flag.
|
||||
///
|
||||
/// Returns the number of bytes written into `buf` on success.
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize>;
|
||||
|
||||
/// Optional speed control for sources that map to a physical
|
||||
/// drive. No-op for everything else.
|
||||
fn set_speed(&mut self, _kbs: u16) {}
|
||||
}
|
||||
|
||||
/// Write 2048-byte sectors to a disc image or composed sink.
|
||||
///
|
||||
/// Direction-typed: a `SectorSink` cannot be read from. The
|
||||
/// terminal [`finish`] takes `Box<Self>` so it can run on `dyn
|
||||
/// SectorSink` and consume the sink (`fsync` + close).
|
||||
///
|
||||
/// [`finish`]: SectorSink::finish
|
||||
pub trait SectorSink: Send {
|
||||
/// Write the sectors in `buf` starting at `lba`. `buf.len()`
|
||||
/// must be a multiple of 2048; the implementation seeks to
|
||||
/// `lba * 2048` before writing.
|
||||
fn write_sectors(&mut self, lba: u32, buf: &[u8]) -> Result<()>;
|
||||
|
||||
/// Flush, fsync, and close. Consumes the sink. Always called
|
||||
/// last; subsequent operations are not defined.
|
||||
fn finish(self: Box<Self>) -> Result<()>;
|
||||
}
|
||||
|
||||
/// 0.17 read trait. Slated for removal once internal call sites
|
||||
/// migrate to [`SectorSource`] in follow-up commits; until then
|
||||
/// it remains the trait that `Drive`, `IsoSectorReader`,
|
||||
/// `BufferedSectorReader`, and existing `&mut dyn SectorReader`
|
||||
/// signatures use unchanged.
|
||||
///
|
||||
/// New code should implement [`SectorSource`] directly. The
|
||||
/// blanket impl below makes any `SectorReader` automatically
|
||||
/// usable wherever a `SectorSource` is expected, so a one-way
|
||||
/// migration off `SectorReader` is possible per-callsite without
|
||||
/// touching the impls.
|
||||
//
|
||||
// NOTE: not marked `#[deprecated]` in this commit — `cargo clippy
|
||||
// -- -D warnings` (the CI gauntlet) treats deprecation as an
|
||||
// error, and the existing `Drive` / `udf::BufferedSectorReader` /
|
||||
// `mux::DiscStream` / `verify` call sites all go through this
|
||||
// trait. The deprecation attribute lands together with the
|
||||
// migration commits that move those call sites to
|
||||
// `SectorSource`. The behavioural contract — "this trait is
|
||||
// going away in 0.18" — is documented above and tracked in
|
||||
// `(internal)/memory/0_18_redesign.md`.
|
||||
pub trait SectorReader: Send {
|
||||
/// Read `count` sectors starting at `lba` into `buf`.
|
||||
/// See [`SectorSource::read_sectors`] for semantics.
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize>;
|
||||
|
||||
/// Total capacity in sectors, if known.
|
||||
fn capacity(&self) -> u32 {
|
||||
0
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, _kbs: u16) {}
|
||||
}
|
||||
|
||||
// Blanket impl: anything implementing the legacy `SectorReader`
|
||||
// trait automatically satisfies `SectorSource`. This is what keeps
|
||||
// existing impls (`Drive`, `IsoSectorReader`, `BufferedSectorReader`,
|
||||
// etc.) compiling without source changes during the migration. The
|
||||
// reverse direction (impl SectorReader for SectorSource) is
|
||||
// intentionally NOT provided — new code targets the new trait.
|
||||
impl<T: SectorReader + ?Sized> SectorSource for T {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
<T as SectorReader>::capacity(self)
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize> {
|
||||
<T as SectorReader>::read_sectors(self, lba, count, buf, recovery)
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
<T as SectorReader>::set_speed(self, kbs)
|
||||
}
|
||||
}
|
||||
|
||||
// Forwarding impls so callers can wrap `&mut dyn SectorReader` /
|
||||
// `Box<dyn SectorReader>` in [`DecryptingSectorSource`] without
|
||||
// having to unbox or re-borrow inside the lib's hot paths. The
|
||||
// generic `&mut T` / `Box<T>` blankets would conflict with the
|
||||
// `SectorReader → SectorSource` blanket above (a downstream crate
|
||||
// could `impl SectorReader for &mut U`); the specific
|
||||
// `dyn SectorReader` instantiations are unambiguous because
|
||||
// `SectorReader` is the very trait whose `dyn` we're targeting.
|
||||
impl SectorSource for &mut (dyn SectorReader + '_) {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
<dyn SectorReader as SectorReader>::capacity(*self)
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize> {
|
||||
<dyn SectorReader as SectorReader>::read_sectors(*self, lba, count, buf, recovery)
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
<dyn SectorReader as SectorReader>::set_speed(*self, kbs)
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for Box<dyn SectorReader> {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
<dyn SectorReader as SectorReader>::capacity(&**self)
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize> {
|
||||
<dyn SectorReader as SectorReader>::read_sectors(&mut **self, lba, count, buf, recovery)
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
<dyn SectorReader as SectorReader>::set_speed(&mut **self, kbs)
|
||||
}
|
||||
}
|
||||
|
||||
pub use decrypting::DecryptingSectorSource;
|
||||
pub use file::{FileSectorSink, FileSectorSource};
|
||||
|
||||
// Backwards-compat alias for the public API. `FileSectorReader` is
|
||||
// the 0.17 name; new code uses `FileSectorSource`. Both point at
|
||||
// the same type. The `#[deprecated]` attribute lands together with
|
||||
// the migration commits that retire the alias from internal uses.
|
||||
pub type FileSectorReader = FileSectorSource;
|
||||
+32
-12
@@ -1,6 +1,7 @@
|
||||
//! Disc sector verification — read every sector and classify health.
|
||||
|
||||
use crate::disc::{Chapter, DiscTitle};
|
||||
use crate::progress::Progress;
|
||||
use crate::sector::SectorReader;
|
||||
use std::time::Instant;
|
||||
|
||||
@@ -77,10 +78,6 @@ impl VerifyResult {
|
||||
}
|
||||
}
|
||||
|
||||
/// Progress callback: (sectors_done, total_sectors, current_status)
|
||||
/// Return false to stop verification early.
|
||||
pub type ProgressFn = Box<dyn FnMut(u64, u64, SectorStatus) -> bool>;
|
||||
|
||||
/// Verify all sectors in a title's extents.
|
||||
/// Reads in batches for speed, falls back to single-sector on failure.
|
||||
/// The progress callback returns false to request early stop.
|
||||
@@ -88,7 +85,7 @@ pub fn verify_title(
|
||||
reader: &mut dyn SectorReader,
|
||||
title: &DiscTitle,
|
||||
batch_sectors: u16,
|
||||
mut on_progress: Option<ProgressFn>,
|
||||
on_progress: Option<&dyn Progress>,
|
||||
) -> VerifyResult {
|
||||
let start = Instant::now();
|
||||
let mut good: u64 = 0;
|
||||
@@ -97,7 +94,6 @@ pub fn verify_title(
|
||||
let mut bad: u64 = 0;
|
||||
let mut ranges: Vec<SectorRange> = Vec::new();
|
||||
let mut sectors_done: u64 = 0;
|
||||
let mut _stopped = false;
|
||||
let mut byte_offset: u64 = 0;
|
||||
|
||||
let total_sectors: u64 = title.extents.iter().map(|e| e.sector_count as u64).sum();
|
||||
@@ -140,9 +136,21 @@ pub fn verify_title(
|
||||
}
|
||||
|
||||
sectors_done += count as u64;
|
||||
if let Some(ref mut cb) = on_progress {
|
||||
if !cb(sectors_done, total_sectors, status) {
|
||||
_stopped = true;
|
||||
if let Some(cb) = on_progress {
|
||||
let pp = crate::progress::PassProgress {
|
||||
kind: crate::progress::PassKind::Verify,
|
||||
work_done: sectors_done,
|
||||
work_total: total_sectors,
|
||||
bytes_good_total: (good + slow + recovered) * 2048,
|
||||
bytes_unreadable_total: bad * 2048,
|
||||
bytes_pending_total: 0,
|
||||
bytes_total_disc: total_sectors * 2048,
|
||||
disc_duration_secs: Some(title.duration_secs),
|
||||
bytes_bad_in_main_title: 0,
|
||||
main_title_duration_secs: Some(title.duration_secs),
|
||||
main_title_size_bytes: Some(total_sectors * 2048),
|
||||
};
|
||||
if !cb.report(&pp) {
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
@@ -214,9 +222,21 @@ pub fn verify_title(
|
||||
}
|
||||
|
||||
sectors_done += 1;
|
||||
if let Some(ref mut cb) = on_progress {
|
||||
if !cb(sectors_done, total_sectors, status) {
|
||||
_stopped = true;
|
||||
if let Some(cb) = on_progress {
|
||||
let pp = crate::progress::PassProgress {
|
||||
kind: crate::progress::PassKind::Verify,
|
||||
work_done: sectors_done,
|
||||
work_total: total_sectors,
|
||||
bytes_good_total: (good + slow + recovered) * 2048,
|
||||
bytes_unreadable_total: bad * 2048,
|
||||
bytes_pending_total: 0,
|
||||
bytes_total_disc: total_sectors * 2048,
|
||||
disc_duration_secs: Some(title.duration_secs),
|
||||
bytes_bad_in_main_title: 0,
|
||||
main_title_duration_secs: Some(title.duration_secs),
|
||||
main_title_size_bytes: Some(total_sectors * 2048),
|
||||
};
|
||||
if !cb.report(&pp) {
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
//! Integration tests for progress reporting, halt behavior, drop safety,
|
||||
//! and the file-backed sector reader round trip.
|
||||
|
||||
// 0.18 trait split: this suite still drives the deprecated `pes::Stream`
|
||||
// trait directly. It will be migrated to `FrameSource`/`FrameSink` in the
|
||||
// follow-up that ports concrete impls.
|
||||
#![allow(deprecated)]
|
||||
|
||||
use libfreemkv::disc::{CopyOptions, DiscRegion};
|
||||
use libfreemkv::error::Result;
|
||||
use libfreemkv::pes::Stream as PesStream;
|
||||
@@ -181,9 +186,10 @@ fn test_disc_copy_progress_callback_fires() {
|
||||
last_bytes: Arc<AtomicU64>,
|
||||
}
|
||||
impl libfreemkv::progress::Progress for CountingReporter {
|
||||
fn report(&self, p: &libfreemkv::progress::PassProgress) {
|
||||
fn report(&self, p: &libfreemkv::progress::PassProgress) -> bool {
|
||||
self.calls.fetch_add(1, Ordering::Relaxed);
|
||||
self.last_bytes.store(p.bytes_good_total, Ordering::Relaxed);
|
||||
true
|
||||
}
|
||||
}
|
||||
let reporter = CountingReporter {
|
||||
@@ -318,7 +324,7 @@ fn test_file_sector_reader_round_trip() {
|
||||
tmp.write_all(&data).expect("write data");
|
||||
tmp.flush().expect("flush");
|
||||
|
||||
let path = tmp.path().to_str().expect("path utf-8").to_string();
|
||||
let path = tmp.path().to_path_buf();
|
||||
let mut fsr = FileSectorReader::open(&path).expect("open FileSectorReader");
|
||||
|
||||
assert_eq!(fsr.capacity(), N_SECTORS as u32, "capacity mismatch");
|
||||
@@ -351,7 +357,7 @@ fn test_file_sector_reader_round_trip() {
|
||||
//
|
||||
// Per RIP_DESIGN.md §2.1 + §3: Disc::copy must reach the end of the disc
|
||||
// regardless of how many reads fail. The only legitimate early exit is the
|
||||
// halt flag. With `skip_on_error + skip_forward` and a reader that returns
|
||||
// halt flag. With `skip_on_error` and a reader that returns
|
||||
// Err for every read, Pass 1 must:
|
||||
// - mark every sector NonTrimmed (so Pass 2 can retry them)
|
||||
// - return cleanly (no panic, no hang)
|
||||
@@ -424,7 +430,7 @@ impl SectorReader for FailingSectorReader {
|
||||
#[test]
|
||||
fn test_disc_copy_completes_full_disc_with_failing_reader() {
|
||||
// 1024 sectors = 2 MB. Reader fails every read. With skip_on_error +
|
||||
// skip_forward, Pass 1 must mark every sector NonTrimmed and return
|
||||
// skip_on_error, Pass 1 must mark every sector NonTrimmed and return
|
||||
// cleanly — no bail, no hang.
|
||||
let capacity_sectors: u32 = 1024;
|
||||
let total_bytes: u64 = capacity_sectors as u64 * SECTOR_SIZE as u64;
|
||||
@@ -438,8 +444,8 @@ fn test_disc_copy_completes_full_disc_with_failing_reader() {
|
||||
|
||||
let opts = CopyOptions {
|
||||
decrypt: false,
|
||||
skip_on_error: true,
|
||||
skip_forward: true,
|
||||
multipass: true,
|
||||
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -454,9 +460,7 @@ fn test_disc_copy_completes_full_disc_with_failing_reader() {
|
||||
let _ = std::fs::remove_file(libfreemkv::disc::mapfile_path_for(&iso_path));
|
||||
|
||||
// Hard bound — even at 0 ms per read, 1024 sectors with skip-forward
|
||||
// should complete in well under a second on any host. If this test runs
|
||||
// for minutes, something has regressed (e.g. stall guard reintroduced
|
||||
// with infinite-loop semantics, or Pass 1 is hanging on each read).
|
||||
// should complete in well under a second on any host.
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(5),
|
||||
"Pass 1 took {elapsed:?} on a 2 MB synthetic disc — expected < 5 s"
|
||||
@@ -520,8 +524,8 @@ fn test_disc_copy_halts_promptly_on_failing_reader() {
|
||||
|
||||
let opts = CopyOptions {
|
||||
decrypt: false,
|
||||
skip_on_error: true,
|
||||
skip_forward: true,
|
||||
multipass: true,
|
||||
|
||||
halt: Some(halt),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -553,16 +557,9 @@ fn test_disc_copy_halts_promptly_on_failing_reader() {
|
||||
|
||||
// ── 8. Hysteresis recovers data the drive can read individually ──────────
|
||||
//
|
||||
// Empirically observed on the LG BU40N: in damaged regions the drive fails
|
||||
// multi-sector READ commands but reads each sector cleanly when asked one
|
||||
// at a time. Disc::copy's hysteresis state machine (0.13.22, replaces the
|
||||
// 0.13.21 bisect-on-fail) drops to bpt=1 on the first multi-sector failure
|
||||
// and stays there until BPT1_EXIT_THRESHOLD consecutive good single-sector
|
||||
// reads, then returns to bpt=batch.
|
||||
//
|
||||
// Fixture: a reader that returns Err for any read with count > 1, and Ok
|
||||
// for count == 1. The full disc must recover via the bpt=1 path with
|
||||
// 100 % bytes_good outcome.
|
||||
// Pass 1 reads in batch (32 sectors = 1 ECC block). Failed blocks are marked
|
||||
// NonTrimmed for Pass 2 recovery. This test verifies that a reader where every
|
||||
// multi-sector read fails produces all NonTrimmed output with zero bytes_good.
|
||||
|
||||
struct BlockSizeFailingReader {
|
||||
capacity: u32,
|
||||
@@ -577,15 +574,11 @@ impl SectorReader for BlockSizeFailingReader {
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
if count == 1 {
|
||||
// Single-sector reads succeed — fill the sector with a marker.
|
||||
for chunk in buf.chunks_mut(SECTOR_SIZE) {
|
||||
chunk.fill((lba & 0xff) as u8);
|
||||
}
|
||||
Ok(buf.len())
|
||||
} else {
|
||||
// Multi-sector reads fail with the BU40N's signature: CHECK
|
||||
// CONDITION + MEDIUM ERROR. The hysteresis must dispatch on
|
||||
// this as marginal-read and drop to bpt=1.
|
||||
Err(libfreemkv::error::Error::ScsiError {
|
||||
opcode: libfreemkv::scsi::SCSI_READ_10,
|
||||
status: libfreemkv::scsi::SCSI_STATUS_CHECK_CONDITION,
|
||||
@@ -604,13 +597,7 @@ impl SectorReader for BlockSizeFailingReader {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_disc_copy_hysteresis_recovers_via_single_sector_reads() {
|
||||
// 256 sectors = 0.5 MB. Reader fails any multi-sector read but
|
||||
// succeeds on bpt=1. The hysteresis path must drop to Single mode on
|
||||
// the first multi-sector failure and recover every sector at bpt=1.
|
||||
// Stays in Single mode until BPT1_EXIT_THRESHOLD reached (10 000
|
||||
// sectors); since this disc is only 256 sectors we never re-enter
|
||||
// Block mode, which is fine — every sector still recovers.
|
||||
fn test_disc_copy_marks_failed_ecc_blocks_as_nontrimmed() {
|
||||
let capacity_sectors: u32 = 256;
|
||||
let total_bytes: u64 = capacity_sectors as u64 * SECTOR_SIZE as u64;
|
||||
|
||||
@@ -625,8 +612,8 @@ fn test_disc_copy_hysteresis_recovers_via_single_sector_reads() {
|
||||
|
||||
let opts = CopyOptions {
|
||||
decrypt: false,
|
||||
skip_on_error: true,
|
||||
skip_forward: true,
|
||||
multipass: true,
|
||||
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -637,20 +624,187 @@ fn test_disc_copy_hysteresis_recovers_via_single_sector_reads() {
|
||||
let _ = std::fs::remove_file(&iso_path);
|
||||
let _ = std::fs::remove_file(libfreemkv::disc::mapfile_path_for(&iso_path));
|
||||
|
||||
// Bisect must recover every sector — the drive could read each one
|
||||
// individually, and our algorithm must descend to that.
|
||||
// Pass 1's job is "fast and accurate, get the most data in the
|
||||
// shortest time." It no longer bisects on marginal media — that's
|
||||
// Pass N's purpose-built role. So a BlockSizeFailingReader that
|
||||
// fails on multi-sector reads and succeeds on single-sector
|
||||
// results in: every batch fails → SkipBlock → whole 32-sector
|
||||
// ECC block marked NonTrimmed → Pass N (Disc::patch) revisits and
|
||||
// recovers via single-sector reads with proper recovery semantics.
|
||||
//
|
||||
// Pass 1 alone:
|
||||
assert_eq!(
|
||||
result.bytes_good, total_bytes,
|
||||
"bisect-on-fail must recover every sector via single-sector reads. \
|
||||
Got bytes_good={} of total {}",
|
||||
result.bytes_good, total_bytes
|
||||
result.bytes_good, 0,
|
||||
"Pass 1 doesn't bisect on marginal media — failed batches become NonTrimmed for Pass N to revisit"
|
||||
);
|
||||
assert_eq!(
|
||||
result.bytes_pending, 0,
|
||||
"no sectors should be left NonTrimmed after a successful bisect"
|
||||
result.bytes_pending, total_bytes,
|
||||
"every sector is NonTrimmed (pending) after Pass 1, awaiting Pass N"
|
||||
);
|
||||
assert!(
|
||||
result.complete,
|
||||
"complete=true expected when every sector recovered"
|
||||
!result.complete,
|
||||
"complete=false because NonTrimmed regions remain (Pass N's work)"
|
||||
);
|
||||
}
|
||||
|
||||
// ── 9. PassProgress carries separate unreadable vs pending byte counts ─────
|
||||
//
|
||||
// The video-damage-time display needs bytes_unreadable_total (confirmed dead)
|
||||
// separate from bytes_pending_total (might still recover). This test verifies
|
||||
// that a Pass 2 with some confirmed failures produces correct field values.
|
||||
|
||||
#[test]
|
||||
fn test_pass_progress_separates_unreadable_from_pending() {
|
||||
let capacity_sectors: u32 = 128;
|
||||
let total_bytes: u64 = capacity_sectors as u64 * SECTOR_SIZE as u64;
|
||||
|
||||
let mut reader = FailingSectorReader::new(capacity_sectors);
|
||||
let disc = synthetic_disc(capacity_sectors);
|
||||
|
||||
let tmp = tempfile::NamedTempFile::new().expect("tempfile create");
|
||||
let iso_path = tmp.path().to_path_buf();
|
||||
drop(tmp);
|
||||
|
||||
let opts = CopyOptions {
|
||||
decrypt: false,
|
||||
multipass: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let pass1 = disc.copy(&mut reader, &iso_path, &opts).expect("pass1 ok");
|
||||
|
||||
assert_eq!(pass1.bytes_good, 0, "pass1: no good sectors");
|
||||
assert_eq!(pass1.bytes_unreadable, 0, "pass1: no confirmed unreadable");
|
||||
assert_eq!(
|
||||
pass1.bytes_pending, total_bytes,
|
||||
"pass1: all sectors NonTrimmed"
|
||||
);
|
||||
|
||||
let last_unreadable = Arc::new(AtomicU64::new(0));
|
||||
let last_pending = Arc::new(AtomicU64::new(0));
|
||||
let last_good = Arc::new(AtomicU64::new(0));
|
||||
let last_dur = Arc::new(AtomicU64::new(0));
|
||||
|
||||
struct SnapshotReporter {
|
||||
unreadable: Arc<AtomicU64>,
|
||||
pending: Arc<AtomicU64>,
|
||||
good: Arc<AtomicU64>,
|
||||
dur: Arc<AtomicU64>,
|
||||
}
|
||||
impl libfreemkv::progress::Progress for SnapshotReporter {
|
||||
fn report(&self, p: &libfreemkv::progress::PassProgress) -> bool {
|
||||
self.unreadable
|
||||
.store(p.bytes_unreadable_total, Ordering::Relaxed);
|
||||
self.pending.store(p.bytes_pending_total, Ordering::Relaxed);
|
||||
self.good.store(p.bytes_good_total, Ordering::Relaxed);
|
||||
if let Some(d) = p.disc_duration_secs {
|
||||
self.dur.store((d * 1000.0) as u64, Ordering::Relaxed);
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
let reporter = SnapshotReporter {
|
||||
unreadable: last_unreadable.clone(),
|
||||
pending: last_pending.clone(),
|
||||
good: last_good.clone(),
|
||||
dur: last_dur.clone(),
|
||||
};
|
||||
|
||||
let pass2_opts = CopyOptions {
|
||||
decrypt: false,
|
||||
multipass: true,
|
||||
progress: Some(&reporter),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let pass2 = disc
|
||||
.copy(&mut reader, &iso_path, &pass2_opts)
|
||||
.expect("pass2 ok");
|
||||
|
||||
let _ = std::fs::remove_file(&iso_path);
|
||||
let _ = std::fs::remove_file(libfreemkv::disc::mapfile_path_for(&iso_path));
|
||||
|
||||
assert_eq!(
|
||||
pass2.bytes_good, 0,
|
||||
"pass2: still no good sectors (reader always fails)"
|
||||
);
|
||||
assert!(
|
||||
pass2.bytes_unreadable > 0,
|
||||
"pass2: some sectors confirmed unreadable"
|
||||
);
|
||||
assert!(
|
||||
pass2.bytes_pending < pass1.bytes_pending,
|
||||
"pass2: fewer pending sectors than pass1"
|
||||
);
|
||||
|
||||
let observed_unreadable = last_unreadable.load(Ordering::Relaxed);
|
||||
let observed_pending = last_pending.load(Ordering::Relaxed);
|
||||
assert!(
|
||||
observed_unreadable > 0,
|
||||
"progress should report confirmed unreadable bytes"
|
||||
);
|
||||
assert!(
|
||||
observed_pending == 0 || observed_pending < total_bytes,
|
||||
"pending should shrink as sectors are confirmed unreadable"
|
||||
);
|
||||
|
||||
// Video damage time: unreadable / total * duration
|
||||
// With no titles on synthetic disc, disc_duration_secs = None
|
||||
assert_eq!(
|
||||
last_dur.load(Ordering::Relaxed),
|
||||
0,
|
||||
"synthetic disc has no titles, duration should be None/0"
|
||||
);
|
||||
}
|
||||
|
||||
// ── 10. Damage time calculation (unit test) ────────────────────────────────
|
||||
//
|
||||
// Verifies the formula: damage_secs = bytes_unreadable / bytes_total * duration
|
||||
// This mirrors the CLI's print_disc_progress logic.
|
||||
|
||||
#[test]
|
||||
fn test_damage_time_calculation() {
|
||||
// 78.8 GB disc, 2h45m movie (9900s), 74 KB unreadable
|
||||
let disc_bytes: u64 = 78_800_000_000;
|
||||
let duration_secs: f64 = 9900.0;
|
||||
|
||||
let cases: Vec<(u64, &str)> = vec![
|
||||
(74 * 1024, "~10ms"), // 74 KB → ~9ms, negligible
|
||||
(10 * 1024 * 1024, "~1.3s"), // 10 MB → ~1.3s
|
||||
(100 * 1024 * 1024, "~13s"), // 100 MB → ~13s
|
||||
(1024 * 1024 * 1024, "~134s"), // 1 GB → ~134s
|
||||
];
|
||||
|
||||
for (bad_bytes, label) in cases {
|
||||
let damage_secs = bad_bytes as f64 / disc_bytes as f64 * duration_secs;
|
||||
match label {
|
||||
"~10ms" => assert!(damage_secs < 0.05, "{label}: {damage_secs:.3}s"),
|
||||
"~1.3s" => assert!(
|
||||
(damage_secs - 1.3).abs() < 0.2,
|
||||
"{label}: {damage_secs:.2}s"
|
||||
),
|
||||
"~13s" => assert!(
|
||||
(damage_secs - 13.0).abs() < 1.0,
|
||||
"{label}: {damage_secs:.1}s"
|
||||
),
|
||||
"~134s" => assert!(
|
||||
(damage_secs - 134.0).abs() < 2.0,
|
||||
"{label}: {damage_secs:.0}s"
|
||||
),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// 0.25s threshold: how many bad bytes = 0.25s of damage?
|
||||
let threshold_bytes = (0.25 / duration_secs * disc_bytes as f64) as u64;
|
||||
assert!(
|
||||
threshold_bytes > 0,
|
||||
"0.25s damage threshold should be > 0 bytes"
|
||||
);
|
||||
// At 9900s / 78.8 GB ≈ 0.25s = ~2 MB
|
||||
let expected_mb = threshold_bytes as f64 / (1024.0 * 1024.0);
|
||||
assert!(
|
||||
(expected_mb - 2.0).abs() < 0.5,
|
||||
"0.25s ≈ {expected_mb:.2} MB (expected ~2 MB)"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
//! Regression tests for Pass N (patch) fix — decrypt key inversion bug.
|
||||
//!
|
||||
//! Tests that decrypt_sectors is invoked correctly when opts.decrypt=true.
|
||||
//! The 2026-05-03 bug at `libfreemkv/src/disc/mod.rs:1938-1942` inverted
|
||||
//! the decrypt key arms, causing patch to pass DecryptKeys::None on encrypted discs.
|
||||
|
||||
use libfreemkv::{aacs, decrypt::DecryptKeys};
|
||||
|
||||
/// Test: decrypt_sectors with AACS keys actually decrypts units.
|
||||
#[test]
|
||||
fn decrypt_sectors_with_aacs_keys_works() {
|
||||
// Build an encrypted aligned unit
|
||||
let mut unit = vec![0xFFu8; aacs::ALIGNED_UNIT_LEN];
|
||||
|
||||
// Set encryption flag (bits 6-7 of byte 0)
|
||||
unit[0] |= 0xC0;
|
||||
|
||||
// Fill with recognizable pattern
|
||||
for (i, byte) in unit
|
||||
.iter_mut()
|
||||
.enumerate()
|
||||
.take(aacs::ALIGNED_UNIT_LEN)
|
||||
.skip(1)
|
||||
{
|
||||
*byte = ((i * 3 + 7) & 0xFF) as u8;
|
||||
}
|
||||
|
||||
let unit_key: [u8; 16] = [0xAAu8; 16];
|
||||
|
||||
// Encrypt the unit using AACS algorithm
|
||||
aacs::decrypt_unit(&mut unit, &unit_key); // decrypt_unit is idempotent on already-encrypted data
|
||||
|
||||
// Now we have encrypted data - create DecryptKeys with actual keys
|
||||
let keys = DecryptKeys::Aacs {
|
||||
unit_keys: vec![(0u32, unit_key)],
|
||||
read_data_key: None,
|
||||
};
|
||||
|
||||
// decrypt_sectors should handle this without error
|
||||
let result = libfreemkv::decrypt::decrypt_sectors(&mut unit, &keys, 0);
|
||||
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"decrypt_sectors with AACS keys should not error"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test: decrypt_sectors with DecryptKeys::None is a no-op.
|
||||
#[test]
|
||||
fn decrypt_sectors_with_none_keys_is_noop() {
|
||||
let mut sector = vec![0x42u8; 2048];
|
||||
|
||||
let keys = DecryptKeys::None;
|
||||
let result = libfreemkv::decrypt::decrypt_sectors(&mut sector, &keys, 0);
|
||||
|
||||
assert!(result.is_ok());
|
||||
assert_eq!(
|
||||
§or[..],
|
||||
&[0x42u8; 2048][..],
|
||||
"DecryptKeys::None should not modify buffer"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test: decrypt_sectors with CSS keys descrambles sectors.
|
||||
#[test]
|
||||
fn decrypt_sectors_with_css_keys_works() {
|
||||
let mut sector = vec![0xFFu8; 2048];
|
||||
|
||||
// Set CSS scramble flag (bits 4-5 of byte 0x14)
|
||||
sector[0x14] |= 0x30;
|
||||
|
||||
let title_key: [u8; 5] = [0x42, 0x13, 0x37, 0xBE, 0xEF]; // Not used - defined later
|
||||
let keys = DecryptKeys::Css { title_key };
|
||||
|
||||
// Descramble (CSS uses same operation for encrypt/decrypt)
|
||||
libfreemkv::decrypt::decrypt_sectors(&mut sector, &keys, 0).unwrap();
|
||||
|
||||
// Flag should be cleared
|
||||
assert_eq!(sector[0x14] & 0x30, 0x00, "CSS flag should be cleared");
|
||||
}
|
||||
|
||||
/// Test: AACS unit encryption detection works.
|
||||
#[test]
|
||||
fn aacs_encryption_flag_detection() {
|
||||
let mut unit = vec![0u8; aacs::ALIGNED_UNIT_LEN];
|
||||
|
||||
// No encryption flag
|
||||
assert!(!aacs::is_unit_encrypted(&unit));
|
||||
|
||||
// Set bit 6
|
||||
unit[0] |= 0x40;
|
||||
assert!(aacs::is_unit_encrypted(&unit));
|
||||
|
||||
// Set bit 7
|
||||
unit[0] = 0x80;
|
||||
assert!(aacs::is_unit_encrypted(&unit));
|
||||
|
||||
// Both bits set
|
||||
unit[0] = 0xC0;
|
||||
assert!(aacs::is_unit_encrypted(&unit));
|
||||
}
|
||||
|
||||
/// Test: DecryptKeys::is_encrypted() correctly identifies encrypted state.
|
||||
#[test]
|
||||
fn decrypt_keys_is_encrypted_variants() {
|
||||
let none = DecryptKeys::None;
|
||||
assert!(!none.is_encrypted());
|
||||
|
||||
let aacs = DecryptKeys::Aacs {
|
||||
unit_keys: vec![],
|
||||
read_data_key: None,
|
||||
};
|
||||
assert!(aacs.is_encrypted());
|
||||
|
||||
let css = DecryptKeys::Css {
|
||||
title_key: [0u8; 5],
|
||||
};
|
||||
assert!(css.is_encrypted());
|
||||
}
|
||||
@@ -0,0 +1,399 @@
|
||||
//! Pass N (Disc::patch) size-aware-skip targeted tests.
|
||||
//!
|
||||
//! The user's failure mode (2026-05-07): "what if we have a 100 sector zone
|
||||
//! and its really 2 25 sector zones and we keep jumping over the good in
|
||||
//! the middle." Today's pre-fix patch escalates skip-distance based on
|
||||
//! `consecutive_skips_without_recovery` with hardcoded 32 → 4096 sector
|
||||
//! caps. A 100-sector bad range whose actual layout is 25 bad + 50 good +
|
||||
//! 25 bad would have the patch skip 32-4096 sectors after a couple of
|
||||
//! failures, leaping over the entire range AND the good middle.
|
||||
//!
|
||||
//! The fix: cap each skip at `range_remaining/4`. These tests exercise
|
||||
//! that boundary.
|
||||
|
||||
use libfreemkv::disc::CopyOptions;
|
||||
use libfreemkv::disc::DiscRegion;
|
||||
use libfreemkv::disc::mapfile::{Mapfile, SectorStatus};
|
||||
use libfreemkv::error::Result;
|
||||
use libfreemkv::{ContentFormat, Disc, DiscFormat, SectorReader};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
const SECTOR_SIZE: usize = 2048;
|
||||
|
||||
/// Reader where you specify exactly which LBAs return Err. Everything else
|
||||
/// returns Ok with the LBA encoded in each byte for verification.
|
||||
struct PatternedSectorReader {
|
||||
capacity: u32,
|
||||
bad_lbas: HashSet<u32>,
|
||||
/// Trace every read so tests can assert what was actually attempted.
|
||||
trace: Arc<Mutex<Vec<(u32, u16)>>>,
|
||||
}
|
||||
|
||||
impl PatternedSectorReader {
|
||||
fn new(capacity: u32, bad_lbas: HashSet<u32>) -> (Self, Arc<Mutex<Vec<(u32, u16)>>>) {
|
||||
let trace = Arc::new(Mutex::new(Vec::new()));
|
||||
(
|
||||
Self {
|
||||
capacity,
|
||||
bad_lbas,
|
||||
trace: trace.clone(),
|
||||
},
|
||||
trace,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for PatternedSectorReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
self.trace.lock().unwrap().push((lba, count));
|
||||
// Whole-batch fails if ANY sector in the batch is bad. (Models a
|
||||
// real drive: a multi-sector READ aborts on the first ECC failure.)
|
||||
for offset in 0..count as u32 {
|
||||
if self.bad_lbas.contains(&(lba + offset)) {
|
||||
return Err(libfreemkv::error::Error::ScsiError {
|
||||
opcode: libfreemkv::scsi::SCSI_READ_10,
|
||||
status: libfreemkv::scsi::SCSI_STATUS_CHECK_CONDITION,
|
||||
sense: Some(libfreemkv::ScsiSense {
|
||||
sense_key: libfreemkv::scsi::SENSE_KEY_MEDIUM_ERROR,
|
||||
asc: 0x11,
|
||||
ascq: 0x00,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
for chunk in buf.chunks_mut(SECTOR_SIZE) {
|
||||
chunk.fill((lba & 0xff) as u8);
|
||||
}
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn capacity(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
|
||||
fn synthetic_disc(capacity_sectors: u32) -> Disc {
|
||||
Disc {
|
||||
volume_id: String::new(),
|
||||
meta_title: None,
|
||||
format: DiscFormat::BluRay,
|
||||
capacity_sectors,
|
||||
capacity_bytes: capacity_sectors as u64 * SECTOR_SIZE as u64,
|
||||
layers: 1,
|
||||
titles: Vec::new(),
|
||||
region: DiscRegion::Free,
|
||||
aacs: None,
|
||||
css: None,
|
||||
encrypted: false,
|
||||
content_format: ContentFormat::BdTs,
|
||||
}
|
||||
}
|
||||
|
||||
/// Pre-populate a mapfile with one large NonTrimmed range so patch's work-
|
||||
/// list has something to do. Caller pre-allocates the ISO at `total_bytes`
|
||||
/// so seeks don't fail.
|
||||
fn prep_iso_and_mapfile(
|
||||
iso_path: &std::path::Path,
|
||||
total_bytes: u64,
|
||||
finished_ranges: &[(u64, u64)],
|
||||
nontrimmed_ranges: &[(u64, u64)],
|
||||
) {
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
let mut f = OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(true)
|
||||
.open(iso_path)
|
||||
.unwrap();
|
||||
f.set_len(total_bytes).unwrap();
|
||||
f.seek(SeekFrom::Start(0)).unwrap();
|
||||
f.write_all(&[]).unwrap();
|
||||
|
||||
let map_path = libfreemkv::disc::mapfile_path_for(iso_path);
|
||||
let mut mf = Mapfile::create(&map_path, total_bytes, "test").unwrap();
|
||||
for &(pos, size) in finished_ranges {
|
||||
mf.record(pos, size, SectorStatus::Finished).unwrap();
|
||||
}
|
||||
for &(pos, size) in nontrimmed_ranges {
|
||||
mf.record(pos, size, SectorStatus::NonTrimmed).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
/// THE critical test. A 100-sector "bad" range hides 50 good sectors in
|
||||
/// the middle (LBAs 125-174). Pre-fix patch would skip-escalate at 32+
|
||||
/// sectors and leap over the whole range. Post-fix: skip is capped at
|
||||
/// range_remaining/4 (=25 sectors initially), which forces convergence.
|
||||
#[test]
|
||||
fn patch_recovers_good_middle_of_a_bad_range() {
|
||||
let capacity_sectors: u32 = 1024;
|
||||
let total_bytes: u64 = capacity_sectors as u64 * SECTOR_SIZE as u64;
|
||||
|
||||
// Bad range layout: LBAs 100-124 bad, 125-174 GOOD, 175-199 bad.
|
||||
let mut bad_lbas = HashSet::new();
|
||||
for lba in 100..125 {
|
||||
bad_lbas.insert(lba);
|
||||
}
|
||||
for lba in 175..200 {
|
||||
bad_lbas.insert(lba);
|
||||
}
|
||||
|
||||
let (mut reader, _trace) = PatternedSectorReader::new(capacity_sectors, bad_lbas);
|
||||
let disc = synthetic_disc(capacity_sectors);
|
||||
|
||||
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||
let iso_path = tmp.path().to_path_buf();
|
||||
drop(tmp);
|
||||
|
||||
// Pre-populate: 0..100 already Finished from an imagined Pass 1,
|
||||
// 100..200 NonTrimmed (the range we want patch to retry),
|
||||
// 200..1024 already Finished.
|
||||
let finished = [
|
||||
(0, 100 * 2048),
|
||||
(200 * 2048, (capacity_sectors as u64 - 200) * 2048),
|
||||
];
|
||||
let nontrimmed = [(100 * 2048, 100 * 2048)];
|
||||
prep_iso_and_mapfile(&iso_path, total_bytes, &finished, &nontrimmed);
|
||||
|
||||
// Run patch.
|
||||
// disc.copy() with multipass=true auto-dispatches to patch when the
|
||||
// mapfile already covers the disc and has retryable ranges.
|
||||
let opts = CopyOptions {
|
||||
decrypt: false,
|
||||
multipass: true,
|
||||
..Default::default()
|
||||
};
|
||||
let pr = disc
|
||||
.copy(&mut reader, &iso_path, &opts)
|
||||
.expect("copy returns Ok");
|
||||
|
||||
// Re-load mapfile and inspect.
|
||||
let map_path = libfreemkv::disc::mapfile_path_for(&iso_path);
|
||||
let map = Mapfile::load(&map_path).unwrap();
|
||||
|
||||
// The good middle (125..175) MUST end up Finished. If size-aware skip
|
||||
// is not enabled, patch would skip 32+ sectors after a few failures
|
||||
// and leap clean over LBA 125 → middle stays NonTrimmed/Unreadable.
|
||||
let finished_ranges = map.ranges_with(&[SectorStatus::Finished]);
|
||||
let total_finished_in_middle: u64 = finished_ranges
|
||||
.iter()
|
||||
.map(|&(pos, sz)| {
|
||||
let start = pos.max(125 * 2048);
|
||||
let end = (pos + sz).min(175 * 2048);
|
||||
end.saturating_sub(start)
|
||||
})
|
||||
.sum();
|
||||
|
||||
// Allow 2 sectors (4 KB) of boundary slop — patch's bisection may
|
||||
// not converge exactly on the good/bad boundary in a single pass,
|
||||
// and that's acceptable. The pre-fix behaviour would have left the
|
||||
// entire good middle as NonTrimmed (~0 bytes recovered).
|
||||
let good_middle_bytes: u64 = 50 * 2048;
|
||||
let min_acceptable: u64 = good_middle_bytes - 2 * 2048;
|
||||
|
||||
// Cleanup before assertions
|
||||
let _ = std::fs::remove_file(&iso_path);
|
||||
let _ = std::fs::remove_file(&map_path);
|
||||
|
||||
assert!(
|
||||
total_finished_in_middle >= min_acceptable,
|
||||
"size-aware skip should have discovered most of the 50 good sectors in the middle. \
|
||||
Recovered {} of {} good middle bytes (min acceptable {}). bytes_good={} bytes_total={}",
|
||||
total_finished_in_middle,
|
||||
good_middle_bytes,
|
||||
min_acceptable,
|
||||
pr.bytes_good,
|
||||
pr.bytes_total,
|
||||
);
|
||||
}
|
||||
|
||||
/// A second test: a bad range that's actually 4 small bad sub-zones
|
||||
/// separated by good sectors. Demonstrates the bisection behaviour
|
||||
/// converges when zones are non-uniform.
|
||||
#[test]
|
||||
fn patch_recovers_multiple_good_middles() {
|
||||
let capacity_sectors: u32 = 2048;
|
||||
let total_bytes: u64 = capacity_sectors as u64 * SECTOR_SIZE as u64;
|
||||
|
||||
// Bad pattern: 1000-1024 bad, 1025-1099 good, 1100-1124 bad,
|
||||
// 1125-1199 good, 1200-1224 bad, 1225-1299 good.
|
||||
let mut bad_lbas = HashSet::new();
|
||||
for lba in 1000..1025 {
|
||||
bad_lbas.insert(lba);
|
||||
}
|
||||
for lba in 1100..1125 {
|
||||
bad_lbas.insert(lba);
|
||||
}
|
||||
for lba in 1200..1225 {
|
||||
bad_lbas.insert(lba);
|
||||
}
|
||||
let (mut reader, _trace) = PatternedSectorReader::new(capacity_sectors, bad_lbas);
|
||||
let disc = synthetic_disc(capacity_sectors);
|
||||
|
||||
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||
let iso_path = tmp.path().to_path_buf();
|
||||
drop(tmp);
|
||||
|
||||
let finished = [
|
||||
(0, 1000 * 2048),
|
||||
(1300 * 2048, (capacity_sectors as u64 - 1300) * 2048),
|
||||
];
|
||||
let nontrimmed = [(1000 * 2048, 300 * 2048)];
|
||||
prep_iso_and_mapfile(&iso_path, total_bytes, &finished, &nontrimmed);
|
||||
|
||||
let opts = CopyOptions {
|
||||
decrypt: false,
|
||||
multipass: true,
|
||||
..Default::default()
|
||||
};
|
||||
let pr = disc
|
||||
.copy(&mut reader, &iso_path, &opts)
|
||||
.expect("copy returns Ok");
|
||||
|
||||
let map_path = libfreemkv::disc::mapfile_path_for(&iso_path);
|
||||
let map = Mapfile::load(&map_path).unwrap();
|
||||
let finished_ranges = map.ranges_with(&[SectorStatus::Finished]);
|
||||
let recovered: u64 = finished_ranges
|
||||
.iter()
|
||||
.map(|&(pos, sz)| {
|
||||
let start = pos.max(1000 * 2048);
|
||||
let end = (pos + sz).min(1300 * 2048);
|
||||
end.saturating_sub(start)
|
||||
})
|
||||
.sum();
|
||||
|
||||
let _ = std::fs::remove_file(&iso_path);
|
||||
let _ = std::fs::remove_file(&map_path);
|
||||
|
||||
// Three good middles of 75 sectors each = 225 good sectors in the
|
||||
// bad range. Total bad = 75. So we want at least most of 225 sectors
|
||||
// (= 460800 bytes) to be Finished after patch.
|
||||
let target = 200 * 2048; // be generous — anything over 200 sectors is convincing
|
||||
assert!(
|
||||
recovered >= target,
|
||||
"size-aware skip should find most of the 3 good middles. \
|
||||
Recovered {} bytes; expected ≥ {}. bytes_good={} bytes_total={}",
|
||||
recovered,
|
||||
target,
|
||||
pr.bytes_good,
|
||||
pr.bytes_total,
|
||||
);
|
||||
}
|
||||
|
||||
/// 0.18 Pass N pipeline split: exercises the new producer/consumer
|
||||
/// path end-to-end on a synthetic patterned reader. Bad range layout
|
||||
/// is small (5 bad LBAs surrounded by good middle) so the producer
|
||||
/// emits a mix of `Recovered` and `Unreadable` items and the consumer
|
||||
/// thread must apply both kinds. Verifies:
|
||||
///
|
||||
/// - `bytes_good` advances (good sectors flow producer→consumer→file
|
||||
/// →mapfile with the data preserved).
|
||||
/// - The recovered LBAs end up Finished; the bad LBAs end up Unreadable.
|
||||
/// - Bytes written at the recovered offsets match what the producer
|
||||
/// read from the patterned source (proves the channel hand-off
|
||||
/// didn't drop or reorder buffers, and the consumer's seek+write
|
||||
/// landed at the right offsets).
|
||||
#[test]
|
||||
fn patch_pipeline_split_recovers_and_records_correctly() {
|
||||
let capacity_sectors: u32 = 512;
|
||||
let total_bytes: u64 = capacity_sectors as u64 * SECTOR_SIZE as u64;
|
||||
|
||||
// Layout: LBAs 200-204 inclusive are bad (5 sectors), 205-249 good.
|
||||
// The pre-existing range is LBAs 200-249 NonTrimmed (100 KB).
|
||||
let mut bad_lbas = HashSet::new();
|
||||
for lba in 200..205 {
|
||||
bad_lbas.insert(lba);
|
||||
}
|
||||
|
||||
let (mut reader, _trace) = PatternedSectorReader::new(capacity_sectors, bad_lbas.clone());
|
||||
let disc = synthetic_disc(capacity_sectors);
|
||||
|
||||
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||
let iso_path = tmp.path().to_path_buf();
|
||||
drop(tmp);
|
||||
|
||||
let finished = [
|
||||
(0, 200 * 2048),
|
||||
(250 * 2048, (capacity_sectors as u64 - 250) * 2048),
|
||||
];
|
||||
let nontrimmed = [(200 * 2048, 50 * 2048)];
|
||||
prep_iso_and_mapfile(&iso_path, total_bytes, &finished, &nontrimmed);
|
||||
|
||||
let opts = CopyOptions {
|
||||
decrypt: false,
|
||||
multipass: true,
|
||||
..Default::default()
|
||||
};
|
||||
let pr = disc
|
||||
.copy(&mut reader, &iso_path, &opts)
|
||||
.expect("copy returns Ok");
|
||||
|
||||
// Bytes_good_total should advance — the good LBAs in the bad range
|
||||
// (205-249, 45 sectors) are all reachable via per-sector retry.
|
||||
// Initial bytes_good = 200 * 2048 + (512-250) * 2048 = 462 sectors.
|
||||
// After patch, bytes_good should be ≥ 462 + 45 = 507 sectors worth.
|
||||
let initial_good_sectors: u64 = 200 + (capacity_sectors as u64 - 250);
|
||||
let min_expected_good_bytes = (initial_good_sectors + 30) * 2048;
|
||||
assert!(
|
||||
pr.bytes_good >= min_expected_good_bytes,
|
||||
"patch should have recovered most good LBAs in the bad range via the pipeline. \
|
||||
bytes_good={} (expected ≥ {}); bytes_total={}",
|
||||
pr.bytes_good,
|
||||
min_expected_good_bytes,
|
||||
pr.bytes_total,
|
||||
);
|
||||
|
||||
// Verify the mapfile records: every good LBA is Finished, every
|
||||
// bad LBA is either Unreadable or NonTrimmed (not Finished).
|
||||
let map_path = libfreemkv::disc::mapfile_path_for(&iso_path);
|
||||
let map = Mapfile::load(&map_path).unwrap();
|
||||
let finished_ranges = map.ranges_with(&[SectorStatus::Finished]);
|
||||
let in_finished = |lba: u32| -> bool {
|
||||
let pos = lba as u64 * 2048;
|
||||
finished_ranges
|
||||
.iter()
|
||||
.any(|&(p, sz)| pos >= p && pos < p + sz)
|
||||
};
|
||||
|
||||
for lba in 205..250 {
|
||||
assert!(
|
||||
in_finished(lba),
|
||||
"good LBA {lba} should be Finished after pipeline patch run"
|
||||
);
|
||||
}
|
||||
for lba in 200..205 {
|
||||
assert!(
|
||||
!in_finished(lba),
|
||||
"bad LBA {lba} should NOT be Finished after pipeline patch run"
|
||||
);
|
||||
}
|
||||
|
||||
// Verify the consumer wrote the producer's bytes at the right
|
||||
// offsets. PatternedSectorReader fills each sector with `(lba & 0xff)
|
||||
// as u8` — picking LBA 220 (well inside the recovered region) gives
|
||||
// a clean signature byte to check.
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
let mut iso = std::fs::File::open(&iso_path).unwrap();
|
||||
iso.seek(SeekFrom::Start(220 * 2048)).unwrap();
|
||||
let mut sector = [0u8; 2048];
|
||||
iso.read_exact(&mut sector).unwrap();
|
||||
let expected_byte = (220u32 & 0xff) as u8;
|
||||
|
||||
let _ = std::fs::remove_file(&iso_path);
|
||||
let _ = std::fs::remove_file(&map_path);
|
||||
|
||||
assert!(
|
||||
sector.iter().all(|&b| b == expected_byte),
|
||||
"consumer should have written PatternedSectorReader's pattern \
|
||||
(byte {expected_byte:#x} for LBA 220) to the recovered offset; \
|
||||
got first 8 bytes = {:?}",
|
||||
§or[..8]
|
||||
);
|
||||
}
|
||||
@@ -203,7 +203,10 @@ fn test_check_cond_not_ready_predicate() {
|
||||
|
||||
let sense = err.scsi_sense().expect("CHECK CONDITION must carry sense");
|
||||
assert!(sense.is_not_ready(), "sense_key 2 ⇒ is_not_ready");
|
||||
assert!(!err.is_marginal_read(), "NOT READY is not marginal");
|
||||
assert!(
|
||||
err.is_marginal_read(),
|
||||
"NOT READY is marginal (BU40N returns this for bad sectors)"
|
||||
);
|
||||
}
|
||||
|
||||
// ── 5. CHECK CONDITION + MEDIUM ERROR (canonical marginal-read) ───────────
|
||||
|
||||
+11
-8
@@ -8,24 +8,25 @@
|
||||
//!
|
||||
//! These tests require a real /dev/sg* device and are therefore #[ignore].
|
||||
|
||||
use libfreemkv::scsi::{DataDirection, SCSI_STATUS_TRANSPORT_FAILURE};
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn test_sgio_transport_timeout_does_not_kill_transport() {
|
||||
let device = "/dev/sg2";
|
||||
let device = std::env::var("FREEMKV_TEST_SG_DEVICE").unwrap_or(device.to_string());
|
||||
let path = Path::new(&device);
|
||||
let _path = Path::new(&device);
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use libfreemkv::scsi::ScsiTransport;
|
||||
use libfreemkv::scsi::linux::SgIoTransport;
|
||||
use libfreemkv::scsi::{DataDirection, SCSI_STATUS_TRANSPORT_FAILURE};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicI32, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
let mut transport = SgIoTransport::open(path).expect("open device");
|
||||
let mut transport = SgIoTransport::open(_path).expect("open device");
|
||||
let fd_before = transport.fd;
|
||||
|
||||
// READ_10 with 1 ms timeout to force kernel timeout.
|
||||
@@ -34,7 +35,7 @@ fn test_sgio_transport_timeout_does_not_kill_transport() {
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let result = transport.execute(&cdb, DataDirection::FromDevice, &mut data, 1);
|
||||
let elapsed = start.elapsed();
|
||||
let _elapsed = start.elapsed();
|
||||
|
||||
assert!(result.is_err(), "expected Err on timeout, got {:?}", result);
|
||||
let err = result.unwrap_err();
|
||||
@@ -89,15 +90,17 @@ fn test_sgio_transport_timeout_does_not_kill_transport() {
|
||||
fn test_drive_read_per_cdb_timeout_bounds_call() {
|
||||
let device = "/dev/sg2";
|
||||
let device = std::env::var("FREEMKV_TEST_SG_DEVICE").unwrap_or(device.to_string());
|
||||
let path = Path::new(&device);
|
||||
let _path = Path::new(&device);
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let mut drive = libfreemkv::Drive::open(path).expect("open drive");
|
||||
use std::time::Duration;
|
||||
|
||||
let mut drive = libfreemkv::Drive::open(_path).expect("open drive");
|
||||
let timeout_ms: u32 = 5_000;
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let _ = drive.read(0, &mut [0u8; 2048], Some(timeout_ms));
|
||||
let _ = drive.read(0, 1, &mut [0u8; 2048], false);
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
let overhead = Duration::from_millis(500);
|
||||
|
||||
+25
-19
@@ -1,5 +1,10 @@
|
||||
//! Integration tests for the PES stream pipeline.
|
||||
|
||||
// 0.18 trait split: this suite still drives the deprecated `pes::Stream`
|
||||
// trait directly. It will be migrated to `FrameSource`/`FrameSink` in the
|
||||
// follow-up that ports concrete impls.
|
||||
#![allow(deprecated)]
|
||||
|
||||
use libfreemkv::mux::meta::M2tsMeta;
|
||||
use libfreemkv::pes::Stream as PesStream;
|
||||
use libfreemkv::*;
|
||||
@@ -302,12 +307,12 @@ fn m2ts_stream_write_read() {
|
||||
keyframe: i == 0,
|
||||
data: vec![i; 100],
|
||||
};
|
||||
stream.write(&frame).unwrap();
|
||||
PesStream::write(&mut stream, &frame).unwrap();
|
||||
}
|
||||
stream.finish().unwrap();
|
||||
PesStream::finish(&mut stream).unwrap();
|
||||
|
||||
// Verify the info is correct
|
||||
let info = stream.info();
|
||||
let info = PesStream::info(&stream);
|
||||
assert_eq!(info.streams.len(), 4);
|
||||
assert_eq!(info.duration_secs, 7200.0);
|
||||
}
|
||||
@@ -356,7 +361,7 @@ fn m2ts_read_returns_error_on_write_stream() {
|
||||
let dt = sample_disc_title();
|
||||
let output = Cursor::new(Vec::new());
|
||||
let mut stream = M2tsStream::create(output, &dt).unwrap();
|
||||
assert!(stream.read().is_err());
|
||||
assert!(PesStream::read(&mut stream).is_err());
|
||||
}
|
||||
|
||||
// ── DiscTitle::empty ──────────────────────────────────────────
|
||||
@@ -578,7 +583,7 @@ fn meta_all_stream_types() {
|
||||
#[test]
|
||||
fn mkvstream_write_finish() {
|
||||
let dt = sample_disc_title();
|
||||
let writer: Box<dyn libfreemkv::mux::WriteSeek> = Box::new(Cursor::new(Vec::new()));
|
||||
let writer: Box<dyn libfreemkv::mux::WriteSeek + Send> = Box::new(Cursor::new(Vec::new()));
|
||||
let mut stream = MkvStream::create(writer, &dt).unwrap();
|
||||
|
||||
// Write some fake PES frames (they won't produce valid MKV content
|
||||
@@ -590,20 +595,20 @@ fn mkvstream_write_finish() {
|
||||
keyframe: i == 0,
|
||||
data: vec![i; 100],
|
||||
};
|
||||
stream.write(&frame).unwrap();
|
||||
PesStream::write(&mut stream, &frame).unwrap();
|
||||
}
|
||||
|
||||
// finish should not panic even without valid codec data
|
||||
stream.finish().unwrap();
|
||||
PesStream::finish(&mut stream).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mkvstream_meta_sets_title() {
|
||||
let dt = sample_disc_title();
|
||||
let writer: Box<dyn libfreemkv::mux::WriteSeek> = Box::new(Cursor::new(Vec::new()));
|
||||
let writer: Box<dyn libfreemkv::mux::WriteSeek + Send> = Box::new(Cursor::new(Vec::new()));
|
||||
let stream = MkvStream::create(writer, &dt).unwrap();
|
||||
|
||||
let info = stream.info();
|
||||
let info = PesStream::info(&stream);
|
||||
assert_eq!(info.playlist, "Test Movie");
|
||||
assert_eq!(info.duration_secs, 7200.0);
|
||||
assert_eq!(info.streams.len(), 4);
|
||||
@@ -640,7 +645,7 @@ fn mkvstream_roundtrip_bdts() {
|
||||
codec_privates: Vec::new(),
|
||||
};
|
||||
|
||||
let writer: Box<dyn libfreemkv::mux::WriteSeek> = Box::new(Cursor::new(Vec::new()));
|
||||
let writer: Box<dyn libfreemkv::mux::WriteSeek + Send> = Box::new(Cursor::new(Vec::new()));
|
||||
let mut stream = MkvStream::create(writer, &dt).unwrap();
|
||||
|
||||
// Write PES frames targeting the audio track
|
||||
@@ -651,13 +656,13 @@ fn mkvstream_roundtrip_bdts() {
|
||||
keyframe: true,
|
||||
data: vec![i; 100],
|
||||
};
|
||||
stream.write(&frame).unwrap();
|
||||
PesStream::write(&mut stream, &frame).unwrap();
|
||||
}
|
||||
|
||||
stream.finish().unwrap();
|
||||
PesStream::finish(&mut stream).unwrap();
|
||||
|
||||
// Verify the info is correct
|
||||
let info = stream.info();
|
||||
let info = PesStream::info(&stream);
|
||||
assert_eq!(info.streams.len(), 1);
|
||||
assert_eq!(info.playlist, "Audio Only");
|
||||
}
|
||||
@@ -724,10 +729,10 @@ fn mkvstream_meta_preserves_all_streams() {
|
||||
codec_privates: Vec::new(),
|
||||
};
|
||||
|
||||
let writer: Box<dyn libfreemkv::mux::WriteSeek> = Box::new(Cursor::new(Vec::new()));
|
||||
let writer: Box<dyn libfreemkv::mux::WriteSeek + Send> = Box::new(Cursor::new(Vec::new()));
|
||||
let stream = MkvStream::create(writer, &dt).unwrap();
|
||||
|
||||
let info = stream.info();
|
||||
let info = PesStream::info(&stream);
|
||||
assert_eq!(info.streams.len(), 5, "all 5 streams should be preserved");
|
||||
assert_eq!(info.playlist, "Stream Test");
|
||||
assert_eq!(info.duration_secs, 3600.0);
|
||||
@@ -832,7 +837,8 @@ fn mkvstream_e2e_h264_produces_valid_mkv() {
|
||||
}
|
||||
}
|
||||
|
||||
let writer: Box<dyn libfreemkv::mux::WriteSeek> = Box::new(SharedWriter(output2.clone()));
|
||||
let writer: Box<dyn libfreemkv::mux::WriteSeek + Send> =
|
||||
Box::new(SharedWriter(output2.clone()));
|
||||
let mut stream2 = MkvStream::create(writer, &dt).unwrap();
|
||||
|
||||
// Write the ES data (SPS+PPS+IDR) as a keyframe PES frame.
|
||||
@@ -842,7 +848,7 @@ fn mkvstream_e2e_h264_produces_valid_mkv() {
|
||||
keyframe: true,
|
||||
data: es_data,
|
||||
};
|
||||
stream2.write(&frame1).unwrap();
|
||||
PesStream::write(&mut stream2, &frame1).unwrap();
|
||||
|
||||
// Write a second non-IDR frame
|
||||
let frame2 = libfreemkv::pes::PesFrame {
|
||||
@@ -851,8 +857,8 @@ fn mkvstream_e2e_h264_produces_valid_mkv() {
|
||||
keyframe: false,
|
||||
data: es_data2,
|
||||
};
|
||||
stream2.write(&frame2).unwrap();
|
||||
stream2.finish().unwrap();
|
||||
PesStream::write(&mut stream2, &frame2).unwrap();
|
||||
PesStream::finish(&mut stream2).unwrap();
|
||||
|
||||
let data = output2.lock().unwrap().clone().into_inner();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user