mirror of
https://github.com/freemkv/libfreemkv.git
synced 2026-10-07 18:13:10 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7788c560d1 | ||
|
|
84e0c2ca69 | ||
|
|
f4a881b25d | ||
|
|
1edda7fedb | ||
|
|
3dd71a0650 | ||
|
|
579c94bd23 | ||
|
|
3b2a68fe7a | ||
|
|
5b56f112a0 | ||
|
|
f27e4c3088 | ||
|
|
b3576f2ae6 | ||
|
|
39f2d0e992 | ||
|
|
50867516b8 | ||
|
|
193e7c1680 | ||
|
|
b6463729e8 | ||
|
|
dffee56102 | ||
|
|
d8f27cdee0 | ||
|
|
fc13268dc6 | ||
|
|
d9b55f02d0 | ||
|
|
35d66d2059 | ||
|
|
d703ce439b | ||
|
|
6112a26d15 | ||
|
|
2a33364253 | ||
|
|
fa3872ddcc | ||
|
|
c427389f36 | ||
|
|
6980f562df | ||
|
|
e3b2c9d850 | ||
|
|
a383200ef1 | ||
|
|
51d15f551e | ||
|
|
523af461a5 | ||
|
|
695b65149e | ||
|
|
d3f9560689 | ||
|
|
1fa5a7d27f | ||
|
|
55219070f1 | ||
|
|
a34c419521 | ||
|
|
b179846f5d | ||
|
|
e110e80e6e | ||
|
|
597fa34099 | ||
|
|
52eeb949e3 | ||
|
|
6ca62b8411 | ||
|
|
04195c27d8 | ||
|
|
5a8f8e54e1 | ||
|
|
1a2c830cf6 | ||
|
|
637c2bfe36 | ||
|
|
0d28357b4d | ||
|
|
5495332f07 | ||
|
|
e22fc6fd47 | ||
|
|
5b98c13e47 | ||
|
|
8f8f1a62a2 | ||
|
|
e3cfd27942 | ||
|
|
aa12bdad62 | ||
|
|
ef3895cdc5 | ||
|
|
2dcf969ac8 | ||
|
|
005f887bf9 | ||
|
|
d7243a6044 | ||
|
|
f1926c38dc | ||
|
|
4709a73c80 | ||
|
|
55bd1ee868 | ||
|
|
5285e8b6ec | ||
|
|
c5af61e46b | ||
|
|
ed73c1ab68 | ||
|
|
5ce2df0888 | ||
|
|
ea591a36a5 | ||
|
|
368e10486e | ||
|
|
a99c4f8487 | ||
|
|
4df470e572 | ||
|
|
a0b13941fe | ||
|
|
275c9da009 | ||
|
|
da183722fa | ||
|
|
4a5424eabe | ||
|
|
293d89c586 | ||
|
|
4840f5134d | ||
|
|
11c8211605 | ||
|
|
1038beaf06 | ||
|
|
f1df57196c | ||
|
|
8ba9dc1c0b | ||
|
|
49b131f39a | ||
|
|
8238ec4ce7 | ||
|
|
4fd5d27df8 | ||
|
|
1fbe272832 | ||
|
|
6bd635725e | ||
|
|
09b77b4dea | ||
|
|
688058b3e8 | ||
|
|
c96bac7977 | ||
|
|
c32acff3e4 | ||
|
|
b58e2d9873 | ||
|
|
a9e802c1c2 | ||
|
|
a876ce846b | ||
|
|
058fd8396f | ||
|
|
5ee28c08b9 | ||
|
|
764230b1eb | ||
|
|
7fb2e07aed | ||
|
|
376aadb335 | ||
|
|
586495b2ee | ||
|
|
72f2224efe | ||
|
|
0bca7a11bd | ||
|
|
72ab714c1e | ||
|
|
d7fb1b35ed | ||
|
|
06be4defd2 | ||
|
|
a832bad697 | ||
|
|
26e2d847d5 | ||
|
|
4442fa2df6 | ||
|
|
23dc55661c | ||
|
|
fbdb50c79f | ||
|
|
7cc74f0087 | ||
|
|
3aa1e528c5 | ||
|
|
7c6b0f82ab | ||
|
|
4226a53e73 | ||
|
|
92f34a289e | ||
|
|
fdbe469d50 | ||
|
|
4ec75a03f2 | ||
|
|
99236ffd55 | ||
|
|
9e3327a569 | ||
|
|
0081955686 | ||
|
|
3244fdd683 | ||
|
|
ef5487e5a5 | ||
|
|
e7dffa63d2 | ||
|
|
e5989c8270 | ||
|
|
e1c938b82a | ||
|
|
ed801a708b | ||
|
|
d94451954c | ||
|
|
385c9f094c | ||
|
|
9f2a13739d | ||
|
|
d9ce69bc9d | ||
|
|
7cd2c937ed | ||
|
|
107524da5e | ||
|
|
055a3c5276 | ||
|
|
50b04790ce | ||
|
|
a00b2c884b | ||
|
|
44e2c64d7e | ||
|
|
d6535b8f57 | ||
|
|
90ab00ed45 | ||
|
|
6bfd7dfd13 | ||
|
|
b9a7f601d4 | ||
|
|
cf0a61f8b4 | ||
|
|
f98f07b2d3 | ||
|
|
b53454fa09 | ||
|
|
f28b6ee6d9 | ||
|
|
d9d778fa64 | ||
|
|
bbfb887a35 | ||
|
|
9884346c24 | ||
|
|
766b7c6636 | ||
|
|
925c30686b | ||
|
|
f52e4c5d22 | ||
|
|
8d67790d2e | ||
|
|
246a439990 | ||
|
|
bd19041648 | ||
|
|
198268b725 | ||
|
|
31424c8203 | ||
|
|
c747ecc589 | ||
|
|
283a561c12 | ||
|
|
2667d68675 | ||
|
|
57f2c22e30 | ||
|
|
5f3545d244 | ||
|
|
6e17ef0859 | ||
|
|
6ec97af104 | ||
|
|
3a6c1aa5a3 | ||
|
|
1ba3264747 | ||
|
|
ae2909fe8d | ||
|
|
6a3d19a453 | ||
|
|
a35596d2d1 | ||
|
|
318f654fed | ||
|
|
5e82ef65de | ||
|
|
8534607329 | ||
|
|
97e1a4cad3 | ||
|
|
3f6bf33de5 | ||
|
|
359301e8fc | ||
|
|
bfa527162a | ||
|
|
45defd47f3 | ||
|
|
fe8933dad2 | ||
|
|
0a3c1b7b70 | ||
|
|
2bcdc97341 | ||
|
|
4863e9c545 | ||
|
|
03db038dd9 | ||
|
|
95001ba2c3 | ||
|
|
a1f4dff6f6 | ||
|
|
7dd5001d45 | ||
|
|
67fe93c0b8 | ||
|
|
5c4f575c46 | ||
|
|
e2cb7a78bb | ||
|
|
a6f1bd19bc | ||
|
|
9e3d0f1383 | ||
|
|
1e6eb0698d | ||
|
|
bd744171e4 | ||
|
|
7de9d4d42c | ||
|
|
9ac0c2fcdf | ||
|
|
3aa27d5d1b | ||
|
|
5083f70cff | ||
|
|
477cf7ee0e | ||
|
|
0b079e27ce | ||
|
|
5f69fd2a44 | ||
|
|
befe6ef69b | ||
|
|
29491d6eaf | ||
|
|
2e602f7e37 | ||
|
|
a126f31b70 | ||
|
|
6ff10df83a | ||
|
|
52b8522a75 | ||
|
|
13eb4331b6 | ||
|
|
1a3f7ce676 | ||
|
|
e1a1f730be | ||
|
|
646c22ae93 | ||
|
|
2f6092a686 | ||
|
|
e5a90a6567 | ||
|
|
afce1031d5 | ||
|
|
2c0bff1303 | ||
|
|
2067ebe485 | ||
|
|
2ce4c20221 | ||
|
|
da3004b120 | ||
|
|
b5ed97801b | ||
|
|
ae76aaf0fa | ||
|
|
2cd4fbead7 | ||
|
|
ebffc6eb88 | ||
|
|
424d3cd4f2 | ||
|
|
d2905ba7bb | ||
|
|
b4de5d343d | ||
|
|
8beeac7df9 | ||
|
|
b33f41e219 | ||
|
|
e85e20f436 | ||
|
|
7bc54be8d3 | ||
|
|
6d9083743b | ||
|
|
c49a68054f | ||
|
|
4bd38787c6 | ||
|
|
b4951b1c5b | ||
|
|
92dc145864 | ||
|
|
36b65f526a | ||
|
|
ca8ebf418f | ||
|
|
34182d956a | ||
|
|
0f967a2084 | ||
|
|
0341995c0d | ||
|
|
fe0ec0c5cf | ||
|
|
022657d481 | ||
|
|
fc8eca44e1 | ||
|
|
43836865be | ||
|
|
5e22199441 | ||
|
|
761b77bbd9 | ||
|
|
cc05ef2a3a | ||
|
|
e27b82ce5b | ||
|
|
8af47e4c19 | ||
|
|
d1f09439a5 | ||
|
|
37e721ee7e | ||
|
|
481b47d90d | ||
|
|
dd7cbd70a9 | ||
|
|
3dea679dac | ||
|
|
64ecca0d12 | ||
|
|
84f6baba38 | ||
|
|
bd5b7795bd | ||
|
|
a57506c1ce | ||
|
|
c33f3e9557 | ||
|
|
166fc4bf8c | ||
|
|
9aaddaa9b8 | ||
|
|
40ec1e1eba | ||
|
|
8843833ea7 | ||
|
|
afae37c8c3 | ||
|
|
d8089ec491 | ||
|
|
ce8132d181 | ||
|
|
71e1f57364 | ||
|
|
d8092ee093 | ||
|
|
dd49d55b11 | ||
|
|
6ba958fe8a | ||
|
|
f30b958408 | ||
|
|
6971f00be2 | ||
|
|
ad86f92d85 | ||
|
|
cce1be29b2 | ||
|
|
081ad4f619 | ||
|
|
43ecad2d43 | ||
|
|
8870efa77b | ||
|
|
a693ef965b | ||
|
|
0596307c19 | ||
|
|
18c6365e08 | ||
|
|
981f30b1b0 | ||
|
|
04086d73a0 | ||
|
|
20e404e59a | ||
|
|
1bea8eb650 | ||
|
|
0165f18fad | ||
|
|
ca3914d537 | ||
|
|
80fcffd190 | ||
|
|
7437af39c8 | ||
|
|
bcd6925170 | ||
|
|
fba1eb189c | ||
|
|
a328ad3cd8 | ||
|
|
9791e60c65 | ||
|
|
8b1ded84cd | ||
|
|
3e4febe7f5 | ||
|
|
aa7afc0c98 | ||
|
|
29f9e916d6 | ||
|
|
30d59063d5 | ||
|
|
6e958b0f71 | ||
|
|
911d260695 | ||
|
|
59d4eb8854 | ||
|
|
8820f7a460 | ||
|
|
8de99bb8e8 | ||
|
|
3e5e570573 | ||
|
|
f95894bba3 | ||
|
|
597f512eca | ||
|
|
8e907393a3 | ||
|
|
c47b9a9c3c | ||
|
|
d7b9d87075 | ||
|
|
e0f40583c4 | ||
|
|
d06a37d3dc | ||
|
|
add9a929a5 | ||
|
|
9bb1d02245 | ||
|
|
dc706e8153 | ||
|
|
0f18906ede | ||
|
|
bd7220c62b | ||
|
|
404c005e0b | ||
|
|
87342290c5 | ||
|
|
8bbf630d82 | ||
|
|
45dddc1810 | ||
|
|
cfa80cb881 | ||
|
|
b510ae9b29 | ||
|
|
9a1c3cc218 | ||
|
|
9ae7d6b38a | ||
|
|
d983985faa | ||
|
|
56fe26c9b8 | ||
|
|
7a83f3244d | ||
|
|
15f5b49fc3 | ||
|
|
ff6004a567 | ||
|
|
bd644d2f60 | ||
|
|
ab63950ef1 | ||
|
|
cab87ffdf4 | ||
|
|
ccb1fadedf | ||
|
|
610f1b62b6 | ||
|
|
f400e4fc72 | ||
|
|
5a9980cb6b | ||
|
|
2f48877992 | ||
|
|
7c59d063ee | ||
|
|
e4ffe7cc91 | ||
|
|
2e23e848df | ||
|
|
fbdfccdd83 | ||
|
|
2f52188f77 | ||
|
|
252e58cab0 | ||
|
|
3b96976a7a | ||
|
|
3385de5704 | ||
|
|
e9c907a2ca | ||
|
|
db1f6bc446 | ||
|
|
c43ba06b5b | ||
|
|
077e4d018f | ||
|
|
f8b5a1eaf1 | ||
|
|
fb1c35e653 | ||
|
|
9268ea2fdb | ||
|
|
9078aa8718 | ||
|
|
c55e6991b8 | ||
|
|
ca931b6522 | ||
|
|
d4d98ce593 |
@@ -6,23 +6,37 @@ on:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
with:
|
||||
components: clippy, rustfmt
|
||||
- run: cargo fmt --check
|
||||
# libfreemkv is a library — Cargo.lock is gitignored. --locked
|
||||
# would always fail on a fresh runner because there's no committed
|
||||
# lockfile to lock against. The binary crates (freemkv, autorip,
|
||||
# bdemu) track Cargo.lock and DO use --locked.
|
||||
- run: cargo clippy -- -D warnings
|
||||
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- run: cargo test
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
- run: cargo test --tests
|
||||
|
||||
check-macos:
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
- run: cargo check
|
||||
|
||||
check-windows:
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
- run: cargo check
|
||||
|
||||
@@ -27,7 +27,9 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
# libfreemkv is a library — Cargo.lock isn't tracked, so --locked
|
||||
# would always fail (no lockfile to lock against on a fresh runner).
|
||||
- run: cargo test
|
||||
|
||||
publish:
|
||||
@@ -35,7 +37,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
- name: Publish to crates.io
|
||||
run: cargo publish
|
||||
env:
|
||||
|
||||
+1830
File diff suppressed because it is too large
Load Diff
+20
-2
@@ -1,7 +1,8 @@
|
||||
[package]
|
||||
name = "libfreemkv"
|
||||
version = "0.8.0"
|
||||
edition = "2021"
|
||||
version = "0.23.1"
|
||||
edition = "2024"
|
||||
rust-version = "1.86"
|
||||
license = "AGPL-3.0-only"
|
||||
description = "Open source raw disc access library for optical drives"
|
||||
repository = "https://github.com/freemkv/libfreemkv"
|
||||
@@ -23,6 +24,16 @@ rand = "0.8"
|
||||
cmac = "0.7"
|
||||
zip = { version = "2", default-features = false, features = ["deflate"] }
|
||||
base64 = "0.22.1"
|
||||
# Trace-level instrumentation for Disc::copy + SgIoTransport::execute. Permitted
|
||||
# under project docs ("Acceptable strings: debug/trace logging"). Consumers (autorip)
|
||||
# wire a tracing subscriber and pipe events into the JSONL debug log.
|
||||
tracing = "0.1"
|
||||
# Bounded MPSC channel with kernel-wakeup send_timeout. Used by `io::pipeline`
|
||||
# so the halt-aware send/finish loops can BLOCK on consumer drain instead of
|
||||
# polling — the 50 ms poll cadence of the previous mpsc-based impl capped mux
|
||||
# throughput at ~1 MB/s (see (internal)/memory/
|
||||
# feedback_send_with_halt_poll_throttle.md, 0.21.7).
|
||||
crossbeam-channel = "0.5"
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
libc = "0.2"
|
||||
@@ -30,3 +41,10 @@ libc = "0.2"
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
[[bench]]
|
||||
name = "sgio_read"
|
||||
harness = false
|
||||
|
||||
|
||||
-30
@@ -1,30 +0,0 @@
|
||||
# libfreemkv — Feature List
|
||||
|
||||
## v0.7.1 (current)
|
||||
|
||||
### Done
|
||||
- [x] Drive access: open, identify, unlock, firmware upload, speed calibration, eject
|
||||
- [x] 206 bundled drive profiles (MediaTek MT1959 A + B variants)
|
||||
- [x] SCSI transport: Linux SG_IO, macOS IOKit, Windows SPTI
|
||||
- [x] UDF 2.50 filesystem parser (metadata partitions, Blu-ray profile)
|
||||
- [x] MPLS playlist parser (play items, STN table, secondary streams)
|
||||
- [x] CLPI clip info parser (EP map, sector extents)
|
||||
- [x] AACS 1.0 decryption (4 VUK paths: KEYDB, media key, processing key, device key)
|
||||
- [x] AACS 2.0 SCSI handshake (P-256/SHA-256 ECDH, bus decryption, read data key)
|
||||
- [x] KEYDB.cfg download, verify, save (raw TCP, zero HTTP deps)
|
||||
- [x] Content reading with adaptive batch sizing, error recovery, 12+ MB/s
|
||||
- [x] Stream labels: 5 BD-J format parsers (Paramount, Criterion, Pixelogic, CTRM, Deluxe)
|
||||
- [x] MKV muxer: 8 codec parsers (H.264, HEVC, AC-3, DTS, TrueHD, PGS, VC-1, LPCM)
|
||||
- [x] SectorReader trait: decouples disc scanning from SCSI
|
||||
- [x] 7 stream types: Disc, ISO, MKV, M2TS, Network, Stdio, Null
|
||||
- [x] IOStream trait with URL-based resolver (scheme://path)
|
||||
- [x] FMKV metadata header for M2TS and network streams
|
||||
- [x] Numeric error codes only (no English text in library)
|
||||
- [x] Event system for progress callbacks
|
||||
|
||||
### Planned
|
||||
- [ ] Windows testing on real hardware
|
||||
- [ ] Pioneer Renesas platform support (48 drives, need GET_CONFIG 010C)
|
||||
- [ ] DVD CSS decryption
|
||||
- [ ] TranscodeStream (ffmpeg integration)
|
||||
- [ ] ISO write with BD-compliant UDF structure
|
||||
@@ -0,0 +1,20 @@
|
||||
# libfreemkv — local dev helper.
|
||||
# Mirrors the cross-crate scripts in (internal)/scripts/test-all.sh
|
||||
# but scoped to this single crate.
|
||||
|
||||
.PHONY: test build check ci clean
|
||||
|
||||
test:
|
||||
cargo test --tests
|
||||
|
||||
build:
|
||||
cargo build --release
|
||||
|
||||
check:
|
||||
cargo fmt --check
|
||||
cargo clippy --all-targets -- -D warnings
|
||||
|
||||
ci: check build test
|
||||
|
||||
clean:
|
||||
cargo clean
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
# libfreemkv
|
||||
|
||||
Rust library for 4K UHD / Blu-ray optical drives. Drive access, disc scanning, stream labels, AACS decryption, KEYDB updates, and content reading in one crate. Bundled drive profiles — no external files needed.
|
||||
Rust library for 4K UHD / Blu-ray / DVD optical drives. Drive access, disc scanning, stream labels, AACS decryption, CSS decryption, KEYDB updates, and content reading in one crate. Bundled drive profiles — no external files needed.
|
||||
|
||||
**12+ MB/s** sustained read speeds on BD. Full init: unlock, firmware upload, speed calibration — all from pure Rust.
|
||||
|
||||
@@ -18,33 +18,83 @@ Part of the [freemkv](https://github.com/freemkv) project.
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
libfreemkv = "0.6"
|
||||
libfreemkv = "0.18"
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```rust
|
||||
use libfreemkv::{DriveSession, Disc, ScanOptions};
|
||||
use libfreemkv::{Drive, Disc, ScanOptions};
|
||||
use std::path::Path;
|
||||
|
||||
// Open drive — profiles are bundled, auto-identified
|
||||
let mut session = DriveSession::open(Path::new("/dev/sr0"))?;
|
||||
session.wait_ready()?; // wait for disc
|
||||
session.init()?; // unlock + firmware upload
|
||||
session.probe_disc()?; // probe disc surface for optimal speeds
|
||||
let mut drive = Drive::open(Path::new("/dev/sg4"))?;
|
||||
drive.wait_ready()?; // wait for disc
|
||||
drive.init()?; // unlock + firmware upload
|
||||
drive.probe_disc()?; // probe disc surface for optimal speeds
|
||||
|
||||
// Scan disc — UDF, playlists, streams, AACS (all automatic)
|
||||
let disc = Disc::scan(&mut session, &ScanOptions::default())?;
|
||||
let disc = Disc::scan(&mut drive, &ScanOptions::default())?;
|
||||
|
||||
for title in &disc.titles {
|
||||
println!("{} — {} streams", title.duration_display(), title.streams.len());
|
||||
}
|
||||
|
||||
// Read content (decrypted transparently if AACS keys available)
|
||||
let mut reader = disc.open_title(&mut session, 0)?;
|
||||
while let Some(unit) = reader.read_unit()? {
|
||||
// 6144 bytes of content per aligned unit
|
||||
// Stream pipeline — read PES frames from any source, write to any output
|
||||
let opts = libfreemkv::InputOptions::default();
|
||||
let mut input = libfreemkv::input("iso://Disc.iso", &opts)?;
|
||||
let title = input.info().clone();
|
||||
let mut output = libfreemkv::output("mkv://Movie.mkv", &title)?;
|
||||
while let Ok(Some(frame)) = input.read() {
|
||||
output.write(&frame)?;
|
||||
}
|
||||
output.finish()?;
|
||||
```
|
||||
|
||||
### Multi-pass recovery rip
|
||||
|
||||
For damaged discs the library exposes two flat verbs — `Disc::sweep` for the
|
||||
forward Pass 1 and `Disc::patch` for retrying bad ranges. The library never
|
||||
loops; the multipass policy is the caller's job. See
|
||||
[`docs/rip-recovery.md`](docs/rip-recovery.md) and the design notes in
|
||||
`(internal)/memory/0_18_redesign.md`.
|
||||
|
||||
```rust
|
||||
use libfreemkv::{SweepOptions, PatchOptions};
|
||||
use libfreemkv::disc::{mapfile, mapfile_path_for};
|
||||
use std::path::Path;
|
||||
|
||||
let iso = Path::new("disc.iso");
|
||||
|
||||
// Pass 1: disc → ISO. Skip-on-error, zero-fill, write the sidecar mapfile.
|
||||
disc.sweep(&mut drive, iso, &SweepOptions {
|
||||
decrypt: true,
|
||||
resume: false,
|
||||
batch_sectors: None,
|
||||
skip_on_error: true,
|
||||
progress: None,
|
||||
halt: None,
|
||||
})?;
|
||||
|
||||
// Pass 2..N: retry every non-finished range. Idempotent.
|
||||
loop {
|
||||
let map = mapfile::Mapfile::load(&mapfile_path_for(iso))?;
|
||||
let stats = map.stats();
|
||||
if stats.bytes_pending + stats.bytes_unreadable == 0 { break; }
|
||||
|
||||
let outcome = disc.patch(&mut drive, iso, &PatchOptions {
|
||||
decrypt: true,
|
||||
block_sectors: None,
|
||||
full_recovery: true,
|
||||
reverse: true,
|
||||
wedged_threshold: 50,
|
||||
progress: None,
|
||||
halt: None,
|
||||
})?;
|
||||
if outcome.bytes_recovered_this_pass == 0 { break; }
|
||||
}
|
||||
|
||||
// Mux from the ISO via the normal stream pipeline (no drive involvement).
|
||||
```
|
||||
|
||||
## What It Does
|
||||
@@ -55,7 +105,7 @@ while let Some(unit) = reader.read_unit()? {
|
||||
- **Stream labels** — 5 BD-J format parsers (Paramount, Criterion, Pixelogic, CTRM, Deluxe)
|
||||
- **AACS decryption** — transparent key resolution and content decrypt (1.0 + 2.0 bus decryption)
|
||||
- **KEYDB updates** — download, verify, save from any HTTP URL (zero deps, raw TCP)
|
||||
- **Content reading** — adaptive batch reads with automatic decryption and error recovery
|
||||
- **Content reading** — adaptive batch reads with automatic decryption
|
||||
- **Stream I/O** — unified stream pipeline for reading and writing any format
|
||||
|
||||
### Streams
|
||||
@@ -63,32 +113,45 @@ while let Some(unit) = reader.read_unit()? {
|
||||
| Stream | Input | Output | Transport |
|
||||
|--------|-------|--------|-----------|
|
||||
| DiscStream | Yes | -- | Optical drive via SCSI |
|
||||
| IsoStream | Yes | -- | Blu-ray ISO image file |
|
||||
| IsoStream | Yes | -- | Blu-ray ISO image file (read via stream pipeline; written via `Disc::sweep()`) |
|
||||
| MkvStream | Yes | Yes | Matroska container |
|
||||
| M2tsStream | Yes | Yes | BD transport stream with FMKV metadata header |
|
||||
| NetworkStream | Yes (listen) | Yes (connect) | TCP with FMKV metadata header |
|
||||
| StdioStream | Yes (stdin) | Yes (stdout) | Raw byte pipe |
|
||||
| NullStream | -- | Yes | Discard sink (byte counter for benchmarks) |
|
||||
|
||||
All streams implement the `IOStream` trait. `open_input()` and `open_output()` resolve URL strings to stream instances. All URLs use the `scheme://path` format — bare paths are rejected.
|
||||
Streams implement `FrameSource` (read) and/or `FrameSink` (write); direction is type-checked. `input()` / `output()` resolve URL strings to PES stream instances. All URLs use the `scheme://path` format — bare paths are rejected.
|
||||
|
||||
AACS decryption requires a KEYDB.cfg file. If available at `~/.config/aacs/KEYDB.cfg` or passed via `ScanOptions`, the library handles everything — handshake, key derivation, and per-sector decryption — without the application needing to know anything about encryption.
|
||||
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
DriveSession — open any drive, identify, init (optional), read sectors
|
||||
├── ScsiTransport — SG_IO (Linux), IOKit (macOS)
|
||||
Drive — open, identify, init, unlock, single-shot read
|
||||
├── ScsiTransport — SG_IO (Linux), IOKit (macOS), SPTI (Windows)
|
||||
├── DriveProfile — per-drive unlock parameters (bundled)
|
||||
└── PlatformDriver — MediaTek (supported), Renesas (planned)
|
||||
|
||||
Disc — scan titles, streams, AACS state
|
||||
Disc — scan titles, streams, AACS/CSS state
|
||||
├── UDF reader — Blu-ray UDF 2.50 with metadata partitions
|
||||
├── MPLS parser — playlists → titles + clips + streams
|
||||
├── CLPI parser — clip info → EP map → sector extents
|
||||
├── IFO parser — DVD title sets, PGC chains, cell addresses
|
||||
├── Labels — 5 BD-J format parsers (detect + parse)
|
||||
├── AACS — key resolution + content decryption
|
||||
├── CSS — DVD CSS cipher (table-driven, no keys needed)
|
||||
└── KEYDB — download + verify + save
|
||||
|
||||
Streams — unified PES pipeline
|
||||
├── FrameSource — read() PES frames (direction-typed)
|
||||
├── FrameSink — write() PES frames (direction-typed)
|
||||
├── DiscStream — sectors → decrypt → TS demux → PES
|
||||
├── IsoStream — ISO file → decrypt → TS demux → PES
|
||||
├── MkvStream — MKV mux/demux
|
||||
├── M2tsStream — BD transport stream
|
||||
├── NetworkStream — TCP with FMKV metadata header
|
||||
├── StdioStream — stdin/stdout pipe
|
||||
└── NullStream — discard sink
|
||||
```
|
||||
|
||||
See [docs/](docs/) for detailed technical documentation on each module.
|
||||
@@ -118,7 +181,7 @@ All errors are structured with numeric codes. No user-facing English text — ap
|
||||
|
||||
## Contributing
|
||||
|
||||
Run `freemkv drive-info --share` with the [freemkv CLI](https://github.com/freemkv/freemkv) to contribute your drive's profile.
|
||||
Run `freemkv info disc:// --share` with the [freemkv CLI](https://github.com/freemkv/freemkv) to contribute your drive's profile.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
# Troubleshooting Guide
|
||||
|
||||
Common problems and solutions for optical drive ripping with freemkv.
|
||||
|
||||
---
|
||||
|
||||
## 1. USB-SATA Bridge Issues
|
||||
|
||||
This is the single most common source of problems when ripping discs over USB.
|
||||
|
||||
### Symptoms
|
||||
|
||||
- The drive disappears mid-rip. The ripping tool reports the device is gone, and `ls /dev/sg*` no longer shows it.
|
||||
- The device re-enumerates under a different name: `sg4` becomes `sg5`, then `sg7`, then `sg11` after each USB port reset.
|
||||
- `dmesg` shows USB port resets: `usb X-Y: reset high-speed USB device`, `xhci_hcd 0000:00:14.0: Cannot enable. Maybe the USB cable is bad?`, or `usb-storage: device reset failed`.
|
||||
- The SCSI layer reports `host_status=7` (Linux USB transport error) in sense data.
|
||||
- The drive works fine for reading data discs or burning, but crashes when hitting damaged sectors during a rip.
|
||||
- After the crash, the drive is completely invisible until physically unplugged and reconnected.
|
||||
|
||||
### Root Cause
|
||||
|
||||
USB-SATA bridges translate between the USB Mass Storage protocol (BOT or UAS) and the drive's native SATA interface. When the optical drive encounters an unreadable sector, it returns a SCSI CHECK CONDITION with sense key 0x03 (MEDIUM ERROR). Some bridge chipsets -- particularly the Initio INIC-36xx family -- have buggy firmware that mishandles this error response.
|
||||
|
||||
Specific failure modes:
|
||||
|
||||
- **Incorrect residue reporting.** The bridge claims a different number of bytes transferred than what actually occurred. The Linux USB storage driver sees this discrepancy as a protocol violation and resets the port to recover. The `US_FL_IGNORE_RESIDUE` quirk exists specifically for this class of bug (see `drivers/usb/storage/transport.c` in the Linux kernel).
|
||||
- **Bridge firmware crash.** On some Initio bridges, a malformed SCSI error response from the drive causes the bridge MCU to hang entirely. The USB controller sees the device stop responding and initiates a port reset. The bridge recovers (it re-enumerates), but the rip is dead -- all state is lost.
|
||||
- **Sense data corruption.** The bridge forwards garbled or truncated sense data to the host, which the SCSI midlayer cannot parse, leading to a transport reset.
|
||||
|
||||
This is a hardware + firmware problem, not a software bug. The same drive connected via direct SATA does not exhibit these symptoms.
|
||||
|
||||
### Known Affected Bridges
|
||||
|
||||
| Chipset | USB IDs | Notes |
|
||||
|---------|---------|-------|
|
||||
| Initio INIC-3609 | `13fd:3609` | Very common in cheap SATA-to-USB enclosures. Highly problematic. |
|
||||
| Initio INIC-3619 | `13fd:3940` | Same firmware family as INIC-3609. |
|
||||
| Initio INIC-3069 | `13fd:0840` | Older variant, same residue bug. |
|
||||
| ASMedia ASM1051 | `174c:5106` | Early ASM SATA bridge. Residue issues on error paths. |
|
||||
| JMicron JMB36x | `152d:0561` | Some firmware versions. Not all JMicroon chips are affected. |
|
||||
|
||||
If your drive came in a pre-built external enclosure (Vantec, Sabrent, OWC, etc.), it almost certainly uses one of these bridge chips internally.
|
||||
|
||||
### The Fix: USB Storage Quirk
|
||||
|
||||
Apply the `US_FL_IGNORE_RESIDUE` kernel quirk for your bridge. This tells the Linux USB storage driver to ignore the residue field in SCSI response frames, preventing the port reset on mismatched byte counts.
|
||||
|
||||
**Step 1: Identify your bridge's vendor:product ID.**
|
||||
|
||||
```bash
|
||||
lsusb
|
||||
```
|
||||
|
||||
Look for your drive's entry. Example output:
|
||||
|
||||
```
|
||||
Bus 002 Device 005: ID 13fd:0840 Initio Corporation INIC-3609
|
||||
```
|
||||
|
||||
Here the vendor ID is `13fd` and the product ID is `0840`.
|
||||
|
||||
**Step 2: Apply the quirk at runtime.**
|
||||
|
||||
```bash
|
||||
echo "13fd:0840:i" > /sys/module/usb_storage/parameters/quirks
|
||||
```
|
||||
|
||||
Replace `13fd:0840` with your device's actual IDs. The `:i` flag means `US_FL_IGNORE_RESIDUE`.
|
||||
|
||||
You can combine multiple flags. Common additions:
|
||||
|
||||
- `:i` -- ignore residue (`US_FL_IGNORE_RESIDUE`)
|
||||
- `:u` -- force BOT mode instead of UAS, for bridges with UAS bugs
|
||||
|
||||
**Step 3: Reconnect the drive.** Unplug and replug the USB cable, or bind/unbind the device. The quirk is applied per-module-load, so existing sessions may need the drive reconnected.
|
||||
|
||||
### Making It Persistent
|
||||
|
||||
Add the quirk to your kernel boot parameters so it survives reboots.
|
||||
|
||||
Edit `/etc/default/grub` (GRUB) and add to `GRUB_CMDLINE_LINUX_DEFAULT`:
|
||||
|
||||
```
|
||||
GRUB_CMDLINE_LINUX_DEFAULT="quiet usb_storage.quirks=13fd:0840:i"
|
||||
```
|
||||
|
||||
Then rebuild the GRUB config:
|
||||
|
||||
```bash
|
||||
sudo update-grub
|
||||
```
|
||||
|
||||
For systemd-boot, add to your loader entry or `/etc/kernel/cmdline`:
|
||||
|
||||
```
|
||||
usb_storage.quirks=13fd:0840:i
|
||||
```
|
||||
|
||||
Multiple devices can be separated by commas:
|
||||
|
||||
```
|
||||
usb_storage.quirks=13fd:0840:i,174c:5106:u
|
||||
```
|
||||
|
||||
### Recommended Bridges
|
||||
|
||||
If you are buying a USB-SATA adapter or enclosure for optical drive use:
|
||||
|
||||
| Bridge | USB IDs | Notes |
|
||||
|--------|---------|-------|
|
||||
| ASMedia ASM1153 | `174c:1153` | Reliable. Widely available in SATA-USB 3.0 cables. |
|
||||
| JMicron JMS578 | `152d:0578` | Good firmware. Supports UASP. |
|
||||
| Icy Box IB-AC640-C3 | N/A | Uses a known-good bridge internally. Plug-and-play. |
|
||||
|
||||
Avoid any enclosure or adapter listing an Initio chipset.
|
||||
|
||||
### Best Solution: Direct SATA
|
||||
|
||||
Connect your optical drive directly to a motherboard SATA port. This eliminates the USB-SATA bridge entirely and is the most reliable configuration:
|
||||
|
||||
- No USB protocol overhead or translation errors.
|
||||
- No bridge firmware bugs.
|
||||
- No port resets or re-enumeration.
|
||||
- Full SATA error recovery handled natively by the kernel's libata driver.
|
||||
- Sustained read speeds are limited only by the drive, not the USB bus.
|
||||
|
||||
If your machine has a free SATA port, use it.
|
||||
|
||||
---
|
||||
|
||||
## 2. Damaged Disc Handling
|
||||
|
||||
### Symptoms
|
||||
|
||||
- SCSI MEDIUM ERROR (sense key 0x03) at specific LBAs. `dmesg` shows `sr X:0:0:0: [srY] Unrecoverable read error` or similar.
|
||||
- Read speed drops to near zero when approaching a damaged area.
|
||||
- The drive makes audible retrying noises (laser repositioning, spindle speed changes).
|
||||
- On USB-connected drives: the bridge crashes (see section 1 above) when the drive returns the error.
|
||||
|
||||
### How freemkv Handles This
|
||||
|
||||
freemkv uses a three-layer recovery model. See [`docs/rip-recovery.md`](docs/rip-recovery.md) for full details.
|
||||
|
||||
- **Pass 1 (Disc::copy):** Fast sweep with 64 KB reads. On failure, zero-fills the block and skips forward. Writes a ddrescue-format mapfile for later retry.
|
||||
- **Pass 2+ (Disc::patch):** Targeted re-reads of bad ranges with a long 30-second timeout per CDB. The drive firmware performs its own ECC and laser power retries within that window.
|
||||
- **In-stream (DiscStream):** Adaptive batch halving -- reduces request size on failure to isolate bad sectors within a larger block.
|
||||
|
||||
This means a disc with some bad sectors will still produce a usable ISO. The damaged areas are zero-filled in pass 1 and retried in subsequent passes. Structure-protected sectors (deliberate unreadable regions from copy protection) will never yield, which is expected.
|
||||
|
||||
### The Drive Taint Issue (LG BU40N)
|
||||
|
||||
Some drives, notably the LG BU40N, exhibit a "taint" behavior after encountering MEDIUM ERRORs:
|
||||
|
||||
1. The drive hits a damaged sector and returns a MEDIUM ERROR.
|
||||
2. From that point forward, **all subsequent reads fail** -- even reads to sectors that were previously successful.
|
||||
3. The only recovery is to physically unplug and reconnect the drive (or power-cycle it).
|
||||
|
||||
This is not a freemkv bug. It is a drive firmware behavior triggered by the interaction between the drive's internal error recovery and the USB-SATA bridge's handling of the error response. The drive firmware enters a degraded state that it does not recover from without a power cycle.
|
||||
|
||||
Workarounds:
|
||||
|
||||
- **Use a direct SATA connection.** This eliminates the bridge interaction that triggers the taint.
|
||||
- **Use a different bridge.** The ASM1153 and JMS578 are less likely to trigger this behavior.
|
||||
- **Accept the partial ISO.** freemkv's skip-forward recovery will zero-fill the unreadable blocks and continue. The resulting ISO may be playable with minor glitches in the affected areas.
|
||||
- **Physical replug between retry passes.** If running multi-pass patch, replug the drive between passes to clear the taint state.
|
||||
|
||||
freemkv deliberately does not attempt inline SCSI resets or eject cycles to recover from this state, because those operations were found to make the problem worse on affected hardware (see the design rationale in [`docs/rip-recovery.md`](docs/rip-recovery.md)).
|
||||
|
||||
---
|
||||
|
||||
## 3. Drive Not Detected
|
||||
|
||||
### Check Hardware Visibility
|
||||
|
||||
```bash
|
||||
lsusb
|
||||
```
|
||||
|
||||
Verify the drive appears in the USB device list. If it does not show up, the drive is not visible to the host at all -- check cables, power, and USB port.
|
||||
|
||||
```bash
|
||||
ls /dev/sg*
|
||||
```
|
||||
|
||||
On Linux, optical drives appear as `/dev/sg*` devices (the SCSI Generic interface). freemkv uses `/dev/sg*`, not `/dev/sr*`. If `lsusb` shows the device but no `/dev/sg*` entry exists, the `sg` kernel module may not be loaded:
|
||||
|
||||
```bash
|
||||
sudo modprobe sg
|
||||
```
|
||||
|
||||
### Check Kernel Messages
|
||||
|
||||
```bash
|
||||
dmesg | grep -i usb | tail -30
|
||||
dmesg | grep -i sg | tail -10
|
||||
```
|
||||
|
||||
Look for:
|
||||
- USB enumeration errors or failed port resets.
|
||||
- `sg_add` messages confirming the sg device was registered.
|
||||
- Permission denied or access errors.
|
||||
|
||||
### Permission Issues
|
||||
|
||||
On most Linux distributions, `/dev/sg*` devices are owned by `root:disk` or `root:cdrom` with restricted permissions. Running freemkv as an unprivileged user will fail with permission errors.
|
||||
|
||||
Options:
|
||||
|
||||
- Add your user to the appropriate group:
|
||||
|
||||
```bash
|
||||
sudo usermod -aG disk $USER
|
||||
```
|
||||
|
||||
Then log out and back in for the change to take effect. On some distributions the group is `cdrom` or `optical` instead of `disk`.
|
||||
|
||||
- Run with elevated privileges:
|
||||
|
||||
```bash
|
||||
sudo freemkv ...
|
||||
```
|
||||
|
||||
- Install a udev rule for persistent per-device permissions. Create `/etc/udev/rules.d/99-sg-optical.rules`:
|
||||
|
||||
```
|
||||
SUBSYSTEM=="scsi_generic", ATTRS{type}=="5", MODE="0666"
|
||||
```
|
||||
|
||||
Then reload udev rules:
|
||||
|
||||
```bash
|
||||
sudo udevadm control --reload-rules && sudo udevadm trigger
|
||||
```
|
||||
|
||||
### Spin-Up Delay
|
||||
|
||||
Optical drives take 30-60 seconds to spin up and become ready after hot-plug or disc insertion. During this window, SCSI commands may return NOT READY or timeout.
|
||||
|
||||
freemkv's `Drive::wait_ready()` handles this automatically by polling with TEST UNIT READY until the drive responds. If you are writing your own code using the library, always call `wait_ready()` before `init()`:
|
||||
|
||||
```rust
|
||||
let mut drive = Drive::open(Path::new("/dev/sg4"))?;
|
||||
drive.wait_ready()?; // blocks until disc is ready, up to 30s
|
||||
drive.init()?;
|
||||
```
|
||||
|
||||
If the drive was just plugged in, wait a full minute before concluding it is not detected.
|
||||
|
||||
---
|
||||
|
||||
## 4. How to Identify Your USB-SATA Bridge
|
||||
|
||||
If you are experiencing the issues described in section 1, you need to know which bridge chipset your adapter or enclosure uses.
|
||||
|
||||
### Step 1: Find the Device
|
||||
|
||||
```bash
|
||||
lsusb
|
||||
```
|
||||
|
||||
Look for entries matching your drive or enclosure. Bridges may appear under their own manufacturer name or as a generic SATA device. Common examples:
|
||||
|
||||
```
|
||||
Bus 002 Device 005: ID 13fd:0840 Initio Corporation
|
||||
Bus 002 Device 006: ID 174c:1153 ASMedia Technology Inc. ASM1153
|
||||
Bus 002 Device 007: ID 152d:0578 JMicron Technology Corp. JMS578
|
||||
```
|
||||
|
||||
### Step 2: Match the IDs
|
||||
|
||||
| Vendor | Product ID | Chipset | Status |
|
||||
|--------|-----------|---------|--------|
|
||||
| `13fd` | `3609` | Initio INIC-3609 | Affected. Apply quirk. |
|
||||
| `13fd` | `3940` | Initio INIC-3619 | Affected. Apply quirk. |
|
||||
| `13fd` | `0840` | Initio INIC-3069 | Affected. Apply quirk. |
|
||||
| `174c` | `5106` | ASMedia ASM1051 | Affected (early firmware). Apply quirk. |
|
||||
| `174c` | `1153` | ASMedia ASM1153 | Good. No quirk needed. |
|
||||
| `152d` | `0561` | JMicron JMB36x | Affected (some firmware). Apply quirk if issues occur. |
|
||||
| `152d` | `0578` | JMicron JMS578 | Good. No quirk needed. |
|
||||
|
||||
### Step 3: Check dmesg for the Bridge Name
|
||||
|
||||
```bash
|
||||
dmesg | grep -i "usb-storage\|uas\|initio\|asmedia\|jmicron"
|
||||
```
|
||||
|
||||
This often reveals the bridge chipset even when `lsusb` shows a generic name.
|
||||
|
||||
### Step 4: If the Enclosure Is Sealed
|
||||
|
||||
Many external drive enclosures (Vantec NexStar, Sabrent, OWC, etc.) do not advertise the bridge chipset on the packaging. In this case:
|
||||
|
||||
1. Check `lsusb` while the enclosure is connected.
|
||||
2. Search the vendor:product ID online -- there are community-maintained lists of which chipsets popular enclosures use.
|
||||
3. If you cannot determine the chipset and are experiencing bridge crashes, assume it is an Initio and apply the quirk with its IDs.
|
||||
4. The definitive test: connect the bare drive to a motherboard SATA port. If the problems disappear, the bridge was the cause.
|
||||
|
||||
---
|
||||
|
||||
## 5. General Debugging Checklist
|
||||
|
||||
When something goes wrong during a rip, gather this information before filing an issue:
|
||||
|
||||
1. **freemkv version:** `freemkv --version` or the crate version in `Cargo.toml`.
|
||||
2. **Drive model:** from the drive label, or from `freemkv info`.
|
||||
3. **Connection type:** USB (with bridge chipset if known) or direct SATA.
|
||||
4. **Operating system and kernel:** `uname -a`.
|
||||
5. **Kernel messages during the failure:** `dmesg | tail -50` immediately after the crash.
|
||||
6. **SCSI device:** which `/dev/sg*` the drive was on, and whether it changed after the failure.
|
||||
7. **The disc:** title, format (BD/DVD/UHD), condition.
|
||||
|
||||
Include all of the above in bug reports. SCSI transport errors that resolve with the `US_FL_IGNORE_RESIDUE` quirk or by switching to direct SATA are bridge firmware bugs, not freemkv bugs.
|
||||
@@ -0,0 +1,87 @@
|
||||
// Mimics ISO dump exactly — read + write + progress
|
||||
use libfreemkv::Drive;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() {
|
||||
let device = std::env::args()
|
||||
.skip(1)
|
||||
.find(|a| !a.starts_with('-'))
|
||||
.unwrap_or_else(|| match libfreemkv::find_drive() {
|
||||
Some(d) => d.device_path().to_string(),
|
||||
None => {
|
||||
eprintln!("No drives found");
|
||||
std::process::exit(1);
|
||||
}
|
||||
});
|
||||
|
||||
let mut drive = Drive::open(Path::new(&device)).unwrap_or_else(|e| {
|
||||
eprintln!("Cannot open {}: {}", device, e);
|
||||
std::process::exit(1);
|
||||
});
|
||||
eprintln!("wait_ready...");
|
||||
let _ = drive.wait_ready();
|
||||
eprintln!("read_capacity...");
|
||||
let cap = drive.read_capacity().unwrap();
|
||||
eprintln!("capacity: {} sectors", cap);
|
||||
|
||||
let batch = libfreemkv::disc::detect_max_batch_sectors(drive.device_path());
|
||||
let mut buf = vec![0u8; batch as usize * 2048];
|
||||
|
||||
// Open /dev/null writer like ISO dump does
|
||||
let file = std::fs::File::create("/dev/null").unwrap();
|
||||
let mut writer = std::io::BufWriter::with_capacity(4 * 1024 * 1024, file);
|
||||
|
||||
eprintln!(
|
||||
"Reading 1000 batches ({:.1} MB) with write + progress...",
|
||||
1000.0 * batch as f64 * 2048.0 / 1_048_576.0
|
||||
);
|
||||
|
||||
let start = Instant::now();
|
||||
let mut ok = 0u32;
|
||||
let mut fail = 0u32;
|
||||
let mut bytes: u64 = 0;
|
||||
|
||||
// Recovery flag: true matches pre-0.11.13 bench behavior — full SCSI
|
||||
// ECC retry loop on errors (slower, what the rip path used before the
|
||||
// adaptive batch sizer landed). Flip to `false` for the fast-fail path
|
||||
// that current rips use; benches are configurable via this constant.
|
||||
const READ_WITH_RECOVERY: bool = true;
|
||||
|
||||
for i in 0..1000u32 {
|
||||
let lba = i * batch as u32;
|
||||
match drive.read(lba, batch, &mut buf, READ_WITH_RECOVERY) {
|
||||
Ok(_) => {
|
||||
writer.write_all(&buf).unwrap();
|
||||
ok += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
fail += 1;
|
||||
if fail <= 5 {
|
||||
eprintln!(" FAIL LBA {}: {}", lba, e);
|
||||
}
|
||||
buf.fill(0);
|
||||
writer.write_all(&buf).unwrap();
|
||||
}
|
||||
}
|
||||
bytes += buf.len() as u64;
|
||||
|
||||
if i % 50 == 0 && i > 0 {
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
let mb = bytes as f64 / 1_048_576.0;
|
||||
eprint!("\r {:.1} MB | {:.1} MB/s ", mb, mb / elapsed);
|
||||
}
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed().as_secs_f64();
|
||||
let mb = ok as f64 * batch as f64 * 2048.0 / 1_048_576.0;
|
||||
eprintln!(
|
||||
"\n{} ok, {} fail, {:.1} MB in {:.1}s = {:.1} MB/s",
|
||||
ok,
|
||||
fail,
|
||||
mb,
|
||||
elapsed,
|
||||
mb / elapsed
|
||||
);
|
||||
}
|
||||
@@ -3,5 +3,34 @@ fn main() {
|
||||
if target == "macos" {
|
||||
println!("cargo:rustc-link-lib=framework=IOKit");
|
||||
println!("cargo:rustc-link-lib=framework=CoreFoundation");
|
||||
|
||||
let out_dir = std::env::var("OUT_DIR").unwrap();
|
||||
let obj = format!("{out_dir}/macos_shim.o");
|
||||
let lib = format!("{out_dir}/libmacos_scsi.a");
|
||||
|
||||
std::process::Command::new("cc")
|
||||
.args([
|
||||
"-c",
|
||||
"src/scsi/macos_shim.c",
|
||||
"-o",
|
||||
&obj,
|
||||
"-framework",
|
||||
"IOKit",
|
||||
"-framework",
|
||||
"CoreFoundation",
|
||||
"-Wall",
|
||||
"-O2",
|
||||
])
|
||||
.status()
|
||||
.expect("failed to compile macos_shim.c");
|
||||
|
||||
std::process::Command::new("ar")
|
||||
.args(["rcs", &lib, &obj])
|
||||
.status()
|
||||
.expect("failed to create static lib");
|
||||
|
||||
println!("cargo:rustc-link-search=native={out_dir}");
|
||||
println!("cargo:rustc-link-lib=static=macos_scsi");
|
||||
println!("cargo:rerun-if-changed=src/scsi/macos_shim.c");
|
||||
}
|
||||
}
|
||||
|
||||
+3
-1
@@ -11,11 +11,13 @@ Technical documentation for [libfreemkv](https://github.com/freemkv/libfreemkv),
|
||||
| Document | What it covers |
|
||||
|----------|---------------|
|
||||
| [Architecture](architecture.md) | Module map, design principles, error codes, platform support |
|
||||
| [Drive Access](drive-access.md) | DriveSession, SCSI transport, profiles, unlock, why raw mode is needed |
|
||||
| [Drive Access](drive-access.md) | Drive, SCSI transport, profiles, unlock, why raw mode is needed |
|
||||
| [Rip Recovery](rip-recovery.md) | Three-layer recovery model: Disc::patch, single-shot Drive::read, DiscStream batch halving |
|
||||
| [AACS Encryption](aacs.md) | Key resolution (4 paths), content decryption, bus encryption, SCSI handshake |
|
||||
| [UDF Filesystem](udf.md) | UDF 2.50 with metadata partitions, pointer chain, how files are read from disc |
|
||||
| [MPLS Playlists](mpls.md) | Playlist format, play items, STN stream table, coding types |
|
||||
| [CLPI Clip Info](clpi.md) | EP map (coarse + fine entries), timestamp-to-sector mapping, extent calculation |
|
||||
| [API Design](api-design.md) | Stream API design, PES pipeline, input/output resolution |
|
||||
|
||||
## Reading Order
|
||||
|
||||
|
||||
+5
-3
@@ -189,12 +189,14 @@ In practice, AACS 2.0 UHD discs work through the backward-compatible AACS 1.0 ha
|
||||
AACS decryption is transparent to the application. The `Disc::scan()` method handles everything automatically:
|
||||
|
||||
```rust
|
||||
use libfreemkv::{DriveSession, Disc};
|
||||
use libfreemkv::{Drive, Disc};
|
||||
use libfreemkv::disc::ScanOptions;
|
||||
use std::path::Path;
|
||||
|
||||
let mut session = DriveSession::open(Path::new("/dev/sr0")).unwrap();
|
||||
let disc = Disc::scan(&mut session, &ScanOptions::default()).unwrap();
|
||||
let mut drive = Drive::open(Path::new("/dev/sg4")).unwrap();
|
||||
drive.wait_ready().unwrap();
|
||||
drive.init().unwrap();
|
||||
let disc = Disc::scan(&mut drive, &ScanOptions::default()).unwrap();
|
||||
|
||||
// Check encryption state
|
||||
if disc.encrypted {
|
||||
|
||||
+151
-169
@@ -12,81 +12,97 @@
|
||||
|
||||
```rust
|
||||
// Open drive — explicit steps, app prints between them
|
||||
let mut session = DriveSession::open(path)?;
|
||||
session.wait_ready()?;
|
||||
session.init()?;
|
||||
session.probe_disc()?;
|
||||
let mut drive = Drive::open(path)?;
|
||||
drive.wait_ready()?;
|
||||
drive.init()?;
|
||||
drive.probe_disc()?;
|
||||
|
||||
// Scan disc
|
||||
let disc = Disc::scan(&mut session, &ScanOptions::default())?;
|
||||
let disc = Disc::scan(&mut drive, &ScanOptions::default())?;
|
||||
|
||||
// Browse
|
||||
disc.titles // Vec<Title>
|
||||
disc.titles // Vec<DiscTitle>
|
||||
disc.format // BD / UHD / DVD
|
||||
disc.capacity_gb()
|
||||
|
||||
// Rip with events
|
||||
disc.rip(&mut session, 0, output, |event| {
|
||||
match event.kind {
|
||||
EventKind::BytesRead { bytes, total } => ...,
|
||||
EventKind::ReadError { sector, error } => ...,
|
||||
EventKind::Retry { attempt } => ...,
|
||||
EventKind::SpeedChange { speed_kbs } => ...,
|
||||
EventKind::Complete { bytes, errors } => ...,
|
||||
}
|
||||
})?;
|
||||
|
||||
// Rip without events
|
||||
disc.rip(&mut session, 0, output, event::ignore)?;
|
||||
```
|
||||
|
||||
## Stream Chains
|
||||
## PES Pipeline (primary API)
|
||||
|
||||
Each stream wraps the next. Builder pattern, no `.build()`.
|
||||
The PES pipeline is the main way to move content. All streams produce/consume
|
||||
PES frames. The pipeline just reads frames and writes frames.
|
||||
|
||||
### Raw m2ts
|
||||
```rust
|
||||
disc.rip(&mut session, 0, File::create("movie.m2ts")?, event::ignore)?;
|
||||
// URL-based — any source to any destination
|
||||
let opts = InputOptions::default();
|
||||
let mut input = libfreemkv::input("disc:///dev/sg4", &opts)?;
|
||||
let title = input.info().clone();
|
||||
let mut output = libfreemkv::output("mkv://Movie.mkv", &title)?;
|
||||
|
||||
while let Ok(Some(frame)) = input.read() {
|
||||
output.write(&frame)?;
|
||||
}
|
||||
output.finish()?;
|
||||
```
|
||||
|
||||
### MKV
|
||||
The `FrameSource` and `FrameSink` traits — direction is type-checked, so
|
||||
calling `read()` on a write-only sink (or `write()` on a read-only source)
|
||||
is a compile error rather than a runtime fault:
|
||||
|
||||
```rust
|
||||
let output = MkvStream::new(File::create("movie.mkv")?)
|
||||
.title(&disc.titles[0])
|
||||
.max_buffer(10 * 1024 * 1024);
|
||||
pub trait FrameSource: Send {
|
||||
fn read(&mut self) -> Result<Option<PesFrame>, Error>;
|
||||
fn info(&self) -> &DiscTitle;
|
||||
fn codec_private(&self, track: usize) -> Option<Vec<u8>> { None }
|
||||
fn headers_ready(&self) -> bool { true }
|
||||
}
|
||||
|
||||
disc.rip(&mut session, 0, output, |e| { ... })?;
|
||||
pub trait FrameSink: Send {
|
||||
fn write(&mut self, frame: &PesFrame) -> Result<(), Error>;
|
||||
fn finish(self: Box<Self>) -> Result<(), Error>;
|
||||
fn info(&self) -> &DiscTitle;
|
||||
}
|
||||
```
|
||||
|
||||
### MKV with progress (CLI)
|
||||
## Streams
|
||||
|
||||
All streams implement `FrameSource` (read) and/or `FrameSink` (write); the
|
||||
directional split prevents runtime "wrong-direction" errors. URL-based
|
||||
resolvers open any stream by string.
|
||||
|
||||
| Stream | Input | Output | URL | Transport |
|
||||
|--------|-------|--------|-----|-----------|
|
||||
| DiscStream | Yes | -- | `disc://` `disc:///dev/sg4` | Optical drive via SCSI |
|
||||
| IsoStream | Yes | Yes | `iso://path.iso` | Blu-ray ISO image |
|
||||
| MkvStream | Yes | Yes | `mkv://path` | Matroska container |
|
||||
| M2tsStream | Yes | Yes | `m2ts://path` | BD-TS with FMKV metadata header |
|
||||
| NetworkStream | Yes (listen) | Yes (connect) | `network://host:port` | TCP with FMKV metadata header |
|
||||
| StdioStream | Yes (stdin) | Yes (stdout) | `stdio://` | Raw byte pipe |
|
||||
| NullStream | -- | Yes | `null://` | Discard sink (byte counter) |
|
||||
|
||||
All URLs require a `scheme://path` format. Bare paths are rejected.
|
||||
|
||||
```rust
|
||||
let output = ProgressStream::new(
|
||||
MkvStream::new(File::create("movie.mkv")?)
|
||||
.title(&disc.titles[0])
|
||||
.max_buffer(10 * 1024 * 1024),
|
||||
total_bytes,
|
||||
|pct, speed| eprint!("\r {}% {:.1} MB/s", pct, speed),
|
||||
);
|
||||
|
||||
disc.rip(&mut session, 0, output, |e| { ... })?;
|
||||
// PES pipeline (frame-level) — input() returns Box<dyn FrameSource>,
|
||||
// output() returns Box<dyn FrameSink>.
|
||||
let input = libfreemkv::input("disc:///dev/sg4", &opts)?; // DiscStream
|
||||
let input = libfreemkv::input("iso://Dune.iso", &opts)?; // IsoStream
|
||||
let output = libfreemkv::output("mkv://Dune.mkv", &title)?; // MkvOutputStream
|
||||
let output = libfreemkv::output("m2ts://Dune.m2ts", &title)?; // M2tsOutputStream
|
||||
let output = libfreemkv::output("network://10.0.0.1:9000", &title)?; // NetworkOutputStream
|
||||
let output = libfreemkv::output("null://", &title)?; // NullOutputStream
|
||||
```
|
||||
|
||||
### Future: transcode
|
||||
```rust
|
||||
let output = ProgressStream::new(
|
||||
TranscodeStream::new(
|
||||
MkvStream::new(File::create("movie.mkv")?)
|
||||
.title(&disc.titles[0])
|
||||
.max_buffer(50 * 1024 * 1024),
|
||||
)
|
||||
.codec(H265)
|
||||
.quality(22),
|
||||
total_bytes,
|
||||
|pct, speed| eprint!("\r {}% {:.1} MB/s", pct, speed),
|
||||
);
|
||||
### FMKV Metadata Header
|
||||
|
||||
M2tsStream and NetworkStream embed a JSON metadata header before the BD-TS data:
|
||||
|
||||
disc.rip(&mut session, 0, output, |e| { ... })?;
|
||||
```
|
||||
[8B magic "FMKV\0\0\0\0"][4B JSON length][JSON metadata][padding to 192B boundary][BD-TS data...]
|
||||
```
|
||||
|
||||
The header carries title name, duration, codec_privates, and full stream layout
|
||||
(PIDs, codecs, languages, labels). This allows the receiving end to set up
|
||||
demuxing and track metadata without scanning the TS.
|
||||
|
||||
## Events
|
||||
|
||||
@@ -98,114 +114,47 @@ pub struct Event {
|
||||
}
|
||||
|
||||
pub enum EventKind {
|
||||
// Init / scan
|
||||
DriveOpened { device: String },
|
||||
DriveReady,
|
||||
InitComplete { success: bool },
|
||||
ProbeComplete { success: bool },
|
||||
ScanComplete { titles: usize },
|
||||
|
||||
// Read pipeline
|
||||
BytesRead { bytes: u64, total: u64 },
|
||||
ReadError { sector: u64, error: Error },
|
||||
Retry { attempt: u32 },
|
||||
SpeedChange { speed_kbs: u16 },
|
||||
ExtentStart { index: usize, start_sector: u64, sector_count: u64 },
|
||||
SectorSkipped { sector: u64 },
|
||||
BatchSizeChanged { new_size: u16, reason: BatchSizeReason },
|
||||
Complete { bytes: u64, errors: u32 },
|
||||
|
||||
// Kept for forward-compat; not emitted in 0.13.6+
|
||||
Retry { attempt: u32 },
|
||||
SectorRecovered { sector: u64 },
|
||||
}
|
||||
```
|
||||
|
||||
Events report what happened. App decides what to do. GUI shows a dialog. CLI prints a line. Server logs to file.
|
||||
Emission notes:
|
||||
|
||||
## Error Codes
|
||||
- `BytesRead { bytes, total }` is emitted from `DiscStream::fill_extents`
|
||||
after each successful sector read. `bytes` is the cumulative running
|
||||
total; `total` is the precomputed extent sum (0 if unknown).
|
||||
- `SpeedChange` is emitted from the public `Drive::set_speed` API path.
|
||||
It is no longer emitted from a recovery hot loop (recovery loop removed
|
||||
in 0.13.6).
|
||||
- `BatchSizeChanged` fires from the `DiscStream` adaptive sizer on shrink
|
||||
(read failed at a larger size) and on probe-up (clean-read streak hit
|
||||
the threshold). Consumers use it to display a "recovering" state
|
||||
distinct from "ripping normally".
|
||||
- `Retry` and `SectorRecovered` are NOT emitted in 0.13.6+. They were
|
||||
tied to the inline `Drive::read` recovery phases that were removed; the
|
||||
variants are kept for forward compatibility so consumers' match arms
|
||||
don't need conditional compilation.
|
||||
|
||||
Lib errors are codes, not messages. Like HTTP status codes.
|
||||
|
||||
```rust
|
||||
pub enum Error {
|
||||
// Drive
|
||||
DriveNotFound,
|
||||
DriveOpenFailed,
|
||||
DriveNotReady,
|
||||
|
||||
// Unlock
|
||||
UnlockFailed,
|
||||
NoProfile,
|
||||
|
||||
// AACS
|
||||
AacsNoKeys,
|
||||
AacsCertVerifyFailed,
|
||||
AacsAgidAllocFailed,
|
||||
AacsHandshakeFailed,
|
||||
AacsVidMacFailed,
|
||||
|
||||
// Disc
|
||||
DiscReadError { sector: u64 },
|
||||
MplsParseError,
|
||||
ClpiParseError,
|
||||
UdfFileNotFound { path: String },
|
||||
|
||||
// Mux
|
||||
LookaheadOverflow,
|
||||
MuxWriteError,
|
||||
|
||||
// SCSI
|
||||
ScsiError { sense: u8 },
|
||||
}
|
||||
```
|
||||
|
||||
App maps codes to localized strings. Lib never contains display text.
|
||||
|
||||
## Streams
|
||||
|
||||
All streams implement the `IOStream` trait (Read + Write). URL-based resolver opens any stream by string.
|
||||
|
||||
| Stream | Input | Output | URL | Transport |
|
||||
|--------|-------|--------|-----|-----------|
|
||||
| DiscStream | Yes | -- | `disc://` `disc:///dev/sg4` | Optical drive via SCSI |
|
||||
| IsoStream | Yes | -- | `iso://path.iso` | Blu-ray ISO image |
|
||||
| MkvStream | Yes | Yes | `mkv://path` | Matroska container |
|
||||
| M2tsStream | Yes | Yes | `m2ts://path` | BD-TS with FMKV metadata header |
|
||||
| NetworkStream | Yes (listen) | Yes (connect) | `network://host:port` | TCP with FMKV metadata header |
|
||||
| StdioStream | Yes (stdin) | Yes (stdout) | `stdio://` | Raw byte pipe |
|
||||
| NullStream | -- | Yes | `null://` | Discard sink (byte counter) |
|
||||
|
||||
All URLs require a `scheme://path` format. Bare paths are rejected.
|
||||
|
||||
```rust
|
||||
// URL-based opening
|
||||
let input = open_input("disc://", &opts)?; // DiscStream (auto-detect)
|
||||
let input = open_input("disc:///dev/sg4", &opts)?; // DiscStream (specific device)
|
||||
let input = open_input("iso://Dune.iso", &opts)?; // IsoStream
|
||||
let input = open_input("m2ts:///tmp/Dune.m2ts", &opts)?; // M2tsStream
|
||||
let input = open_input("mkv://Dune.mkv", &opts)?; // MkvStream
|
||||
let input = open_input("network://0.0.0.0:9000", &opts)?; // NetworkStream (listen)
|
||||
let input = open_input("stdio://", &opts)?; // StdioStream (stdin)
|
||||
|
||||
let output = open_output("mkv://Dune.mkv", &meta)?; // MkvStream
|
||||
let output = open_output("m2ts://Dune.m2ts", &meta)?; // M2tsStream
|
||||
let output = open_output("network://10.0.0.1:9000", &meta)?;// NetworkStream (connect)
|
||||
let output = open_output("stdio://", &meta)?; // StdioStream (stdout)
|
||||
let output = open_output("null://", &meta)?; // NullStream
|
||||
|
||||
// Direct construction (for advanced use)
|
||||
let mkv = MkvStream::new(writer).meta(&title).max_buffer(10 * 1024 * 1024);
|
||||
let m2ts = M2tsStream::new(writer).meta(&title);
|
||||
let net = NetworkStream::connect("10.0.0.1:9000")?.meta(&title);
|
||||
let null = NullStream::new().meta(&title);
|
||||
```
|
||||
|
||||
### FMKV Metadata Header
|
||||
|
||||
M2tsStream and NetworkStream embed a JSON metadata header before the BD-TS data:
|
||||
|
||||
```
|
||||
[8B magic "FMKV\0\0\0\0"][4B JSON length][JSON metadata][padding to 192B boundary][BD-TS data...]
|
||||
```
|
||||
|
||||
The header carries title name, duration, and full stream layout (PIDs, codecs, languages, labels). This allows the receiving end to set up demuxing and track metadata without scanning the TS.
|
||||
|
||||
### MkvStream Internals
|
||||
|
||||
LookaheadBuffer (default 5MB, configurable):
|
||||
1. Phase 1: buffer incoming data, scan for codec setup (SPS/PPS)
|
||||
2. Found it? Write MKV header, flush buffer, switch to streaming
|
||||
3. Buffer full? Error — app handles it
|
||||
4. Phase 2: parse TS → frames → MKV clusters, direct to output
|
||||
|
||||
Reading: extracts MKV frames, wraps back into BD-TS PES packets.
|
||||
Events report what happened. App decides what to do. GUI shows a dialog. CLI
|
||||
prints a line. Server logs to file.
|
||||
|
||||
## File Layout
|
||||
|
||||
@@ -214,38 +163,71 @@ libfreemkv/src/
|
||||
├── lib.rs Public exports
|
||||
├── error.rs Error codes (no English)
|
||||
├── event.rs Event types for callbacks
|
||||
├── drive.rs DriveSession (open, init, read)
|
||||
├── disc.rs Disc (scan, rip, titles)
|
||||
├── scsi/ SCSI transport (Linux, macOS)
|
||||
├── halt.rs Halt cancellation token (Arc<AtomicBool> wrapper)
|
||||
├── io/ Pipeline + WritebackFile primitives
|
||||
│ ├── mod.rs Re-exports WritebackFile, Pipeline, Sink, Flow
|
||||
│ ├── pipeline.rs Generic Pipeline<I, R> + Sink trait
|
||||
│ ├── writeback_file.rs WritebackFile (was crate::io::Writer)
|
||||
│ └── writeback.rs sync_file_range pipeline
|
||||
├── drive/ Drive (open, init, single-shot read)
|
||||
│ ├── mod.rs Drive struct, init, read (single-shot), reset, eject
|
||||
│ ├── capture.rs Drive profile capture for contribution
|
||||
│ ├── linux.rs Linux drive discovery
|
||||
│ ├── macos.rs macOS drive discovery
|
||||
│ └── windows.rs Windows drive discovery
|
||||
├── disc/ Disc (scan, titles, AACS setup, sweep, patch)
|
||||
│ ├── mod.rs Disc struct, scan, titles, formats
|
||||
│ ├── sweep.rs Disc::sweep (Pass 1 forward sweep)
|
||||
│ ├── patch.rs Disc::patch (Pass N retry over mapfile)
|
||||
│ ├── mapfile.rs ddrescue-format mapfile
|
||||
│ └── read_error.rs ReadCtx / ReadAction state machine
|
||||
├── scsi/ SCSI transport (Linux SG_IO, macOS IOKit, Windows SPTI)
|
||||
├── platform/ Drive unlock (MT1959 A/B)
|
||||
├── aacs/ AACS decryption
|
||||
├── udf.rs UDF filesystem parser
|
||||
├── mpls.rs Playlist parser
|
||||
├── clpi.rs Clip info parser
|
||||
├── aacs/ AACS decryption (handshake, keys, keydb, decrypt)
|
||||
├── css/ DVD CSS cipher
|
||||
├── decrypt.rs Unified decrypt dispatcher (AACS/CSS/None)
|
||||
├── pes.rs PES frame types, FrameSource / FrameSink traits
|
||||
├── sector/ Sector I/O (was sector.rs in 0.17)
|
||||
│ ├── mod.rs SectorSource, SectorSink traits
|
||||
│ ├── file.rs FileSectorSource, FileSectorSink (ISO-backed)
|
||||
│ └── decrypting.rs DecryptingSectorSource decorator
|
||||
├── udf.rs UDF 2.50 filesystem parser
|
||||
├── mpls.rs MPLS playlist parser
|
||||
├── clpi.rs CLPI clip info parser
|
||||
├── ifo.rs DVD IFO parser
|
||||
├── labels/ BD-J label extraction (5 format parsers)
|
||||
├── keydb.rs KEYDB download, parse, save
|
||||
├── identity.rs DriveId from INQUIRY
|
||||
├── profile.rs Bundled drive profiles
|
||||
├── speed.rs DriveSpeed enum
|
||||
├── mux/
|
||||
│ ├── mod.rs IOStream trait, public exports
|
||||
│ ├── resolve.rs URL parser + open_input/open_output
|
||||
│ ├── meta.rs M2tsMeta (FMKV header format)
|
||||
│ ├── disc.rs DiscStream (optical drive)
|
||||
│ ├── mod.rs Public mux exports
|
||||
│ ├── resolve.rs URL parser + input/output (Box<dyn FrameSource/Sink>)
|
||||
│ ├── meta.rs FMKV header format
|
||||
│ ├── disc.rs DiscStream (optical drive → PES)
|
||||
│ ├── iso.rs IsoStream (ISO image read)
|
||||
│ ├── isowriter.rs ISO image writer (UDF, AVDP, multi-extent)
|
||||
│ ├── mkvstream.rs MkvStream (bidirectional Matroska)
|
||||
│ ├── m2ts.rs M2tsStream (BD-TS + FMKV header)
|
||||
│ ├── mkvout.rs MkvOutputStream (PES → MKV)
|
||||
│ ├── m2ts.rs M2tsStream (BD-TS)
|
||||
│ ├── pesout.rs PES output streams (M2ts, Network, Stdio, Null)
|
||||
│ ├── network.rs NetworkStream (TCP + FMKV header)
|
||||
│ ├── stdio.rs StdioStream (stdin/stdout pipe)
|
||||
│ ├── iso.rs IsoStream (Blu-ray ISO image)
|
||||
│ ├── null.rs NullStream (discard + byte counter)
|
||||
│ ├── lookahead.rs LookaheadBuffer (codec header scanning)
|
||||
│ ├── ts.rs BD-TS demuxer + PAT/PMT scanner
|
||||
│ ├── tsreader.rs TS reader utilities
|
||||
│ ├── tsmux.rs TS muxer (PES → BD-TS packets)
|
||||
│ ├── ps.rs MPEG-2 PS demuxer (DVD)
|
||||
│ ├── ebml.rs EBML read/write primitives
|
||||
│ ├── mkv.rs MKV muxer (tracks, clusters, cues)
|
||||
│ └── codec/ Frame parsers (H.264, HEVC, VC-1, AC3, DTS, TrueHD, PGS, LPCM)
|
||||
│ └── codec/ Frame parsers (H.264, HEVC, MPEG-2, VC-1, AC3, EAC3, DTS, TrueHD, LPCM, PGS)
|
||||
└── ...
|
||||
|
||||
freemkv/src/
|
||||
├── main.rs CLI dispatcher (URL routing)
|
||||
├── pipe.rs Generic source → dest copy
|
||||
├── rip.rs Rip with progress display
|
||||
├── remux.rs Remux with progress display
|
||||
├── disc_info.rs Disc info display
|
||||
├── pipe.rs PES pipeline — source → dest copy
|
||||
├── disc_info.rs Disc/file info display
|
||||
├── info.rs Drive info + profile submission
|
||||
├── strings.rs i18n string table
|
||||
├── output.rs Verbosity-filtered output
|
||||
|
||||
+74
-35
@@ -13,9 +13,9 @@ AACS keys are derived internally, and all SCSI communication is handled in-proce
|
||||
|
||||
1. **CLI is dumb.** All drive communication, disc parsing, AACS decryption, and
|
||||
format handling live in the library. CLI binaries are thin wrappers that call
|
||||
`DriveSession::open()` and `Disc::scan()`.
|
||||
`Drive::open()` and `Disc::scan()`.
|
||||
|
||||
2. **No external files.** 206 drive profiles are compiled into the binary via
|
||||
2. **No external files.** Bundled drive profiles are compiled into the binary via
|
||||
`include_str!`. No configuration directory, no runtime file lookups for drive
|
||||
support.
|
||||
|
||||
@@ -23,12 +23,16 @@ AACS keys are derived internally, and all SCSI communication is handled in-proce
|
||||
available. Callers read cleartext sectors without knowing whether the disc
|
||||
was encrypted.
|
||||
|
||||
4. **Structured errors, no English.** Every error has a numeric code (E1000-E7000).
|
||||
4. **Structured errors, no English.** Every error has a numeric code (E1000-E8000).
|
||||
The library never formats user-facing messages -- applications do that.
|
||||
|
||||
5. **Library-agnostic.** No concept of "supported" vs "unsupported" drives at a
|
||||
policy level. If a profile exists, the library uses it.
|
||||
|
||||
6. **Streams are dumb pipes.** Streams read/write PES frames. They don't know
|
||||
about encryption, transport format, or source type. Decrypt is a stream-internal
|
||||
concern; the pipeline just moves frames.
|
||||
|
||||
---
|
||||
|
||||
## Module Map
|
||||
@@ -37,26 +41,43 @@ AACS keys are derived internally, and all SCSI communication is handled in-proce
|
||||
libfreemkv (lib.rs)
|
||||
│
|
||||
├── Drive Access
|
||||
│ ├── drive DriveSession — open, identify, unlock, read
|
||||
│ ├── scsi ScsiTransport trait + SG_IO implementation
|
||||
│ ├── drive Drive — open, identify, init, unlock, single-shot read
|
||||
│ ├── scsi ScsiTransport trait + platform backends (sg async, IOKit, SPTI)
|
||||
│ ├── platform/ Platform trait — per-chipset command handlers
|
||||
│ │ └── mt1959 MediaTek MT1959 driver (LG, ASUS, hp)
|
||||
│ │ └── mt1959 MediaTek MT1959 driver (LG, ASUS, HP)
|
||||
│ ├── profile DriveProfile loading, matching, bundled JSON
|
||||
│ ├── identity DriveId from INQUIRY + GET_CONFIG 010C
|
||||
│ └── speed DriveSpeed enum, SET CD SPEED CDB builder
|
||||
│ ├── speed DriveSpeed enum, SET CD SPEED CDB builder
|
||||
│ └── event Event system for drive status callbacks
|
||||
│
|
||||
├── Disc Scanning
|
||||
│ ├── disc Disc::scan() — titles, streams, extents, AACS setup
|
||||
│ ├── udf UDF 2.50 filesystem reader (metadata partitions)
|
||||
│ ├── mpls MPLS playlist parser — clips, streams, STN table
|
||||
│ ├── clpi CLPI clip info parser — EP map, sector extents
|
||||
│ └── jar BD-J JAR label extraction (audio/subtitle names)
|
||||
│ ├── ifo DVD IFO parser — title sets, PGC chains, cell addresses
|
||||
│ └── labels/ BD-J label extraction (5 formats: Paramount, Criterion, Pixelogic, CTRM, Deluxe)
|
||||
│
|
||||
├── Encryption
|
||||
│ ├── aacs KEYDB parsing, VUK lookup, MKB processing, unit decryption
|
||||
│ └── aacs_handshake ECDH bus authentication, Volume ID, Read Data Key
|
||||
│ ├── aacs/ AACS handshake, KEYDB, VUK lookup, MKB, unit decryption
|
||||
│ ├── css DVD CSS cipher — table-driven, no external keys needed
|
||||
│ └── decrypt decrypt_sectors() — unified AACS/CSS/None dispatcher
|
||||
│
|
||||
└── error Error enum with numeric codes E1000-E7000
|
||||
├── Streaming
|
||||
│ ├── mux/ Stream implementations (Disc, ISO, MKV, M2TS, Network, Stdio, Null)
|
||||
│ ├── pes PES frame types; FrameSource / FrameSink direction-typed traits
|
||||
│ └── sector/ SectorSource / SectorSink traits, FileSector{Source,Sink}, DecryptingSectorSource
|
||||
│
|
||||
├── I/O Primitives
|
||||
│ ├── halt Halt cancellation token (one Arc<AtomicBool>, cloneable)
|
||||
│ └── io/ Pipeline<I, R> + Sink trait + WritebackFile (bounded-cache writer)
|
||||
│
|
||||
├── Support
|
||||
│ ├── keydb KEYDB.cfg download, parse, verify, save
|
||||
│ ├── error Error enum with numeric codes E1000-E8000
|
||||
│ └── profile Bundled drive profiles
|
||||
│
|
||||
└── lib.rs Public API re-exports
|
||||
```
|
||||
|
||||
---
|
||||
@@ -64,27 +85,35 @@ libfreemkv (lib.rs)
|
||||
## Drive Access Flow
|
||||
|
||||
```
|
||||
DriveSession::open("/dev/sr0")
|
||||
Drive::open(Path::new("/dev/sg4"))
|
||||
│
|
||||
├─ scsi::open() Open /dev/sr0 via SG_IO
|
||||
├─ scsi::open() Open /dev/sg4 (async write/poll/read)
|
||||
├─ DriveId::from_drive() INQUIRY + GET_CONFIG 010C
|
||||
├─ profile::find_by_drive_id() Match against 206 bundled profiles
|
||||
├─ profile::find_by_drive_id() Match against bundled profiles
|
||||
├─ Platform::new() Instantiate chipset driver (Mt1959)
|
||||
└─ Platform::unlock() Activate raw disc access mode
|
||||
└─ Drive ready for init/unlock/read
|
||||
```
|
||||
|
||||
After open, the session provides:
|
||||
- `read_sectors(lba, count, buf)` -- raw sector reads (through platform driver)
|
||||
- `read_disc(lba, count, buf)` -- standard READ(10) for filesystem data
|
||||
- `scsi_execute(cdb, dir, buf, timeout)` -- arbitrary SCSI commands
|
||||
- `status()`, `calibrate()`, `read_config()`, `read_register()`
|
||||
After open:
|
||||
- `init()` -- unlock + firmware upload + speed calibration
|
||||
- `probe_disc()` -- probe disc surface for optimal speeds
|
||||
- `read(lba, count, buf, recovery)` -- single-shot read; `recovery` only selects the per-CDB timeout (1.5 s vs. 30 s)
|
||||
- `wait_ready()` -- wait for disc insertion
|
||||
- `eject()` -- eject tray
|
||||
|
||||
Recovery is layered above `Drive::read`, not inside it. Layer 1
|
||||
(`Disc::patch`) handles bad-range retry by replaying the ddrescue mapfile.
|
||||
Layer 3 (`DiscStream::fill_extents` adaptive batch sizer) handles in-loop
|
||||
request-size adaptation. Inline recovery (gentle retry → SCSI reset → retry)
|
||||
was removed in 0.13.6 — see [`rip-recovery.md`](rip-recovery.md) and
|
||||
the stop-wedge postmortem (2026-04-25).
|
||||
|
||||
---
|
||||
|
||||
## Disc Scanning Flow
|
||||
|
||||
```
|
||||
Disc::scan(&mut session, &ScanOptions)
|
||||
Disc::scan(&mut drive, &ScanOptions)
|
||||
│
|
||||
├─ READ CAPACITY Get disc size in sectors
|
||||
├─ udf::read_filesystem() Parse UDF 2.50 (AVDP → VDS → metadata → FSD → root)
|
||||
@@ -92,14 +121,24 @@ Disc::scan(&mut session, &ScanOptions)
|
||||
│ ├─ mpls::parse() Extract play items, STN streams
|
||||
│ └─ For each clip:
|
||||
│ └─ clpi::parse() EP map → sector extents for the clip's time range
|
||||
├─ labels::detect() Parse BD-J JARs for stream labels
|
||||
├─ Detect AACS Check for /AACS directory on disc
|
||||
└─ Disc::setup_aacs() Handshake + KEYDB → VUK → unit keys (if encrypted)
|
||||
```
|
||||
|
||||
For DVD:
|
||||
```
|
||||
Disc::scan_dvd(&mut drive, &ScanOptions)
|
||||
│
|
||||
├─ ifo::parse() Parse VIDEO_TS.IFO — title sets, PGC chains
|
||||
├─ CSS detection Check disc structure flag
|
||||
└─ CSS key cracking Table-driven, no KEYDB needed
|
||||
```
|
||||
|
||||
The result is a `Disc` with:
|
||||
- `titles: Vec<Title>` -- sorted by duration, each with streams and sector extents
|
||||
- `aacs: Option<AacsState>` -- decryption keys if available
|
||||
- `encrypted: bool` -- whether the disc uses AACS
|
||||
- `titles: Vec<DiscTitle>` -- sorted by duration, each with streams, sector extents, codec_privates
|
||||
- `decrypt_keys()` -- DecryptKeys for content decryption
|
||||
- `encrypted: bool` -- whether the disc uses AACS/CSS
|
||||
|
||||
---
|
||||
|
||||
@@ -114,13 +153,14 @@ Four key resolution paths, tried in order:
|
||||
| 3 | Processing Keys + MKB → Media Key → VUK | Medium |
|
||||
| 4 | Device Keys + MKB subset-difference tree → VUK | Slow |
|
||||
|
||||
The AACS handshake (`aacs_handshake`) performs ECDH key agreement over the
|
||||
The AACS handshake (`aacs/handshake`) performs ECDH key agreement over the
|
||||
AACS 1.0 160-bit elliptic curve to obtain:
|
||||
- **Volume ID** -- needed for VUK derivation (paths 2-4)
|
||||
- **Read Data Key** -- needed for AACS 2.0 (UHD) bus decryption
|
||||
|
||||
Content decryption uses AES-128-CBC on 6144-byte aligned units. The
|
||||
`ContentReader` handles this transparently.
|
||||
`ContentReader` handles this transparently. Streams that read sectors
|
||||
(DiscStream, IsoStream) decrypt internally — the pipeline sees clean bytes.
|
||||
|
||||
---
|
||||
|
||||
@@ -138,6 +178,7 @@ is baked into the library.
|
||||
| E5xxx | I/O errors | `IoError` (wraps `std::io::Error`) |
|
||||
| E6xxx | Disc format errors | `DiscError` (UDF, MPLS, CLPI parse failures) |
|
||||
| E7xxx | AACS errors | `AacsError` (key resolution, handshake, decryption) |
|
||||
| E8xxx | KEYDB errors | `KeydbError` (download, parse, save) |
|
||||
|
||||
---
|
||||
|
||||
@@ -145,9 +186,9 @@ is baked into the library.
|
||||
|
||||
| Platform | Transport | Status |
|
||||
|----------|-----------|--------|
|
||||
| Linux | SG_IO ioctl on `/dev/sr*` | Implemented |
|
||||
| macOS | IOKit SCSI passthrough | Planned |
|
||||
| Windows | SPTI (`IOCTL_SCSI_PASS_THROUGH_DIRECT`) | Planned |
|
||||
| Linux | async sg write/poll/read on `/dev/sg*` | Supported |
|
||||
| macOS | IOKit SCSITask | Supported |
|
||||
| Windows | SPTI (`IOCTL_SCSI_PASS_THROUGH_DIRECT`) | Supported |
|
||||
|
||||
The `ScsiTransport` trait abstracts the platform. Adding a new platform requires
|
||||
implementing `execute()` for that OS and wiring it into `scsi::open()`.
|
||||
@@ -158,11 +199,11 @@ implementing `execute()` for that OS and wiring it into `scsi::open()`.
|
||||
|
||||
| Chipset | Drives | Status |
|
||||
|---------|--------|--------|
|
||||
| MediaTek MT1959 | LG, ASUS, hp | Implemented (206 profiles) |
|
||||
| MediaTek MT1959 | LG, ASUS, HP | Supported (bundled profiles) |
|
||||
| Renesas RS8xxx/RS9xxx | Pioneer, some HL-DT-ST | Planned |
|
||||
|
||||
The `Platform` trait abstracts chipset-specific commands. Each chipset implements
|
||||
10 handlers (unlock, config, register, calibrate, keepalive, status, probe,
|
||||
handlers (unlock, config, register, calibrate, keepalive, status, probe,
|
||||
read_sectors, timing). All handlers are accessed via SCSI READ BUFFER with
|
||||
chipset-specific mode and buffer ID bytes.
|
||||
|
||||
@@ -174,7 +215,5 @@ chipset-specific mode and buffer ID bytes.
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Linux builds produce a static library and two binaries (`freemkv-info`,
|
||||
`freemkv-test`). The `libc` dependency is Linux-only. On non-Linux platforms,
|
||||
the library compiles but `scsi::open()` returns a platform-not-supported error
|
||||
until the IOKit/SPTI backends are implemented.
|
||||
Produces a Rust library crate. The `libc` dependency is unix-only (gated).
|
||||
All three platforms build and pass CI.
|
||||
|
||||
+62
-41
@@ -9,12 +9,18 @@ This is the starting point for understanding the library.
|
||||
Insert disc
|
||||
│
|
||||
▼
|
||||
1. Open drive (drive.rs)
|
||||
1. Open drive (drive/mod.rs)
|
||||
│ INQUIRY → identify drive
|
||||
│ Match bundled profile → chipset, unlock parameters
|
||||
│
|
||||
▼
|
||||
2. AACS handshake (aacs_handshake.rs) — optional, separate transport
|
||||
2. Init drive (drive/mod.rs → platform/mt1959)
|
||||
│ Firmware upload (if needed, 10s recovery wait)
|
||||
│ Unlock → vendor-specific command activates raw read mode
|
||||
│ Speed calibration → probe_disc()
|
||||
│
|
||||
▼
|
||||
3. AACS handshake (aacs/handshake.rs) — optional
|
||||
│ Allocate AGID
|
||||
│ Exchange certificates + nonces (ECDH)
|
||||
│ Derive bus key
|
||||
@@ -22,11 +28,6 @@ Insert disc
|
||||
│ (fails gracefully if drive doesn't support AACS for this disc)
|
||||
│
|
||||
▼
|
||||
3. Unlock drive (drive.rs → platform/mt1959.rs)
|
||||
│ Vendor-specific command activates raw read mode
|
||||
│ Required — drive firmware blocks all reads without it
|
||||
│
|
||||
▼
|
||||
4. Read UDF filesystem (udf.rs)
|
||||
│ Sector 256: AVDP → find Volume Descriptor Sequence
|
||||
│ VDS: Partition Descriptor (physical start) + Logical Volume (metadata start)
|
||||
@@ -35,80 +36,100 @@ Insert disc
|
||||
│ → docs/udf.md
|
||||
│
|
||||
▼
|
||||
5. Read AACS files from disc (aacs.rs)
|
||||
5. Read AACS files from disc (aacs/mod.rs)
|
||||
│ AACS/Unit_Key_RO.inf → SHA1 = disc hash
|
||||
│ AACS/Content000.cer → AACS version (1.0 or 2.0), bus encryption flag
|
||||
│ MKB via SCSI → for key derivation fallback
|
||||
│
|
||||
▼
|
||||
6. Resolve AACS keys (aacs.rs → resolve_keys)
|
||||
│ Path 1: disc hash → KEYDB.cfg → VUK (fast, 99% of discs)
|
||||
│ Path 2: KEYDB media key + Volume ID → VUK
|
||||
│ Path 3: MKB + processing keys → media key → VUK
|
||||
│ Path 4: MKB + device keys → subset-difference tree → VUK
|
||||
│ VUK → decrypt unit keys from Unit_Key_RO.inf
|
||||
6. Resolve encryption keys (decrypt.rs → resolve_encryption)
|
||||
│ BD AACS:
|
||||
│ Path 1: disc hash → KEYDB.cfg → VUK (fast, 99% of discs)
|
||||
│ Path 2: KEYDB media key + Volume ID → VUK
|
||||
│ Path 3: MKB + processing keys → media key → VUK
|
||||
│ Path 4: MKB + device keys → subset-difference tree → VUK
|
||||
│ VUK → decrypt unit keys from Unit_Key_RO.inf
|
||||
│ DVD CSS:
|
||||
│ Table-driven cipher — no KEYDB needed
|
||||
│ → docs/aacs.md
|
||||
│
|
||||
▼
|
||||
7. Parse playlists (mpls.rs)
|
||||
7. Parse playlists (mpls.rs) — BD/UHD only
|
||||
│ BDMV/PLAYLIST/*.mpls → titles with play items
|
||||
│ Each play item: clip ID, in/out timestamps
|
||||
│ STN table: video, audio, subtitle streams with codec + language
|
||||
│ → docs/mpls.md
|
||||
│
|
||||
▼
|
||||
8. Parse clip info (clpi.rs)
|
||||
8. Parse clip info (clpi.rs) — BD/UHD only
|
||||
│ BDMV/CLIPINF/*.clpi → EP map (timestamp → sector mapping)
|
||||
│ Coarse + fine entries → full PTS and SPN
|
||||
│ SPN → byte offset → sector extents for reading
|
||||
│ → docs/clpi.md
|
||||
│
|
||||
▼
|
||||
9. Parse BD-J labels (jar.rs) — optional
|
||||
9. Parse BD-J labels (labels/) — optional
|
||||
│ BDMV/JAR/*.jar → Java class constant pool strings
|
||||
│ 5 format parsers: Paramount, Criterion, Pixelogic, CTRM, Deluxe
|
||||
│ Audio track labels: "English Descriptive Audio", "French 5.1", etc.
|
||||
│
|
||||
▼
|
||||
10. Read + decrypt content (disc.rs → ContentReader)
|
||||
│ For each aligned unit (6144 bytes = 3 sectors):
|
||||
│ Read 3 sectors from disc
|
||||
│ If AACS 2.0: bus decrypt (read_data_key, per-sector AES-CBC)
|
||||
│ If encrypted: unit decrypt (per-unit key derivation + AES-CBC)
|
||||
│ Output decrypted content
|
||||
10. Stream content (mux/disc.rs → DiscStream)
|
||||
│ Read sectors → decrypt → TS demux → PES frames
|
||||
│ Or: read sectors → decrypt → raw bytes (for ISO output)
|
||||
│ Drive::read() is single-shot. DiscStream::fill_extents adapts the
|
||||
│ batch size on failure (halve / probe-up). Bad-range retry is layer
|
||||
│ 1 above this — Disc::patch re-runs against the mapfile.
|
||||
│
|
||||
▼
|
||||
Decrypted m2ts stream → ready for muxing/backup
|
||||
PES frames → output stream (MKV, M2TS, network, etc.)
|
||||
```
|
||||
|
||||
## API Summary
|
||||
|
||||
```rust
|
||||
// Steps 1 + 3 (open + unlock)
|
||||
let mut session = DriveSession::open(Path::new("/dev/sr0"))?;
|
||||
// Open + init drive
|
||||
let mut drive = Drive::open(Path::new("/dev/sg4"))?;
|
||||
drive.wait_ready()?;
|
||||
drive.init()?;
|
||||
drive.probe_disc()?;
|
||||
|
||||
// Steps 2 + 4-9 (AACS + scan)
|
||||
let disc = Disc::scan(&mut session, &ScanOptions::with_keydb("keydb.cfg"))?;
|
||||
// Scan disc (UDF + playlists + AACS — all automatic)
|
||||
let disc = Disc::scan(&mut drive, &ScanOptions::default())?;
|
||||
|
||||
// Step 10 (read + decrypt)
|
||||
let mut reader = disc.open_title(&mut session, 0)?;
|
||||
while let Some(unit) = reader.read_unit()? {
|
||||
output.write_all(&unit)?;
|
||||
// Stream pipeline — PES frames from any source to any output.
|
||||
// 0.18: input() returns Box<dyn FrameSource>, output() returns Box<dyn FrameSink>;
|
||||
// direction is type-checked, so calling .write() on an input is a compile error.
|
||||
let opts = InputOptions::default();
|
||||
let mut input = libfreemkv::input("disc:///dev/sg4", &opts)?;
|
||||
let title = input.info().clone();
|
||||
let mut output = libfreemkv::output("mkv://Movie.mkv", &title)?;
|
||||
while let Ok(Some(frame)) = input.read() {
|
||||
output.write(&frame)?;
|
||||
}
|
||||
output.finish()?;
|
||||
```
|
||||
|
||||
Three lines. Everything else is internal.
|
||||
|
||||
## Module Reference
|
||||
|
||||
| Module | Doc | Purpose |
|
||||
|--------|-----|---------|
|
||||
| drive.rs | [drive-access.md](drive-access.md) | Open, identify, unlock, read |
|
||||
| scsi.rs | [drive-access.md](drive-access.md) | Platform SCSI transport |
|
||||
| drive/ | [drive-access.md](drive-access.md) | Open, identify, init, unlock, single-shot read |
|
||||
| scsi/ | [drive-access.md](drive-access.md) | Platform SCSI transport (Linux, macOS, Windows) |
|
||||
| udf.rs | [udf.md](udf.md) | UDF 2.50 filesystem |
|
||||
| mpls.rs | [mpls.md](mpls.md) | MPLS playlists + STN streams |
|
||||
| clpi.rs | [clpi.md](clpi.md) | CLPI clip info + EP map |
|
||||
| aacs.rs | [aacs.md](aacs.md) | Key resolution + content decrypt |
|
||||
| aacs_handshake.rs | [aacs.md](aacs.md) | SCSI bus authentication |
|
||||
| disc.rs | -- | High-level scan + read API |
|
||||
| jar.rs | -- | BD-J audio track labels |
|
||||
| error.rs | -- | Error codes (E1xxx-E7xxx) |
|
||||
| ifo.rs | -- | DVD IFO parser |
|
||||
| aacs/ | [aacs.md](aacs.md) | Key resolution + content decrypt + bus handshake |
|
||||
| css/ | -- | DVD CSS cipher |
|
||||
| decrypt.rs | -- | Unified decrypt dispatcher (AACS/CSS/None) |
|
||||
| disc/ | [rip-recovery.md](rip-recovery.md) | Disc::scan + Disc::sweep + Disc::patch + mapfile |
|
||||
| labels/ | -- | BD-J stream labels (5 format parsers) |
|
||||
| mux/ | -- | Stream implementations (7 stream types) |
|
||||
| pes.rs | -- | PES frame types + FrameSource / FrameSink traits |
|
||||
| sector/ | -- | SectorSource / SectorSink + DecryptingSectorSource decorator |
|
||||
| io/ | -- | Pipeline<I, R> + Sink trait + WritebackFile |
|
||||
| halt.rs | -- | Halt cancellation token |
|
||||
| keydb.rs | -- | KEYDB download, parse, save |
|
||||
| error.rs | -- | Error codes (E1xxx-E8xxx) |
|
||||
| event.rs | -- | Drive event system |
|
||||
|
||||
+102
-87
@@ -5,51 +5,72 @@ optical drives.
|
||||
|
||||
---
|
||||
|
||||
## DriveSession
|
||||
## Drive
|
||||
|
||||
`DriveSession` is the primary API. It owns the SCSI transport, the matched
|
||||
`Drive` is the primary API. It owns the SCSI transport, the matched
|
||||
drive profile, and the chipset-specific platform driver.
|
||||
|
||||
### Opening a Drive
|
||||
|
||||
```rust
|
||||
// Full open: identify → match profile → unlock
|
||||
let mut session = DriveSession::open(Path::new("/dev/sr0"))?;
|
||||
|
||||
// No-unlock open: identify → match profile only
|
||||
let mut session = DriveSession::open_no_unlock(Path::new("/dev/sr0"))?;
|
||||
|
||||
// Explicit profile (skip auto-detection)
|
||||
let mut session = DriveSession::open_with_profile(Path::new("/dev/sr0"), profile)?;
|
||||
let mut drive = Drive::open(Path::new("/dev/sg4"))?;
|
||||
```
|
||||
|
||||
**`open()`** performs the full sequence: open device, send INQUIRY, match
|
||||
profile, instantiate platform driver, and unlock. Unlock failures are silently
|
||||
ignored (unencrypted discs do not need it). After `open()`, both raw sector
|
||||
reads and standard READ(10) work immediately.
|
||||
`open()` performs: open device → send INQUIRY → match profile → instantiate
|
||||
platform driver. The drive is ready for `wait_ready()` and `init()`.
|
||||
|
||||
**`open_no_unlock()`** skips the unlock step. This is required when AACS bus
|
||||
authentication must happen before unlock. The handshake uses standard SCSI
|
||||
commands that work without raw mode. After authentication completes, the caller
|
||||
can invoke `session.unlock()` manually.
|
||||
|
||||
**`open_with_profile()`** bypasses profile auto-detection. Useful for testing
|
||||
or when a custom profile is loaded from an external source.
|
||||
|
||||
### Session Operations
|
||||
### Drive Operations
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `unlock()` | Activate raw disc access mode via platform driver |
|
||||
| `is_unlocked()` | Check if raw mode is active |
|
||||
| `calibrate()` | Build speed lookup table for the current disc |
|
||||
| `read_sectors(lba, count, buf)` | Raw sector read (requires unlock + calibrate) |
|
||||
| `read_disc(lba, count, buf)` | Standard READ(10) with 5s timeout |
|
||||
| `status()` | Query drive status and feature flags |
|
||||
| `read_config()` | Read drive configuration block (1888 bytes) |
|
||||
| `read_register(index)` | Read 16-byte hardware register |
|
||||
| `probe(sub_cmd, addr, len)` | Generic READ BUFFER with caller parameters |
|
||||
| `scsi_execute(cdb, dir, buf, timeout)` | Send an arbitrary SCSI CDB |
|
||||
| `wait_ready()` | Wait for disc insertion (30s timeout, TUR polling) |
|
||||
| `init()` | Firmware upload + unlock + speed calibration |
|
||||
| `probe_disc()` | Probe disc surface for optimal speeds |
|
||||
| `read(lba, count, buf, recovery)` | Read sectors. Single-shot — no inline retries or reset. |
|
||||
| `reset()` | Eject-cycle escape hatch. Caller-invoked only; not on the read path. |
|
||||
| `lock_tray()` | Prevent tray ejection during rip |
|
||||
| `unlock_tray()` | Allow tray ejection (also runs on Drop) |
|
||||
| `eject()` | Eject disc tray |
|
||||
| `drive_status()` | Query physical state (disc present, tray open, etc.) |
|
||||
| `has_profile()` | Whether a bundled profile matched |
|
||||
| `close()` | Consume Drive, cleanup (also runs via Drop) |
|
||||
|
||||
### init() Sequence
|
||||
|
||||
`init()` orchestrates the full drive unlock:
|
||||
|
||||
1. Platform driver `run_init()` — sends vendor-specific SCSI commands
|
||||
2. If firmware upload needed: upload, wait 10s for drive reset, retry
|
||||
3. Speed calibration after unlock
|
||||
4. Max 3 attempts before giving up
|
||||
|
||||
### read() — single-shot
|
||||
|
||||
`Drive::read(lba, count, buf, recovery)` is the single read method. It issues
|
||||
exactly one READ(10) CDB and returns the result. The `recovery` parameter only
|
||||
selects the per-CDB timeout:
|
||||
|
||||
| `recovery` | Timeout | Used by |
|
||||
|------------|----------|------------------------------------------|
|
||||
| `false` | 1.5 s | `Disc::sweep` fast skip-forward pass, `DiscStream::fill_extents` |
|
||||
| `true` | 30 s | `Disc::patch` retry pass over the mapfile |
|
||||
|
||||
On any SCSI failure or timeout, `read` returns `Err(DiscRead)` immediately.
|
||||
There are no inline retries, no SCSI reset, no Phase 1/2/3 escalation.
|
||||
|
||||
Recovery is layered above `Drive::read`:
|
||||
|
||||
- **Layer 1 — `Disc::patch`** loops over the ddrescue mapfile and re-issues
|
||||
`read(.., recovery=true)` against each non-`+` range.
|
||||
- **Layer 3 — `DiscStream::fill_extents`** halves the request size on
|
||||
failure, retries at the same LBA, and probes back up on a clean-read
|
||||
streak.
|
||||
|
||||
Inline recovery (5× gentle retry → close + reset + reopen → 5× more) was
|
||||
removed in 0.13.6. See the stop-wedge postmortem (2026-04-25)
|
||||
for rationale: the inline reset wedged drive firmware on the LG BU40N (Initio
|
||||
USB-SATA bridge) without ever recovering a sector. See
|
||||
[`rip-recovery.md`](rip-recovery.md) for the full three-layer model.
|
||||
|
||||
---
|
||||
|
||||
@@ -58,7 +79,7 @@ or when a custom profile is loaded from an external source.
|
||||
### Trait
|
||||
|
||||
```rust
|
||||
pub trait ScsiTransport {
|
||||
pub trait ScsiTransport: Send {
|
||||
fn execute(
|
||||
&mut self,
|
||||
cdb: &[u8],
|
||||
@@ -72,17 +93,43 @@ pub trait ScsiTransport {
|
||||
All drive communication goes through this trait. The library never opens file
|
||||
descriptors or calls ioctls outside of a `ScsiTransport` implementation.
|
||||
|
||||
### Linux: SG_IO
|
||||
### Platform Backends
|
||||
|
||||
The `SgIoTransport` implementation:
|
||||
| Platform | Implementation | Device |
|
||||
|----------|---------------|--------|
|
||||
| Linux | `SgIoTransport` — async `write`/`poll`/`read` on `/dev/sg*` | `/dev/sg*` |
|
||||
| macOS | `MacScsiTransport` — IOKit SCSITask | IOKit service |
|
||||
|
||||
1. Opens the device path with `O_RDWR | O_NONBLOCK`.
|
||||
2. Constructs an `sg_io_hdr` struct with the CDB, data buffer, and timeout.
|
||||
3. Calls `ioctl(fd, SG_IO, &hdr)`.
|
||||
4. Returns `ScsiResult` with status, bytes transferred, and sense data.
|
||||
The Linux backend uses the sg driver's asynchronous interface: `write()` submits
|
||||
the command, `poll()` waits with an enforceable wall-clock timeout, `read()`
|
||||
retrieves the result. If `poll()` times out, the fd is abandoned (closed in a
|
||||
background thread) and a fresh fd opened — the kernel's USB error recovery
|
||||
cannot block us. Opens with `O_RDWR | O_NONBLOCK`.
|
||||
|
||||
On non-zero SCSI status, the transport parses sense key, ASC, and ASCQ from the
|
||||
sense buffer and returns `Error::ScsiError`.
|
||||
The macOS backend uses a C shim (`macos_shim.c`) for IOKit exclusive access.
|
||||
The shim handles:
|
||||
1. `shim_open_exclusive(bsd_name)` — unmounts the target device via `diskutil`,
|
||||
then walks the IOKit registry to find the `IOBDServices` matching the
|
||||
requested BSD name (IOBDServices → IOBDBlockStorageDriver → IOMedia → "BSD Name"),
|
||||
then creates MMCDeviceInterface → SCSITaskDeviceInterface → ObtainExclusiveAccess.
|
||||
2. `shim_list_drives()` — registry-based enumeration with zero SCSI, zero exclusive
|
||||
access, zero unmounts. Reads IOBDServices "Device Characteristics" for
|
||||
vendor/model/firmware and child IOMedia "BSD Name" for the device path.
|
||||
3. `shim_execute()` / `shim_close()` — raw CDB dispatch and cleanup.
|
||||
|
||||
On non-zero SCSI status, the transport parses sense key from the sense buffer
|
||||
and returns `Error::ScsiError`.
|
||||
|
||||
`SgIoTransport::reset` (Linux) does pure userspace state cleanup: an open +
|
||||
close pair to make the kernel cancel any SG_IO commands queued against a
|
||||
previous fd, a 2 s sleep to let the kernel finish that cancellation, then a
|
||||
fresh fd to send ALLOW MEDIUM REMOVAL to clear any stale tray lock. It does
|
||||
NOT issue `SG_SCSI_RESET` or escalate via STOP+START UNIT. Both were tried
|
||||
in 0.13.0–0.13.5 against the LG BU40N (Initio USB-SATA bridge); both failed
|
||||
to recover wedged drives and made the wedge worse. The macOS reset (which
|
||||
had been a no-op) was removed entirely in 0.13.6, and the top-level
|
||||
`scsi::reset()` / `reset_with_timeout()` / `reset_blocking()` wrappers were
|
||||
removed at the same time (no callers).
|
||||
|
||||
### CDB Builders
|
||||
|
||||
@@ -119,8 +166,8 @@ date for drives where Feature 010C is unavailable.
|
||||
|
||||
## Drive Profiles
|
||||
|
||||
Profiles are JSON objects compiled into the binary (`profiles.json`,
|
||||
206 entries). Each profile contains:
|
||||
Profiles are JSON objects compiled into the binary (`profiles.json`).
|
||||
Each profile contains:
|
||||
|
||||
| Field | Purpose |
|
||||
|-------|---------|
|
||||
@@ -148,7 +195,7 @@ let profiles = profile::load_all(Path::new("/path/to/profiles.json"))?;
|
||||
|
||||
### MediaTek MT1959
|
||||
|
||||
Covers all LG, ASUS, and hp optical drives. Two sub-variants share identical
|
||||
Covers all LG, ASUS, and HP optical drives. Two sub-variants share identical
|
||||
logic with different SCSI parameters:
|
||||
|
||||
| Variant | READ BUFFER mode | Buffer ID |
|
||||
@@ -156,7 +203,7 @@ logic with different SCSI parameters:
|
||||
| MT1959-A | 0x01 | 0x44 |
|
||||
| MT1959-B | 0x02 | 0x77 |
|
||||
|
||||
The Platform trait maps to 10 command handlers:
|
||||
The Platform trait maps to command handlers:
|
||||
|
||||
| Handler | Function | Description |
|
||||
|---------|----------|-------------|
|
||||
@@ -183,64 +230,32 @@ Optical drive firmware restricts what applications can read from disc. Without
|
||||
unlock:
|
||||
|
||||
- **READ(10) works for unencrypted filesystem data.** UDF structures, MPLS
|
||||
playlists, and CLPI clip info are readable without unlock. The `read_disc()`
|
||||
method uses standard READ(10) and works on any drive.
|
||||
playlists, and CLPI clip info are readable without unlock. Standard READ(10)
|
||||
works on any drive.
|
||||
|
||||
- **READ(10) fails for encrypted content sectors.** The drive firmware returns
|
||||
SCSI errors (sense key 0x05, illegal request) when an application attempts to
|
||||
read sectors containing encrypted m2ts content without prior AACS
|
||||
authentication via the bus key.
|
||||
|
||||
- **The kernel sr driver blocks block-device reads.** On Linux, the kernel's
|
||||
SCSI CD-ROM driver (`sr`) refuses to expose encrypted disc content through
|
||||
`/dev/sr0` as a block device. Even if you open the block device directly,
|
||||
reads to encrypted regions fail.
|
||||
|
||||
- **Raw mode bypasses firmware restrictions.** After unlock, the drive accepts
|
||||
READ(10) with the raw read flag (CDB byte 1 = 0x08) for all sectors,
|
||||
regardless of encryption status. This is how raw sector ripping works.
|
||||
regardless of encryption status.
|
||||
|
||||
### open() vs open_no_unlock()
|
||||
### AACS Before Unlock
|
||||
|
||||
AACS bus authentication uses standard MMC REPORT KEY / SEND KEY commands.
|
||||
These must execute before unlock because:
|
||||
|
||||
1. The AACS handshake establishes a bus key via ECDH.
|
||||
2. The bus key encrypts the Volume ID and Read Data Key responses.
|
||||
3. The Volume ID is needed to derive the Volume Unique Key (VUK).
|
||||
4. The VUK is needed to decrypt unit keys from `Unit_Key_RO.inf`.
|
||||
|
||||
If `open()` unlocks first, some drives reject the subsequent AACS commands.
|
||||
The correct sequence for encrypted discs is:
|
||||
|
||||
```rust
|
||||
// 1. Open without unlock
|
||||
let mut session = DriveSession::open_no_unlock(device)?;
|
||||
|
||||
// 2. AACS handshake (uses standard SCSI, no unlock needed)
|
||||
let auth = aacs_handshake::aacs_authenticate(&mut session, &key, &cert)?;
|
||||
let vid = aacs_handshake::read_volume_id(&mut session, &mut auth)?;
|
||||
|
||||
// 3. Now unlock for raw reads
|
||||
session.unlock()?;
|
||||
session.calibrate()?;
|
||||
|
||||
// 4. Read and decrypt content
|
||||
session.read_sectors(lba, count, &mut buf)?;
|
||||
```
|
||||
|
||||
In practice, `Disc::scan()` handles this internally. The default `open()` call
|
||||
unlocks immediately and is correct for most use cases -- the scan re-opens a
|
||||
second session with `open_no_unlock()` for the AACS handshake when needed.
|
||||
On some drives these must execute before unlock. The `Disc::scan()` handles
|
||||
this internally — it manages the handshake/unlock ordering automatically.
|
||||
|
||||
---
|
||||
|
||||
## Speed Control
|
||||
|
||||
After `calibrate()`, the platform driver maintains a 64-entry speed lookup table
|
||||
built by probing the disc surface. On each `read_sectors()` call, the driver:
|
||||
After `probe_disc()`, the platform driver maintains a speed lookup table
|
||||
built by probing the disc surface. On each `read()` call, the driver:
|
||||
|
||||
1. Looks up the optimal speed for the target LBA in the table.
|
||||
1. Looks up the optimal speed for the target LBA.
|
||||
2. Issues SET CD SPEED (0xBB) if the speed differs from current.
|
||||
3. Performs the READ(10).
|
||||
|
||||
|
||||
@@ -1,240 +0,0 @@
|
||||
# MKV Native Muxer — Architecture
|
||||
|
||||
## Goal
|
||||
|
||||
Replace raw m2ts output with in-pipeline MKV muxing.
|
||||
Disc → TS demux → MKV mux → .mkv file. One pass, no temp files.
|
||||
|
||||
## Data Flow
|
||||
|
||||
```
|
||||
ContentReader::read_batch() returns &[u8] of raw BD transport stream
|
||||
↓
|
||||
TsDemuxer::feed(batch) parses 192-byte BD-TS packets, extracts PES
|
||||
↓
|
||||
PES reassembly per PID builds complete PES packets with PTS/DTS
|
||||
↓
|
||||
ElementaryStreamParser per track finds frame boundaries, extracts codec headers
|
||||
↓
|
||||
MkvMuxer::write_frame(track, pts, data) writes EBML clusters + blocks
|
||||
↓
|
||||
.mkv file on disk
|
||||
```
|
||||
|
||||
## Current Integration Point
|
||||
|
||||
```rust
|
||||
// rip.rs line ~287
|
||||
match reader.read_batch() {
|
||||
Ok(Some(batch)) => {
|
||||
writer.write_all(batch)?; // ← replace with muxer.feed(batch)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Becomes:
|
||||
```rust
|
||||
match reader.read_batch() {
|
||||
Ok(Some(batch)) => {
|
||||
muxer.feed(batch)?;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Components
|
||||
|
||||
### 1. BD Transport Stream Demuxer (`ts.rs`)
|
||||
|
||||
BD uses 192-byte packets (not standard 188):
|
||||
```
|
||||
[0-3] TP_extra_header: 2-bit copy_permission + 30-bit arrival_time_stamp
|
||||
[4] Sync byte: 0x47
|
||||
[5] TEI + PUSI + priority + PID[12:8]
|
||||
[6] PID[7:0]
|
||||
[7] Scrambling + adaptation + continuity_counter
|
||||
[8..] Adaptation field (if present) + payload
|
||||
```
|
||||
|
||||
API:
|
||||
```rust
|
||||
pub struct TsDemuxer {
|
||||
pes_assemblers: HashMap<u16, PesAssembler>, // PID → assembler
|
||||
}
|
||||
|
||||
impl TsDemuxer {
|
||||
pub fn new(pids: &[u16]) -> Self;
|
||||
pub fn feed(&mut self, data: &[u8]) -> Vec<PesPacket>;
|
||||
}
|
||||
|
||||
pub struct PesPacket {
|
||||
pub pid: u16,
|
||||
pub pts: Option<i64>, // 90kHz ticks
|
||||
pub dts: Option<i64>, // 90kHz ticks
|
||||
pub data: Vec<u8>, // elementary stream data
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Elementary Stream Parsers (`codec/`)
|
||||
|
||||
Each codec parser finds frame boundaries and extracts initialization data.
|
||||
|
||||
**H.264 (`codec/h264.rs`):**
|
||||
- Parse NAL units (start code 00 00 01 or 00 00 00 01)
|
||||
- Extract SPS + PPS for codecPrivate
|
||||
- Frame boundary = Access Unit Delimiter (NAL type 9) or SPS
|
||||
|
||||
**HEVC (`codec/hevc.rs`):**
|
||||
- Parse NAL units
|
||||
- Extract VPS + SPS + PPS for codecPrivate
|
||||
- Frame boundary = VCL NAL with first_slice_segment_in_pic_flag
|
||||
|
||||
**AC3/EAC3 (`codec/ac3.rs`):**
|
||||
- Syncword 0x0B77
|
||||
- Parse frame size from header
|
||||
- No codecPrivate needed (or minimal)
|
||||
|
||||
**DTS (`codec/dts.rs`):**
|
||||
- Syncword 0x7FFE8001
|
||||
- Parse frame size
|
||||
- No codecPrivate needed
|
||||
|
||||
**TrueHD (`codec/truehd.rs`):**
|
||||
- Major sync: 0xF8726FBA
|
||||
- Access unit = major sync + minor syncs
|
||||
- AC3 core embedded in first substream
|
||||
|
||||
**LPCM (`codec/lpcm.rs`):**
|
||||
- Fixed frame sizes based on sample rate + channels
|
||||
- Header describes format
|
||||
|
||||
**PGS (`codec/pgs.rs`):**
|
||||
- Segment types: PCS, WDS, PDS, ODS, END
|
||||
- Each segment is a complete unit
|
||||
- No codecPrivate needed
|
||||
|
||||
### 3. MKV/EBML Muxer (`mkv.rs`)
|
||||
|
||||
Matroska uses EBML (Extensible Binary Meta Language).
|
||||
|
||||
**EBML primitives:**
|
||||
- Variable-length element ID (1-4 bytes)
|
||||
- Variable-length size (1-8 bytes)
|
||||
- Data: uint, int, float, string, UTF-8, binary, date
|
||||
|
||||
**MKV structure:**
|
||||
```
|
||||
EBML Header
|
||||
Segment
|
||||
├── SeekHead (index of top-level elements)
|
||||
├── Info (title, duration, muxing app)
|
||||
├── Tracks (one entry per stream)
|
||||
│ ├── TrackEntry (video)
|
||||
│ │ ├── CodecID: "V_MPEG4/ISO/AVC" or "V_MPEGH/ISO/HEVC"
|
||||
│ │ ├── CodecPrivate: SPS+PPS (H.264) or VPS+SPS+PPS (HEVC)
|
||||
│ │ └── Video: PixelWidth, PixelHeight, DisplayWidth, DisplayHeight
|
||||
│ ├── TrackEntry (audio)
|
||||
│ │ ├── CodecID: "A_AC3" or "A_TRUEHD" or "A_DTS"
|
||||
│ │ └── Audio: SamplingFrequency, Channels, BitDepth
|
||||
│ └── TrackEntry (subtitle)
|
||||
│ └── CodecID: "S_HDMV/PGS"
|
||||
├── Chapters (optional, from MPLS chapter marks)
|
||||
├── Cluster (every ~5 seconds)
|
||||
│ ├── Timestamp (cluster base time)
|
||||
│ ├── SimpleBlock (track, relative_ts, data)
|
||||
│ ├── SimpleBlock ...
|
||||
│ └── ...
|
||||
├── Cluster ...
|
||||
├── Cues (seek index, written at end)
|
||||
└── Tags (metadata)
|
||||
```
|
||||
|
||||
**API:**
|
||||
```rust
|
||||
pub struct MkvMuxer<W: Write + Seek> {
|
||||
writer: W,
|
||||
tracks: Vec<MkvTrack>,
|
||||
cluster_start: Option<i64>,
|
||||
cue_points: Vec<CuePoint>,
|
||||
}
|
||||
|
||||
impl<W: Write + Seek> MkvMuxer<W> {
|
||||
pub fn new(writer: W, tracks: &[MkvTrack]) -> Result<Self>;
|
||||
pub fn write_frame(&mut self, track_idx: usize, pts_ns: i64, keyframe: bool, data: &[u8]) -> Result<()>;
|
||||
pub fn finish(self) -> Result<()>; // writes Cues + fixes SeekHead
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Pipeline Glue (`mux.rs`)
|
||||
|
||||
Ties everything together:
|
||||
|
||||
```rust
|
||||
pub struct MuxPipeline<W: Write + Seek> {
|
||||
demuxer: TsDemuxer,
|
||||
parsers: HashMap<u16, Box<dyn CodecParser>>,
|
||||
muxer: MkvMuxer<W>,
|
||||
pid_to_track: HashMap<u16, usize>,
|
||||
}
|
||||
|
||||
impl<W: Write + Seek> MuxPipeline<W> {
|
||||
pub fn new(writer: W, streams: &[Stream]) -> Result<Self>;
|
||||
pub fn feed(&mut self, ts_data: &[u8]) -> Result<()>;
|
||||
pub fn finish(self) -> Result<()>;
|
||||
}
|
||||
```
|
||||
|
||||
## File Layout
|
||||
|
||||
```
|
||||
libfreemkv/src/
|
||||
├── mux/
|
||||
│ ├── mod.rs MuxPipeline (glue)
|
||||
│ ├── ts.rs BD-TS demuxer (192-byte packets)
|
||||
│ ├── ebml.rs EBML primitives (write variable-length ints)
|
||||
│ ├── mkv.rs MKV muxer (Segment, Tracks, Clusters)
|
||||
│ └── codec/
|
||||
│ ├── mod.rs CodecParser trait
|
||||
│ ├── h264.rs H.264 NAL parser
|
||||
│ ├── hevc.rs HEVC NAL parser
|
||||
│ ├── ac3.rs AC3/EAC3 frame parser
|
||||
│ ├── dts.rs DTS frame parser
|
||||
│ ├── truehd.rs TrueHD/Atmos parser
|
||||
│ ├── lpcm.rs LPCM frame parser
|
||||
│ └── pgs.rs PGS subtitle parser
|
||||
```
|
||||
|
||||
## MKV Codec IDs
|
||||
|
||||
| Our Codec | MKV CodecID | codecPrivate |
|
||||
|-----------|------------|--------------|
|
||||
| H264 | V_MPEG4/ISO/AVC | AVCDecoderConfigurationRecord (SPS+PPS) |
|
||||
| Hevc | V_MPEGH/ISO/HEVC | HEVCDecoderConfigurationRecord (VPS+SPS+PPS) |
|
||||
| Vc1 | V_MS/VFW/FOURCC | BITMAPINFOHEADER |
|
||||
| Mpeg2 | V_MPEG2 | sequence_header |
|
||||
| Ac3 | A_AC3 | none |
|
||||
| Ac3Plus | A_EAC3 | none |
|
||||
| TrueHd | A_TRUEHD | none |
|
||||
| DtsHdMa | A_DTS | none (core + extension) |
|
||||
| Dts | A_DTS | none |
|
||||
| Lpcm | A_PCM/INT/BIG | none |
|
||||
| Pgs | S_HDMV/PGS | none |
|
||||
|
||||
## Timestamps
|
||||
|
||||
BD uses 90kHz PTS/DTS. MKV uses nanoseconds.
|
||||
Conversion: `ns = pts * 1_000_000_000 / 90_000` = `pts * 100_000 / 9`
|
||||
|
||||
MKV TimestampScale default = 1,000,000 (1ms precision).
|
||||
For BD content, 1ms is sufficient.
|
||||
|
||||
## Build Order
|
||||
|
||||
1. `ebml.rs` — EBML write primitives (smallest, no dependencies)
|
||||
2. `ts.rs` — BD-TS demuxer (parse 192-byte packets, PES assembly)
|
||||
3. `codec/ac3.rs` — simplest codec parser (fixed syncword)
|
||||
4. `mkv.rs` — MKV muxer (header, tracks, clusters, blocks)
|
||||
5. `mux.rs` — pipeline glue
|
||||
6. Test with AC3-only stream (simplest case)
|
||||
7. `codec/h264.rs` — video parser (NAL units, SPS/PPS)
|
||||
8. Full BD rip test (video + audio + subs)
|
||||
9. Remaining codecs (HEVC, DTS, TrueHD, PGS, LPCM, VC-1)
|
||||
@@ -0,0 +1,201 @@
|
||||
# Rip recovery — three-layer architecture
|
||||
|
||||
`libfreemkv` supports a multi-stage rip model for damaged or protection-bearing
|
||||
discs: a fast forward sweep that tolerates read failures, in-loop request-size
|
||||
adaptation that survives transient drive trouble without bailing, and targeted
|
||||
retry passes against a persistent bad-range map. The stream pipeline
|
||||
(`DiscStream` + `input`/`output`) operates against the resulting ISO image, so
|
||||
the mux stage never touches the drive.
|
||||
|
||||
Recovery is layered cleanly. Each layer has one responsibility and does not
|
||||
reach into the others.
|
||||
|
||||
| Layer | Where it lives | What it does |
|
||||
|-------|---------------|--------------|
|
||||
| 1 — Bad-range retry | `Disc::patch` (one pass over the mapfile per call) | Re-reads non-`+` ranges with the long timeout. Idempotent; caller invokes N times. |
|
||||
| 2 — Single-shot primitive | `Drive::read` in `src/drive/mod.rs` | One CDB, one timeout, one result. No inline retries, no SCSI reset. |
|
||||
| 3 — In-loop request adaptation | `DiscStream::fill_extents` adaptive batch sizer | Halves the batch on failure, retries at the same LBA, walks back up on a clean-read streak. |
|
||||
|
||||
The library exposes flat verbs; the caller drives the multipass loop. Autorip
|
||||
runs `Disc::sweep` once, then loops `Disc::patch` until either the mapfile is
|
||||
clean or the configured retry budget is exhausted, then hands the ISO off to
|
||||
the mux pipeline. The `freemkv` CLI does the same shape with a
|
||||
terminal-output progress sink. Layer 3 runs inside any consumer of
|
||||
`DiscStream` (direct PES pipeline, ISO playback, etc.) without caller
|
||||
involvement.
|
||||
|
||||
Three primitives compose the disc-side flow:
|
||||
|
||||
| Primitive | What it does |
|
||||
|---------------------------|-----------------------------------------------------------------------|
|
||||
| `Disc::sweep` | disc → ISO, one forward pass. Writes a sidecar `.mapfile`. Opt-in skip-on-error. |
|
||||
| `Disc::patch` | Re-reads bad ranges from the drive. One pass per call; caller invokes N times. |
|
||||
| `DiscStream` (ISO source) | Reads sectors from the ISO, feeds decrypt → demux → codec → mux. |
|
||||
|
||||
## Data model
|
||||
|
||||
### Mapfile
|
||||
|
||||
Format: [ddrescue](https://www.gnu.org/software/ddrescue/manual/ddrescue_manual.html)-compatible
|
||||
plain text, greppable, tool-interoperable. Flushed to disk on every `record()`
|
||||
so a crashed rip loses at most one block.
|
||||
|
||||
```
|
||||
# Rescue Logfile. Created by libfreemkv v0.13.6
|
||||
# Current pos / status / pass / pass_time
|
||||
0x000000000 ? 1 0
|
||||
# pos size status
|
||||
0x000000000 0x12a35d000 +
|
||||
0x12a35d000 0x000003000 -
|
||||
0x12a360000 0x009c4a000 +
|
||||
0x12d00a000 0x000064000 *
|
||||
```
|
||||
|
||||
Status characters match ddrescue:
|
||||
|
||||
| Char | Meaning |
|
||||
|------|----------------------------------------------------|
|
||||
| `?` | Not yet attempted |
|
||||
| `*` | Fast-pass failed; needs edge-trim |
|
||||
| `/` | Trimmed; interior needs sector scrape |
|
||||
| `-` | Unreadable this session |
|
||||
| `+` | Finished (good) |
|
||||
|
||||
Position and size are hex byte offsets into the ISO.
|
||||
|
||||
### `SweepOptions` and `PatchOptions`
|
||||
|
||||
The library no longer dispatches between sweep and patch internally — the
|
||||
caller picks the verb explicitly per pass. The two option structs are flat
|
||||
and have no overlap:
|
||||
|
||||
```rust
|
||||
SweepOptions {
|
||||
decrypt: true,
|
||||
resume: false,
|
||||
batch_sectors: None,
|
||||
skip_on_error: true, // damage-jump + zero-fill on read failure
|
||||
progress: Some(&reporter),
|
||||
halt: Some(flag),
|
||||
}
|
||||
|
||||
PatchOptions {
|
||||
decrypt: true,
|
||||
block_sectors: None,
|
||||
full_recovery: true,
|
||||
reverse: true, // walk bad ranges high → low LBA
|
||||
wedged_threshold: 50,
|
||||
progress: Some(&reporter),
|
||||
halt: Some(flag),
|
||||
}
|
||||
```
|
||||
|
||||
Caller-orchestrated dispatch (the policy `Disc::copy` used to embed):
|
||||
|
||||
- No mapfile → `sweep` (fresh Pass 1).
|
||||
- Mapfile with `?` ranges → `sweep` with `resume: true`.
|
||||
- Mapfile covers full disc, only `*` / `/` / `-` ranges → `patch`.
|
||||
- Mapfile clean → done; no further pass needed.
|
||||
|
||||
Each consumer (autorip, `freemkv` CLI) implements the loop in roughly five
|
||||
lines of `Mapfile::stats()` checks.
|
||||
|
||||
## Algorithm
|
||||
|
||||
### Pass 1 — fast sweep (`Disc::sweep`)
|
||||
|
||||
1. Read one ECC block (32 sectors for UHD, 16 for BD/DVD) at the current LBA.
|
||||
2. On success: write data to ISO, mark `+`, advance.
|
||||
3. On failure (with `multipass`): zero-fill, mark `*`, advance.
|
||||
4. Track a sliding window of the last 16 ECC block results. When ≥12% are failures
|
||||
→ **damage-jump**: skip ahead by `256×batch×multiplier` sectors (8 MB base for
|
||||
UHD). Double the multiplier on each jump (8→16→32→64 MB...). Zero-fill the gap as `*`.
|
||||
5. On 16 consecutive good reads: reset jump multiplier to 1, restore max read speed.
|
||||
6. Speed control: damage zone entry → minimum speed, exit → maximum speed.
|
||||
7. Only transport failures (USB bridge crash) abort the pass.
|
||||
|
||||
Pass 1 completes when every byte has been visited (either `+` or `*`).
|
||||
|
||||
### Pass 2+ — patch (`Disc::patch`)
|
||||
|
||||
`Disc::patch` reads the mapfile and iterates every non-`+` range. Default: **reverse** mode
|
||||
(walks ranges from highest LBA to lowest, within each range from end to start).
|
||||
|
||||
1. Issue a single-sector read with 60 s timeout (`recovery=true`). Drive firmware
|
||||
does its own ECC recovery inside that window.
|
||||
2. On success: write the good bytes into the ISO, mark `+`.
|
||||
3. On failure with non-marginal SCSI sense: bail immediately (drive won't produce data).
|
||||
4. On failure with marginal sense: mark `-`, continue.
|
||||
5. Update the mapfile after every block — crash-safe resume.
|
||||
6. Wedged-drive exit: 50 consecutive failures with zero recovery → bail this pass.
|
||||
|
||||
### In-stream — adaptive batch halving (`DiscStream::fill_extents`)
|
||||
|
||||
When a consumer reads a `DiscStream` directly (no ISO intermediate),
|
||||
`fill_extents` runs an adaptive sizer in front of `Drive::read`:
|
||||
|
||||
1. Try the current preferred batch size (e.g. 32 sectors, one BD ECC block).
|
||||
2. On failure: halve the batch and retry at the same LBA. Emit
|
||||
`EventKind::BatchSizeChanged { reason: Shrunk }`.
|
||||
3. On a clean-read streak: probe back up toward the preferred size. Emit
|
||||
`EventKind::BatchSizeChanged { reason: Probed }`.
|
||||
4. If a single-sector read fails: skip (zero-fill, emit
|
||||
`EventKind::SectorSkipped`) when `skip_errors` is set, otherwise return
|
||||
`Err(DiscRead)`.
|
||||
|
||||
This is layer 3. It exists so a transient single-sector glitch in a 32-sector
|
||||
batch can be isolated and read individually without the caller needing to
|
||||
implement retry logic.
|
||||
|
||||
## Design choices
|
||||
|
||||
**`Drive::read` is single-shot.** No inline retry phases, no SCSI reset,
|
||||
no eject cycle. The `recovery` flag controls only the per-CDB timeout
|
||||
(1.5 s vs. 30 s); on any failure it returns `Err(DiscRead)` immediately.
|
||||
Inline recovery (5× gentle retry → close + SCSI reset + reopen → 5× more)
|
||||
was removed in 0.13.6. See the stop-wedge postmortem (2026-04-25) for rationale:
|
||||
the inline reset on the LG BU40N (Initio USB-SATA bridge)
|
||||
wedged drive firmware below the bridge without ever recovering a sector,
|
||||
and the gentle-retry phase produced long stretches of 0 KB/s with no
|
||||
recoveries to show for it. Recovery responsibility is now layered: layer 1
|
||||
handles ranges, layer 3 handles request size, neither touches the
|
||||
wedge-prone reset path.
|
||||
|
||||
**No `MODE SELECT` to disable drive retries.** Research showed neither ddrescue
|
||||
nor MakeMKV does this. Drive firmware has access to raw analog signal, laser
|
||||
power control, and drive-specific ECC tuning that userspace can't replicate —
|
||||
disabling it throws away recovery headroom on marginal sectors. We fail fast
|
||||
via short SG_IO timeouts in pass 1 and let the firmware work the long timeout
|
||||
in pass 2 / patch.
|
||||
|
||||
**No SCSI reset from any retry path.** `SgIoTransport::reset` (Linux) is
|
||||
trimmed to a kernel SG_IO state flush plus ALLOW MEDIUM REMOVAL — the
|
||||
`SG_SCSI_RESET` ioctl and STOP/START UNIT escalation were removed in 0.13.6.
|
||||
The macOS reset (which had been a no-op) was removed entirely. The top-level
|
||||
`scsi::reset()` / `reset_with_timeout()` / `reset_blocking()` wrappers were
|
||||
also removed (no callers). The remaining `Drive::reset()` is only invoked
|
||||
explicitly by callers that need an eject-cycle escape hatch — it is never
|
||||
reached from a read path.
|
||||
|
||||
**ISO intermediate, even for single-pass.** Pass 1 always writes an ISO. The
|
||||
mux stage reads the ISO via `IsoSectorReader`. For single-pass (no retries),
|
||||
this adds ~2-3 min (local disk mux) but gains resumability across crashes,
|
||||
re-muxability without re-ripping, and a persistent forensic artifact. Callers
|
||||
who need pure speed can bypass and use `DiscStream::new(Box::new(drive), …)`
|
||||
directly — the lib doesn't forbid it, and layer 3 (adaptive batch halving)
|
||||
still applies there.
|
||||
|
||||
**Mapfile in ddrescue format.** Plain text so users can `less` it, `diff` it,
|
||||
or feed it to ddrescue's own tooling. Crash-safe (flush-per-record). Entries
|
||||
coalesce on adjacent same-status ranges so files stay small.
|
||||
|
||||
**Patches target `-`, `*`, `/`, and `?` alike.** The status state machine is
|
||||
ddrescue's but `patch` collapses the distinction — it just tries every
|
||||
non-finished range with the long timeout. Future work can specialize (trim vs.
|
||||
scrape vs. retry with direction reversal) if there's measured benefit.
|
||||
|
||||
## References
|
||||
|
||||
- [ddrescue manual, Algorithm chapter](https://www.gnu.org/software/ddrescue/manual/ddrescue_manual.html)
|
||||
- [ddrescue optical media notes](https://www.electric-spoon.com/doc/gddrescue/html/Optical-media.html)
|
||||
- Source: [`src/disc/mapfile.rs`](../src/disc/mapfile.rs), [`src/disc/sweep.rs`](../src/disc/sweep.rs) (`Disc::sweep`), [`src/disc/patch.rs`](../src/disc/patch.rs) (`Disc::patch`), [`src/drive/mod.rs`](../src/drive/mod.rs) (`Drive::read`), [`src/mux/disc.rs`](../src/mux/disc.rs) (`DiscStream::fill_extents`).
|
||||
@@ -120,6 +120,14 @@ Each directory read involves two sector reads: one for the ICB, then one or more
|
||||
|
||||
`read_file()` reads a file by navigating the directory tree, reading the file's ICB to get its data extent, then reading the data sector by sector from the **physical partition** (partition_start + LBA, not metadata_start).
|
||||
|
||||
## Buffered Sector Reads
|
||||
|
||||
USB optical drives have ~500ms round-trip latency per SCSI command. Since `read_filesystem()` and `read_file()` issue one SCSI READ per sector, a full disc scan can require hundreds of commands -- taking 10+ minutes on USB.
|
||||
|
||||
`Disc::scan()` wraps the drive in a `BufferedSectorReader` before reading. On a single-sector read, the buffer prefetches a batch of sectors (sized from the kernel's `max_hw_sectors_kb` for the device) and caches them. Subsequent reads to nearby LBAs return from cache with zero SCSI overhead. After parsing the UDF directory structure, the entire metadata partition is pre-read into the cache, so all ICB lookups during title scanning and encryption resolution are instant.
|
||||
|
||||
The buffer is transparent -- `read_filesystem()`, `read_file()`, and all downstream code still call `read_sectors(lba, 1, buf)` as before. The batching happens inside the `SectorSource` implementation.
|
||||
|
||||
### UDF Filename Encoding
|
||||
|
||||
UDF filenames use a compression ID as the first byte:
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
// Minimal ISO dumper — find exact stall point
|
||||
use libfreemkv::Drive;
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if args.len() < 3 {
|
||||
eprintln!("Usage: iso_dump <device> <output>");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let mut drive = Drive::open(Path::new(&args[1])).unwrap();
|
||||
drive.wait_ready().unwrap();
|
||||
let _ = drive.init();
|
||||
let _ = drive.probe_disc();
|
||||
|
||||
// AACS handshake — required to read past the protected area
|
||||
eprint!("Scanning disc... ");
|
||||
let _ = libfreemkv::Disc::scan(&mut drive, &libfreemkv::ScanOptions::default());
|
||||
eprintln!("OK");
|
||||
|
||||
let cap = drive.read_capacity().unwrap();
|
||||
let batch = libfreemkv::disc::detect_max_batch_sectors(drive.device_path());
|
||||
|
||||
eprintln!("Device: {} | {} sectors | batch {}", args[1], cap, batch);
|
||||
|
||||
let file = std::fs::File::create(&args[2]).unwrap();
|
||||
let mut w = BufWriter::with_capacity(4 * 1024 * 1024, file);
|
||||
let mut buf = vec![0u8; batch as usize * 2048];
|
||||
let mut lba: u32 = 0;
|
||||
let start = Instant::now();
|
||||
let mut last = Instant::now();
|
||||
let mut bytes: u64 = 0;
|
||||
let mut last_bytes: u64 = 0;
|
||||
|
||||
while lba < cap {
|
||||
let count = ((cap - lba) as u16).min(batch);
|
||||
let n = count as usize * 2048;
|
||||
|
||||
// Tiny yield between reads — test if pacing prevents firmware throttle
|
||||
std::thread::yield_now();
|
||||
let t0 = Instant::now();
|
||||
let ok = drive.read(lba, count, &mut buf[..n], true).is_ok();
|
||||
let read_ms = t0.elapsed().as_millis();
|
||||
|
||||
// Flag slow reads
|
||||
if read_ms > 2000 {
|
||||
eprintln!("\n SLOW READ: LBA {} took {}ms (ok={})", lba, read_ms, ok);
|
||||
}
|
||||
|
||||
if !ok {
|
||||
buf[..n].fill(0);
|
||||
}
|
||||
w.write_all(&buf[..n]).unwrap();
|
||||
lba += count as u32;
|
||||
bytes += n as u64;
|
||||
|
||||
if last.elapsed().as_millis() >= 1000 {
|
||||
let delta = bytes - last_bytes;
|
||||
let speed = delta as f64 / last.elapsed().as_secs_f64() / 1_048_576.0;
|
||||
let avg = bytes as f64 / start.elapsed().as_secs_f64() / 1_048_576.0;
|
||||
let pct = bytes as f64 / (cap as f64 * 2048.0) * 100.0;
|
||||
eprint!(
|
||||
"\r {:.1}% LBA {} | {:.0} MB/s (avg {:.0}) | {:.1} GB ",
|
||||
pct,
|
||||
lba,
|
||||
speed,
|
||||
avg,
|
||||
bytes as f64 / 1e9
|
||||
);
|
||||
last_bytes = bytes;
|
||||
last = Instant::now();
|
||||
}
|
||||
}
|
||||
w.flush().unwrap();
|
||||
eprintln!(
|
||||
"\nDone: {:.1} GB in {:.0}s",
|
||||
bytes as f64 / 1e9,
|
||||
start.elapsed().as_secs_f64()
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
max_width = 100
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
//! AACS content decryption — AES primitives, unit decryption, bus encryption.
|
||||
|
||||
use aes::cipher::{generic_array::GenericArray, BlockDecrypt, BlockEncrypt, KeyInit};
|
||||
use aes::Aes128;
|
||||
use aes::cipher::{BlockDecrypt, BlockEncrypt, KeyInit, generic_array::GenericArray};
|
||||
|
||||
// ── AACS constants ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
+9
-12
@@ -18,7 +18,7 @@
|
||||
//! - AACS 2.0: drives accept AACS 1.0 host certs for backward compatibility
|
||||
//! (full P-256/SHA-256 AACS 2.0 handshake prepared but rarely needed)
|
||||
|
||||
use crate::drive::DriveSession;
|
||||
use crate::drive::Drive;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::scsi::DataDirection;
|
||||
use num_bigint::BigUint;
|
||||
@@ -26,14 +26,14 @@ use num_traits::{One, Zero};
|
||||
use sha1::{Digest, Sha1};
|
||||
|
||||
/// Execute a SCSI command that reads data from the device.
|
||||
fn scsi_read(session: &mut DriveSession, cdb: &[u8], len: usize) -> Result<Vec<u8>> {
|
||||
fn scsi_read(session: &mut Drive, cdb: &[u8], len: usize) -> Result<Vec<u8>> {
|
||||
let mut buf = vec![0u8; len];
|
||||
session.scsi_execute(cdb, DataDirection::FromDevice, &mut buf, 5_000)?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// Execute a SCSI command that writes data to the device.
|
||||
fn scsi_write(session: &mut DriveSession, cdb: &[u8], data: &[u8]) -> Result<()> {
|
||||
fn scsi_write(session: &mut Drive, cdb: &[u8], data: &[u8]) -> Result<()> {
|
||||
let mut buf = data.to_vec();
|
||||
session.scsi_execute(cdb, DataDirection::ToDevice, &mut buf, 5_000)?;
|
||||
Ok(())
|
||||
@@ -674,8 +674,8 @@ fn generate_host_key_pair() -> ([u8; 20], [u8; 20], [u8; 20]) {
|
||||
|
||||
/// AES-128-CMAC over 16 bytes of data.
|
||||
fn aes_cmac_16(data: &[u8; 16], key: &[u8; 16]) -> [u8; 16] {
|
||||
use aes::cipher::{generic_array::GenericArray, BlockEncrypt, KeyInit};
|
||||
use aes::Aes128;
|
||||
use aes::cipher::{BlockEncrypt, KeyInit, generic_array::GenericArray};
|
||||
|
||||
let cipher = Aes128::new(GenericArray::from_slice(key));
|
||||
|
||||
@@ -765,7 +765,7 @@ pub struct AacsAuth {
|
||||
/// Requires a host private key (20 bytes) and host certificate (92 bytes)
|
||||
/// from the KEYDB.cfg HC entry.
|
||||
pub fn aacs_authenticate(
|
||||
session: &mut DriveSession,
|
||||
session: &mut Drive,
|
||||
host_priv_key: &[u8; 20],
|
||||
host_cert: &[u8],
|
||||
) -> Result<AacsAuth> {
|
||||
@@ -890,7 +890,7 @@ pub fn aacs_authenticate(
|
||||
/// Falls back to aacs_authenticate (AACS 1.0) if AACS 2.0 host credentials
|
||||
/// are not available.
|
||||
pub fn aacs2_authenticate(
|
||||
session: &mut DriveSession,
|
||||
session: &mut Drive,
|
||||
host_priv_key_v1: &[u8; 20],
|
||||
host_cert_v1: &[u8],
|
||||
host_priv_key_v2: Option<&[u8; 32]>,
|
||||
@@ -914,7 +914,7 @@ pub fn aacs2_authenticate(
|
||||
/// Native AACS 2.0 handshake using P-256/SHA-256.
|
||||
/// Same SCSI protocol, larger payloads (32-byte keys, 132-byte certs).
|
||||
fn aacs2_authenticate_p256(
|
||||
session: &mut DriveSession,
|
||||
session: &mut Drive,
|
||||
host_priv_key: &[u8; 32],
|
||||
host_cert: &[u8],
|
||||
) -> Result<AacsAuth> {
|
||||
@@ -1029,7 +1029,7 @@ fn aacs2_authenticate_p256(
|
||||
}
|
||||
|
||||
/// Read Volume ID after successful authentication.
|
||||
pub fn read_volume_id(session: &mut DriveSession, auth: &mut AacsAuth) -> Result<[u8; 16]> {
|
||||
pub fn read_volume_id(session: &mut Drive, auth: &mut AacsAuth) -> Result<[u8; 16]> {
|
||||
// REPORT DISC STRUCTURE format 0x80
|
||||
let cdb = cdb_report_disc_structure(auth.agid, 0x80, 36);
|
||||
let response = scsi_read(session, &cdb, 36).map_err(|_| Error::AacsVidRead)?;
|
||||
@@ -1050,10 +1050,7 @@ pub fn read_volume_id(session: &mut DriveSession, auth: &mut AacsAuth) -> Result
|
||||
}
|
||||
|
||||
/// Read data keys after successful authentication (for AACS 2.0 bus encryption).
|
||||
pub fn read_data_keys(
|
||||
session: &mut DriveSession,
|
||||
auth: &mut AacsAuth,
|
||||
) -> Result<([u8; 16], [u8; 16])> {
|
||||
pub fn read_data_keys(session: &mut Drive, auth: &mut AacsAuth) -> Result<([u8; 16], [u8; 16])> {
|
||||
// REPORT DISC STRUCTURE format 0x84
|
||||
let cdb = cdb_report_disc_structure(auth.agid, 0x84, 36);
|
||||
let response = scsi_read(session, &cdb, 36).map_err(|_| Error::AacsDataKey)?;
|
||||
|
||||
+4
-8
@@ -57,7 +57,7 @@ pub struct DiscEntry {
|
||||
/// Parse a hex string like "0xABCD..." into bytes.
|
||||
pub(crate) fn parse_hex(s: &str) -> Option<Vec<u8>> {
|
||||
let s = s.trim().trim_start_matches("0x").trim_start_matches("0X");
|
||||
if !s.len().is_multiple_of(2) {
|
||||
if s.len() % 2 != 0 {
|
||||
return None;
|
||||
}
|
||||
let mut out = Vec::with_capacity(s.len() / 2);
|
||||
@@ -167,7 +167,7 @@ impl KeyDb {
|
||||
.to_string();
|
||||
// Try with 0x prefix and without
|
||||
self.disc_entries
|
||||
.get(&format!("0x{}", hash))
|
||||
.get(&format!("0x{hash}"))
|
||||
.or_else(|| self.disc_entries.get(&hash))
|
||||
.and_then(|e| e.vuk)
|
||||
}
|
||||
@@ -180,7 +180,7 @@ impl KeyDb {
|
||||
.trim_start_matches("0x")
|
||||
.to_string();
|
||||
self.disc_entries
|
||||
.get(&format!("0x{}", hash))
|
||||
.get(&format!("0x{hash}"))
|
||||
.or_else(|| self.disc_entries.get(&hash))
|
||||
}
|
||||
|
||||
@@ -361,11 +361,7 @@ mod tests {
|
||||
/// Get KEYDB path from KEYDB_PATH environment variable. Returns None if not set or not found.
|
||||
fn keydb_path() -> Option<std::path::PathBuf> {
|
||||
let path = std::path::PathBuf::from(std::env::var("KEYDB_PATH").ok()?);
|
||||
if path.exists() {
|
||||
Some(path)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
if path.exists() { Some(path) } else { None }
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+71
-16
@@ -55,7 +55,7 @@ pub fn disc_hash_hex(hash: &[u8; 20]) -> String {
|
||||
let mut s = String::with_capacity(42);
|
||||
s.push_str("0x");
|
||||
for b in hash {
|
||||
s.push_str(&format!("{:02X}", b));
|
||||
s.push_str(&format!("{b:02X}"));
|
||||
}
|
||||
s
|
||||
}
|
||||
@@ -188,9 +188,13 @@ pub fn derive_media_key_from_pk(mkb: &[u8], processing_keys: &[[u8; 16]]) -> Opt
|
||||
// Try each processing key against each UV/cvalue pair
|
||||
for pk in processing_keys {
|
||||
for i in 0..num_uvs {
|
||||
if (i + 1) * 16 > cvalues.len() { continue; }
|
||||
if (i + 1) * 16 > cvalues.len() {
|
||||
continue;
|
||||
}
|
||||
let record_start = i * 5;
|
||||
if record_start + 5 > uvs.len() { continue; }
|
||||
if record_start + 5 > uvs.len() {
|
||||
continue;
|
||||
}
|
||||
let _u_mask_shift = uvs[record_start];
|
||||
let uv = &uvs[record_start + 1..record_start + 5];
|
||||
let cv = &cvalues[i * 16..(i + 1) * 16];
|
||||
@@ -243,6 +247,7 @@ fn validate_processing_key(
|
||||
/// Find Verify Media Key Record (type 0x10) in MKB.
|
||||
fn mkb_find_mk_dv(mkb: &[u8]) -> Option<[u8; 16]> {
|
||||
let mut pos = 0;
|
||||
let mut type10_seen: Vec<(usize, usize)> = Vec::new();
|
||||
while pos + 4 <= mkb.len() {
|
||||
let rec_type = mkb[pos];
|
||||
let rec_len = u32::from_be_bytes([0, mkb[pos + 1], mkb[pos + 2], mkb[pos + 3]]) as usize;
|
||||
@@ -250,14 +255,32 @@ fn mkb_find_mk_dv(mkb: &[u8]) -> Option<[u8; 16]> {
|
||||
break;
|
||||
}
|
||||
|
||||
if rec_type == 0x10 {
|
||||
type10_seen.push((pos, rec_len));
|
||||
}
|
||||
|
||||
if rec_type == 0x10 && rec_len >= 20 {
|
||||
// mk_dv is at offset 4 (after record header)
|
||||
let mut dv = [0u8; 16];
|
||||
dv.copy_from_slice(&mkb[pos + 4..pos + 20]);
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "mkb_mk_dv_found",
|
||||
pos,
|
||||
rec_len,
|
||||
"mk_dv extracted from MKB"
|
||||
);
|
||||
return Some(dv);
|
||||
}
|
||||
pos += rec_len;
|
||||
}
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "mkb_mk_dv_not_found",
|
||||
type10_seen = ?type10_seen,
|
||||
scanned_bytes = pos,
|
||||
"no 0x10 record with rec_len>=20 found"
|
||||
);
|
||||
None
|
||||
}
|
||||
|
||||
@@ -341,7 +364,7 @@ fn aesg3(key: &[u8; 16], inc: u8) -> [u8; 16] {
|
||||
|
||||
/// Compute v_mask from a UV value.
|
||||
fn calc_v_mask(uv: u32) -> u32 {
|
||||
let mut v_mask: u32 = 0xFFFFFFFF;
|
||||
let mut v_mask: u32 = 0xFFFF_FFFF;
|
||||
while (uv & !v_mask) == 0 && v_mask != 0 {
|
||||
v_mask <<= 1;
|
||||
}
|
||||
@@ -411,7 +434,7 @@ pub fn derive_media_key_from_dk(mkb: &[u8], device_keys: &[DeviceKey]) -> Option
|
||||
continue;
|
||||
}
|
||||
|
||||
let u_mask: u32 = 0xFFFFFFFF << u_mask_shift;
|
||||
let u_mask: u32 = 0xFFFF_FFFF << u_mask_shift;
|
||||
let v_mask = calc_v_mask(uv);
|
||||
|
||||
if ((device_number & u_mask) == (uv & u_mask))
|
||||
@@ -419,7 +442,7 @@ pub fn derive_media_key_from_dk(mkb: &[u8], device_keys: &[DeviceKey]) -> Option
|
||||
{
|
||||
// Found matching subset-difference — find the right device key
|
||||
let dev_key_v_mask = calc_v_mask(dk.uv);
|
||||
let dev_key_u_mask: u32 = 0xFFFFFFFF << dk.u_mask_shift;
|
||||
let dev_key_u_mask: u32 = 0xFFFF_FFFF << dk.u_mask_shift;
|
||||
|
||||
if u_mask == dev_key_u_mask && (uv & dev_key_v_mask) == (dk.uv & dev_key_v_mask) {
|
||||
// Derive processing key via tree traversal
|
||||
@@ -448,9 +471,7 @@ const MKB_PACK_SIZE: usize = 32772;
|
||||
|
||||
/// Read MKB from drive via SCSI (REPORT DISC STRUCTURE format 0x83).
|
||||
/// Returns the concatenated MKB data from all packs.
|
||||
pub fn read_mkb_from_drive(
|
||||
session: &mut crate::drive::DriveSession,
|
||||
) -> crate::error::Result<Vec<u8>> {
|
||||
pub fn read_mkb_from_drive(session: &mut crate::drive::Drive) -> crate::error::Result<Vec<u8>> {
|
||||
use crate::scsi::{DataDirection, SCSI_READ_DISC_STRUCTURE};
|
||||
|
||||
let cdb = [
|
||||
@@ -626,32 +647,70 @@ pub fn resolve_keys(
|
||||
}
|
||||
};
|
||||
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "resolve_keys_start",
|
||||
aacs2,
|
||||
bus_encryption,
|
||||
disc_hash = %hash_hex,
|
||||
mkb_present = mkb_data.is_some(),
|
||||
"resolve_keys: starting"
|
||||
);
|
||||
|
||||
// Path 1: Look up VUK by disc hash in KEYDB
|
||||
if let Some(entry) = keydb.find_disc(&hash_hex) {
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path1_hit_entry", "disc hash found in keydb");
|
||||
if let Some(vuk) = entry.vuk {
|
||||
return Some(build(vuk, 1));
|
||||
}
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path1_no_vuk", "disc hash entry has no VUK");
|
||||
} else {
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path1_miss", "disc hash NOT in keydb");
|
||||
}
|
||||
|
||||
// Path 2: Find entry with matching VID → derive VUK from MK + VID
|
||||
let mut path2_mk_did_count = 0usize;
|
||||
for entry in keydb.disc_entries.values() {
|
||||
if let (Some(mk), Some(did)) = (entry.media_key, entry.disc_id) {
|
||||
path2_mk_did_count += 1;
|
||||
if did == *volume_id {
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path2_hit", "MK+VID entry matched volume_id");
|
||||
return Some(build(derive_vuk(&mk, volume_id), 2));
|
||||
}
|
||||
}
|
||||
}
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path2_miss", mk_did_entries = path2_mk_did_count, "no MK+VID entry matched volume_id");
|
||||
|
||||
// Path 3: MKB + processing keys → media key → VUK
|
||||
if let Some(mkb) = mkb_data {
|
||||
let mk_dv = mkb_find_mk_dv(mkb);
|
||||
let subdiff = mkb_find_subdiff_records(mkb);
|
||||
let cvalues = mkb_find_cvalues(mkb);
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "resolve_keys_mkb_records",
|
||||
mk_dv_found = mk_dv.is_some(),
|
||||
subdiff_found = subdiff.is_some(),
|
||||
subdiff_len = subdiff.as_ref().map(|s| s.len()).unwrap_or(0),
|
||||
cvalues_found = cvalues.is_some(),
|
||||
cvalues_len = cvalues.as_ref().map(|c| c.len()).unwrap_or(0),
|
||||
"MKB record scan results"
|
||||
);
|
||||
|
||||
if let Some(mk) = derive_media_key_from_pk(mkb, &keydb.processing_keys) {
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path3_hit", "media key derived from processing key");
|
||||
return Some(build(derive_vuk(&mk, volume_id), 3));
|
||||
}
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path3_miss", pk_count = keydb.processing_keys.len(), "PK derivation failed");
|
||||
|
||||
// Path 4: MKB + device keys → processing key → media key → VUK
|
||||
if let Some(mk) = derive_media_key_from_dk(mkb, &keydb.device_keys) {
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path4_hit", "media key derived from device key");
|
||||
return Some(build(derive_vuk(&mk, volume_id), 4));
|
||||
}
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path4_miss", dk_count = keydb.device_keys.len(), "DK derivation failed");
|
||||
} else {
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_no_mkb", "no MKB data available; paths 3/4 skipped");
|
||||
}
|
||||
|
||||
None
|
||||
@@ -659,18 +718,14 @@ pub fn resolve_keys(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::decrypt::{aes_ecb_encrypt, ALIGNED_UNIT_LEN};
|
||||
use super::super::decrypt::{ALIGNED_UNIT_LEN, aes_ecb_encrypt};
|
||||
use super::super::keydb::{DiscEntry, KeyDb};
|
||||
use super::*;
|
||||
|
||||
/// Get KEYDB path from KEYDB_PATH environment variable. Returns None if not set or not found.
|
||||
fn keydb_path() -> Option<std::path::PathBuf> {
|
||||
let path = std::path::PathBuf::from(std::env::var("KEYDB_PATH").ok()?);
|
||||
if path.exists() {
|
||||
Some(path)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
if path.exists() { Some(path) } else { None }
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -847,7 +902,7 @@ mod tests {
|
||||
data[23] = 1; // top_menu = CPS unit 1
|
||||
data[24] = 0;
|
||||
data[25] = 1; // num_titles = 1
|
||||
// Title 0 entry: 2 bytes pad + CPS unit
|
||||
// Title 0 entry: 2 bytes pad + CPS unit
|
||||
data[28] = 0;
|
||||
data[29] = 1; // CPS unit 1
|
||||
|
||||
|
||||
+5
-5
@@ -21,12 +21,12 @@ pub mod keys;
|
||||
// Explicit re-exports — only items needed by external consumers and sibling crate modules.
|
||||
// AES primitives (aes_ecb_encrypt, aes_ecb_decrypt, aes_cbc_decrypt) are pub(crate) in decrypt.rs.
|
||||
pub use decrypt::{
|
||||
decrypt_bus, decrypt_unit, decrypt_unit_full, decrypt_unit_try_keys, is_unit_encrypted,
|
||||
ALIGNED_UNIT_LEN,
|
||||
ALIGNED_UNIT_LEN, decrypt_bus, decrypt_unit, decrypt_unit_full, decrypt_unit_try_keys,
|
||||
is_unit_encrypted,
|
||||
};
|
||||
pub use keydb::{DeviceKey, DiscEntry, HostCert, KeyDb};
|
||||
pub use keys::{
|
||||
decrypt_unit_key, derive_media_key_from_dk, derive_media_key_from_pk, derive_vuk, disc_hash,
|
||||
disc_hash_hex, mkb_version, parse_content_cert, parse_unit_key_ro, read_mkb_from_drive,
|
||||
resolve_keys, ContentCert, ResolvedKeys, UnitKeyFile,
|
||||
ContentCert, ResolvedKeys, UnitKeyFile, decrypt_unit_key, derive_media_key_from_dk,
|
||||
derive_media_key_from_pk, derive_vuk, disc_hash, disc_hash_hex, mkb_version,
|
||||
parse_content_cert, parse_unit_key_ro, read_mkb_from_drive, resolve_keys,
|
||||
};
|
||||
|
||||
+159
-2
@@ -20,6 +20,35 @@ pub struct ClipInfo {
|
||||
pub ep_coarse: Vec<EpCoarse>,
|
||||
/// Fine EP entries for the primary video stream
|
||||
pub ep_fine: Vec<EpFine>,
|
||||
/// Per-stream metadata from the ProgramInfo section (BD spec).
|
||||
/// Cross-validates the MPLS STN view — see `labels/clpi.rs`.
|
||||
/// Empty when program_info is missing or malformed.
|
||||
pub streams: Vec<ClpiStream>,
|
||||
}
|
||||
|
||||
/// One stream descriptor from the CLPI ProgramInfo / stream_coding_info
|
||||
/// table. Mirrors the same fields the MPLS STN table carries — see
|
||||
/// `mpls::StreamEntry` for the playlist-side equivalent.
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct ClpiStream {
|
||||
/// PID of the stream in the MPEG-TS (matches MPLS).
|
||||
pub pid: u16,
|
||||
/// SCSI/BD coding type byte (0x80 LPCM, 0x83 TrueHD, 0x86 DTS-HD MA,
|
||||
/// 0x90 PG, etc.). See `labels::mpls_universal::coding_type_to_codec_hint`.
|
||||
pub coding_type: u8,
|
||||
/// ISO 639-2 3-char language code. Empty for video streams.
|
||||
pub language: String,
|
||||
/// Audio format byte (1=mono, 3=stereo, 6=5.1, 12=7.1).
|
||||
/// Zero for non-audio streams.
|
||||
pub audio_format: u8,
|
||||
/// Audio sample rate (1=48kHz, 4=96kHz, 5=192kHz). Zero for non-audio.
|
||||
pub audio_rate: u8,
|
||||
/// Video format byte (1=480i, 4=1080i, 5=720p, 6=1080p, 8=2160p).
|
||||
/// Zero for non-video.
|
||||
pub video_format: u8,
|
||||
/// Video rate (1=23.976, 2=24, 3=25, 4=29.97, 6=50, 7=59.94).
|
||||
pub video_rate: u8,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -46,7 +75,7 @@ impl ClipInfo {
|
||||
|
||||
/// Reconstruct full SPN from coarse + fine entry.
|
||||
pub fn full_spn(coarse: &EpCoarse, fine: &EpFine) -> u32 {
|
||||
(coarse.spn_coarse & 0xFFFE0000) + fine.spn_fine
|
||||
(coarse.spn_coarse & 0xFFFE_0000) + fine.spn_fine
|
||||
}
|
||||
|
||||
/// Get all EP entries as (PTS, SPN) pairs, fully resolved.
|
||||
@@ -128,7 +157,7 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
|
||||
|
||||
// Header offsets
|
||||
let _seq_info_start = u32::from_be_bytes([data[8], data[9], data[10], data[11]]) as usize;
|
||||
let _prog_info_start = u32::from_be_bytes([data[12], data[13], data[14], data[15]]) as usize;
|
||||
let prog_info_start = u32::from_be_bytes([data[12], data[13], data[14], data[15]]) as usize;
|
||||
let cpi_start = u32::from_be_bytes([data[16], data[17], data[18], data[19]]) as usize;
|
||||
|
||||
// ClipInfo section at offset 40
|
||||
@@ -139,6 +168,16 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
|
||||
0
|
||||
};
|
||||
|
||||
// Parse ProgramInfo (per-stream language + codec). Best-effort:
|
||||
// malformed program_info doesn't fail the parse, just gives an
|
||||
// empty streams list. EP map is unaffected — sector-range lookups
|
||||
// continue to work.
|
||||
let streams = if prog_info_start > 0 && prog_info_start + 6 < data.len() {
|
||||
parse_program_info(&data[prog_info_start..])
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
// Parse CPI / EP Map
|
||||
let (ep_coarse, ep_fine) = if cpi_start > 0 && cpi_start + 8 < data.len() {
|
||||
parse_cpi(&data[cpi_start..])?
|
||||
@@ -151,9 +190,127 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
|
||||
source_packet_count,
|
||||
ep_coarse,
|
||||
ep_fine,
|
||||
streams,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse the ProgramInfo section: per-stream (pid, coding_type,
|
||||
/// language, codec sub-fields). Layout per BD spec / libbluray
|
||||
/// clpi_parse.c:
|
||||
///
|
||||
/// ```text
|
||||
/// ProgramInfo:
|
||||
/// length: 4 bytes
|
||||
/// reserved: 1 byte
|
||||
/// num_programs: 1 byte
|
||||
/// for each program:
|
||||
/// spn_program_sequence_start: 4 bytes
|
||||
/// program_map_pid: 2 bytes
|
||||
/// num_streams: 1 byte
|
||||
/// num_groups: 1 byte
|
||||
/// for each stream:
|
||||
/// pid: 2 bytes
|
||||
/// stream_coding_info_length: 1 byte
|
||||
/// stream_coding_info: (varies by coding_type)
|
||||
/// coding_type: 1 byte
|
||||
/// per-type bytes (see match arms below)
|
||||
/// ```
|
||||
///
|
||||
/// Returns `Vec::new()` on any structural mismatch — we don't propagate
|
||||
/// errors because the EP map is the primary CLPI output, and a corrupt
|
||||
/// program_info shouldn't break sector-range lookups.
|
||||
fn parse_program_info(data: &[u8]) -> Vec<ClpiStream> {
|
||||
let mut out = Vec::new();
|
||||
if data.len() < 6 {
|
||||
return out;
|
||||
}
|
||||
// length: 4 bytes (skipped — we trust the section bounds in the
|
||||
// caller's slice and read the bytes that follow). Reserved 1 byte
|
||||
// at offset 4. num_programs at offset 5.
|
||||
let num_programs = data[5] as usize;
|
||||
let mut pos = 6usize;
|
||||
for _ in 0..num_programs {
|
||||
// Program header: 4 (spn) + 2 (pmt_pid) + 1 (num_streams) + 1 (num_groups) = 8 bytes
|
||||
if pos + 8 > data.len() {
|
||||
return out;
|
||||
}
|
||||
let num_streams = data[pos + 6] as usize;
|
||||
pos += 8;
|
||||
|
||||
for _ in 0..num_streams {
|
||||
// Stream header: 2 (pid) + 1 (sci_length) + sci bytes
|
||||
if pos + 3 > data.len() {
|
||||
return out;
|
||||
}
|
||||
let pid = u16::from_be_bytes([data[pos], data[pos + 1]]);
|
||||
let sci_len = data[pos + 2] as usize;
|
||||
let sci_end = pos + 3 + sci_len;
|
||||
if sci_end > data.len() || sci_len < 1 {
|
||||
return out;
|
||||
}
|
||||
let sci = &data[pos + 3..sci_end];
|
||||
let coding_type = sci[0];
|
||||
|
||||
let mut audio_format = 0u8;
|
||||
let mut audio_rate = 0u8;
|
||||
let mut video_format = 0u8;
|
||||
let mut video_rate = 0u8;
|
||||
let mut language = String::new();
|
||||
|
||||
match coding_type {
|
||||
// Video — MPEG-2 (0x02), H.264 (0x1B), HEVC (0x24)
|
||||
0x02 | 0x1B | 0x24 => {
|
||||
if sci.len() >= 2 {
|
||||
video_format = (sci[1] >> 4) & 0x0F;
|
||||
video_rate = sci[1] & 0x0F;
|
||||
}
|
||||
}
|
||||
// Primary audio — LPCM(0x80), AC-3(0x81), DTS(0x82),
|
||||
// TrueHD(0x83), AC-3+(0x84), DTS-HD(0x85), DTS-HD MA(0x86)
|
||||
0x80..=0x86 => {
|
||||
if sci.len() >= 2 {
|
||||
audio_format = (sci[1] >> 4) & 0x0F;
|
||||
audio_rate = sci[1] & 0x0F;
|
||||
}
|
||||
if sci.len() >= 5 {
|
||||
language = String::from_utf8_lossy(&sci[2..5]).to_string();
|
||||
}
|
||||
}
|
||||
// Secondary audio (0xA1 AC-3+, 0xA2 DTS-HD)
|
||||
0xA1 | 0xA2 => {
|
||||
if sci.len() >= 2 {
|
||||
audio_format = (sci[1] >> 4) & 0x0F;
|
||||
audio_rate = sci[1] & 0x0F;
|
||||
}
|
||||
if sci.len() >= 5 {
|
||||
language = String::from_utf8_lossy(&sci[2..5]).to_string();
|
||||
}
|
||||
}
|
||||
// PG (0x90), IG (0x91): coding_type + 3-byte language [+ char_code for PG]
|
||||
0x90 | 0x91 => {
|
||||
if sci.len() >= 4 {
|
||||
language = String::from_utf8_lossy(&sci[1..4]).to_string();
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
out.push(ClpiStream {
|
||||
pid,
|
||||
coding_type,
|
||||
language,
|
||||
audio_format,
|
||||
audio_rate,
|
||||
video_format,
|
||||
video_rate,
|
||||
});
|
||||
|
||||
pos = sci_end;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Parse the CPI section containing the EP map.
|
||||
fn parse_cpi(data: &[u8]) -> Result<(Vec<EpCoarse>, Vec<EpFine>)> {
|
||||
if data.len() < 8 {
|
||||
|
||||
+638
@@ -0,0 +1,638 @@
|
||||
//! CSS drive authentication — full key hierarchy.
|
||||
//!
|
||||
//! Protocol:
|
||||
//! 1. Bus authentication (challenge-response) → bus key
|
||||
//! 2. Read disc key block (READ DVD STRUCTURE) → XOR with bus key → decrypt with player keys → disc key
|
||||
//! 3. Read title key (REPORT KEY format 0x04) → XOR with bus key → decrypt with disc key → title key
|
||||
//!
|
||||
//! Based on libdvdcss (VideoLAN) and Stevenson 1999 analysis.
|
||||
|
||||
use crate::drive::Drive;
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
// ── Player keys (from libdvdcss, Stevenson's PlayerKey cracker) ───────────
|
||||
|
||||
const PLAYER_KEYS: [[u8; 5]; 31] = [
|
||||
[0x01, 0xaf, 0xe3, 0x12, 0x80],
|
||||
[0x12, 0x11, 0xca, 0x04, 0x3b],
|
||||
[0x14, 0x0c, 0x9e, 0xd0, 0x09],
|
||||
[0x14, 0x71, 0x35, 0xba, 0xe2],
|
||||
[0x1a, 0xa4, 0x33, 0x21, 0xa6],
|
||||
[0x26, 0xec, 0xc4, 0xa7, 0x4e],
|
||||
[0x2c, 0xb2, 0xc1, 0x09, 0xee],
|
||||
[0x2f, 0x25, 0x9e, 0x96, 0xdd],
|
||||
[0x33, 0x2f, 0x49, 0x6c, 0xe0],
|
||||
[0x35, 0x5b, 0xc1, 0x31, 0x0f],
|
||||
[0x36, 0x67, 0xb2, 0xe3, 0x85],
|
||||
[0x39, 0x3d, 0xf1, 0xf1, 0xbd],
|
||||
[0x3b, 0x31, 0x34, 0x0d, 0x91],
|
||||
[0x45, 0xed, 0x28, 0xeb, 0xd3],
|
||||
[0x48, 0xb7, 0x6c, 0xce, 0x69],
|
||||
[0x4b, 0x65, 0x0d, 0xc1, 0xee],
|
||||
[0x4c, 0xbb, 0xf5, 0x5b, 0x23],
|
||||
[0x51, 0x67, 0x67, 0xc5, 0xe0],
|
||||
[0x53, 0x94, 0xe1, 0x75, 0xbf],
|
||||
[0x57, 0x2c, 0x8b, 0x31, 0xae],
|
||||
[0x63, 0xdb, 0x4c, 0x5b, 0x4a],
|
||||
[0x7b, 0x1e, 0x5e, 0x2b, 0x57],
|
||||
[0x85, 0xf3, 0x85, 0xa0, 0xe0],
|
||||
[0xab, 0x1e, 0xe7, 0x7b, 0x72],
|
||||
[0xab, 0x36, 0xe3, 0xeb, 0x76],
|
||||
[0xb1, 0xb8, 0xf9, 0x38, 0x03],
|
||||
[0xb8, 0x5d, 0xd8, 0x53, 0xbd],
|
||||
[0xbf, 0x92, 0xc3, 0xb0, 0xe2],
|
||||
[0xcf, 0x1a, 0xb2, 0xf8, 0x0a],
|
||||
[0xec, 0xa0, 0xcf, 0xb3, 0xff],
|
||||
[0xfc, 0x95, 0xa9, 0x87, 0x35],
|
||||
];
|
||||
|
||||
// ── CryptKey tables ───────────────────────────────────────────────────────
|
||||
|
||||
const CRYPT_TAB0: [u8; 256] = [
|
||||
0xB7, 0xF4, 0x82, 0x57, 0xDA, 0x4D, 0xDB, 0xE2, 0x2F, 0x52, 0x1A, 0xA8, 0x68, 0x5A, 0x8A, 0xFF,
|
||||
0xFB, 0x0E, 0x6D, 0x35, 0xF7, 0x5C, 0x76, 0x12, 0xCE, 0x25, 0x79, 0x29, 0x39, 0x62, 0x08, 0x24,
|
||||
0xA5, 0x85, 0x7B, 0x56, 0x01, 0x23, 0x68, 0xCF, 0x0A, 0xE2, 0x5A, 0xED, 0x3D, 0x59, 0xB0, 0xA9,
|
||||
0xB0, 0x2C, 0xF2, 0xB8, 0xEF, 0x32, 0xA9, 0x40, 0x80, 0x71, 0xAF, 0x1E, 0xDE, 0x8F, 0x58, 0x88,
|
||||
0xB8, 0x3A, 0xD0, 0xFC, 0xC4, 0x1E, 0xB5, 0xA0, 0xBB, 0x3B, 0x0F, 0x01, 0x7E, 0x1F, 0x9F, 0xD9,
|
||||
0xAA, 0xB8, 0x3D, 0x9D, 0x74, 0x1E, 0x25, 0xDB, 0x37, 0x56, 0x8F, 0x16, 0xBA, 0x49, 0x2B, 0xAC,
|
||||
0xD0, 0xBD, 0x95, 0x20, 0xBE, 0x7A, 0x28, 0xD0, 0x51, 0x64, 0x63, 0x1C, 0x7F, 0x66, 0x10, 0xBB,
|
||||
0xC4, 0x56, 0x1A, 0x04, 0x6E, 0x0A, 0xEC, 0x9C, 0xD6, 0xE8, 0x9A, 0x7A, 0xCF, 0x8C, 0xDB, 0xB1,
|
||||
0xEF, 0x71, 0xDE, 0x31, 0xFF, 0x54, 0x3E, 0x5E, 0x07, 0x69, 0x96, 0xB0, 0xCF, 0xDD, 0x9E, 0x47,
|
||||
0xC7, 0x96, 0x8F, 0xE4, 0x2B, 0x59, 0xC6, 0xEE, 0xB9, 0x86, 0x9A, 0x64, 0x84, 0x72, 0xE2, 0x5B,
|
||||
0xA2, 0x96, 0x58, 0x99, 0x50, 0x03, 0xF5, 0x38, 0x4D, 0x02, 0x7D, 0xE7, 0x7D, 0x75, 0xA7, 0xB8,
|
||||
0x67, 0x87, 0x84, 0x3F, 0x1D, 0x11, 0xE5, 0xFC, 0x1E, 0xD3, 0x83, 0x16, 0xA5, 0x29, 0xF6, 0xC7,
|
||||
0x15, 0x61, 0x29, 0x1A, 0x43, 0x4F, 0x9B, 0xAF, 0xC5, 0x87, 0x34, 0x6C, 0x0F, 0x3B, 0xA8, 0x1D,
|
||||
0x45, 0x58, 0x25, 0xDC, 0xA8, 0xA3, 0x3B, 0xD1, 0x79, 0x1B, 0x48, 0xF2, 0xE9, 0x93, 0x1F, 0xFC,
|
||||
0xDB, 0x2A, 0x90, 0xA9, 0x8A, 0x3D, 0x39, 0x18, 0xA3, 0x8E, 0x58, 0x6C, 0xE0, 0x12, 0xBB, 0x25,
|
||||
0xCD, 0x71, 0x22, 0xA2, 0x64, 0xC6, 0xE7, 0xFB, 0xAD, 0x94, 0x77, 0x04, 0x9A, 0x39, 0xCF, 0x7C,
|
||||
];
|
||||
|
||||
const CRYPT_TAB1: [u8; 256] = [
|
||||
0x8C, 0x47, 0xB0, 0xE1, 0xEB, 0xFC, 0xEB, 0x56, 0x10, 0xE5, 0x2C, 0x1A, 0x5D, 0xEF, 0xBE, 0x4F,
|
||||
0x08, 0x75, 0x97, 0x4B, 0x0E, 0x25, 0x8E, 0x6E, 0x39, 0x5A, 0x87, 0x53, 0xC4, 0x1F, 0xF4, 0x5C,
|
||||
0x4E, 0xE6, 0x99, 0x30, 0xE0, 0x42, 0x88, 0xAB, 0xE5, 0x85, 0xBC, 0x8F, 0xD8, 0x3C, 0x54, 0xC9,
|
||||
0x53, 0x47, 0x18, 0xD6, 0x06, 0x5B, 0x41, 0x2C, 0x67, 0x1E, 0x41, 0x74, 0x33, 0xE2, 0xB4, 0xE0,
|
||||
0x23, 0x29, 0x42, 0xEA, 0x55, 0x0F, 0x25, 0xB4, 0x24, 0x2C, 0x99, 0x13, 0xEB, 0x0A, 0x0B, 0xC9,
|
||||
0xF9, 0x63, 0x67, 0x43, 0x2D, 0xC7, 0x7D, 0x07, 0x60, 0x89, 0xD1, 0xCC, 0xE7, 0x94, 0x77, 0x74,
|
||||
0x9B, 0x7E, 0xD7, 0xE6, 0xFF, 0xBB, 0x68, 0x14, 0x1E, 0xA3, 0x25, 0xDE, 0x3A, 0xA3, 0x54, 0x7B,
|
||||
0x87, 0x9D, 0x50, 0xCA, 0x27, 0xC3, 0xA4, 0x50, 0x91, 0x27, 0xD4, 0xB0, 0x82, 0x41, 0x97, 0x79,
|
||||
0x94, 0x82, 0xAC, 0xC7, 0x8E, 0xA5, 0x4E, 0xAA, 0x78, 0x9E, 0xE0, 0x42, 0xBA, 0x28, 0xEA, 0xB7,
|
||||
0x74, 0xAD, 0x35, 0xDA, 0x92, 0x60, 0x7E, 0xD2, 0x0E, 0xB9, 0x24, 0x5E, 0x39, 0x4F, 0x5E, 0x63,
|
||||
0x09, 0xB5, 0xFA, 0xBF, 0xF1, 0x22, 0x55, 0x1C, 0xE2, 0x25, 0xDB, 0xC5, 0xD8, 0x50, 0x03, 0x98,
|
||||
0xC4, 0xAC, 0x2E, 0x11, 0xB4, 0x38, 0x4D, 0xD0, 0xB9, 0xFC, 0x2D, 0x3C, 0x08, 0x04, 0x5A, 0xEF,
|
||||
0xCE, 0x32, 0xFB, 0x4C, 0x92, 0x1E, 0x4B, 0xFB, 0x1A, 0xD0, 0xE2, 0x3E, 0xDA, 0x6E, 0x7C, 0x4D,
|
||||
0x56, 0xC3, 0x3F, 0x42, 0xB1, 0x3A, 0x23, 0x4D, 0x6E, 0x84, 0x56, 0x68, 0xF4, 0x0E, 0x03, 0x64,
|
||||
0xD0, 0xA9, 0x92, 0x2F, 0x8B, 0xBC, 0x39, 0x9C, 0xAC, 0x09, 0x5E, 0xEE, 0xE5, 0x97, 0xBF, 0xA5,
|
||||
0xCE, 0xFA, 0x28, 0x2C, 0x6D, 0x4F, 0xEF, 0x77, 0xAA, 0x1B, 0x79, 0x8E, 0x97, 0xB4, 0xC3, 0xF4,
|
||||
];
|
||||
|
||||
const CRYPT_TAB2: [u8; 256] = [
|
||||
0xB7, 0x75, 0x81, 0xD5, 0xDC, 0xCA, 0xDE, 0x66, 0x23, 0xDF, 0x15, 0x26, 0x62, 0xD1, 0x83, 0x77,
|
||||
0xE3, 0x97, 0x76, 0xAF, 0xE9, 0xC3, 0x6B, 0x8E, 0xDA, 0xB0, 0x6E, 0xBF, 0x2B, 0xF1, 0x19, 0xB4,
|
||||
0x95, 0x34, 0x48, 0xE4, 0x37, 0x94, 0x5D, 0x7B, 0x36, 0x5F, 0x65, 0x53, 0x07, 0xE2, 0x89, 0x11,
|
||||
0x98, 0x85, 0xD9, 0x12, 0xC1, 0x9D, 0x84, 0xEC, 0xA4, 0xD4, 0x88, 0xB8, 0xFC, 0x2C, 0x79, 0x28,
|
||||
0xD8, 0xDB, 0xB3, 0x1E, 0xA2, 0xF9, 0xD0, 0x44, 0xD7, 0xD6, 0x60, 0xEF, 0x14, 0xF4, 0xF6, 0x31,
|
||||
0xD2, 0x41, 0x46, 0x67, 0x0A, 0xE1, 0x58, 0x27, 0x43, 0xA3, 0xF8, 0xE0, 0xC8, 0xBA, 0x5A, 0x5C,
|
||||
0x80, 0x6C, 0xC6, 0xF2, 0xE8, 0xAD, 0x7D, 0x04, 0x0D, 0xB9, 0x3C, 0xC2, 0x25, 0xBD, 0x49, 0x63,
|
||||
0x8C, 0x9F, 0x51, 0xCE, 0x20, 0xC5, 0xA1, 0x50, 0x92, 0x2D, 0xDD, 0xBC, 0x8D, 0x4F, 0x9A, 0x71,
|
||||
0x2F, 0x30, 0x1D, 0x73, 0x39, 0x13, 0xFB, 0x1A, 0xCB, 0x24, 0x59, 0xFE, 0x05, 0x96, 0x57, 0x0F,
|
||||
0x1F, 0xCF, 0x54, 0xBE, 0xF5, 0x06, 0x1B, 0xB2, 0x6D, 0xD3, 0x4D, 0x32, 0x56, 0x21, 0x33, 0x0B,
|
||||
0x52, 0xE7, 0xAB, 0xEB, 0xA6, 0x74, 0x00, 0x4C, 0xB1, 0x7F, 0x82, 0x99, 0x87, 0x0E, 0x5E, 0xC0,
|
||||
0x8F, 0xEE, 0x6F, 0x55, 0xF3, 0x7E, 0x08, 0x90, 0xFA, 0xB6, 0x64, 0x70, 0x47, 0x4A, 0x17, 0xA7,
|
||||
0xB5, 0x40, 0x8A, 0x38, 0xE5, 0x68, 0x3E, 0x8B, 0x69, 0xAA, 0x9B, 0x42, 0xA5, 0x10, 0x01, 0x35,
|
||||
0xFD, 0x61, 0x9E, 0xE6, 0x16, 0x9C, 0x86, 0xED, 0xCD, 0x2E, 0xFF, 0xC4, 0x5B, 0xA0, 0xAE, 0xCC,
|
||||
0x4B, 0x3B, 0x03, 0xBB, 0x1C, 0x2A, 0xAC, 0x0C, 0x3F, 0x93, 0xC7, 0x72, 0x7A, 0x09, 0x22, 0x3D,
|
||||
0x45, 0x78, 0xA9, 0xA8, 0xEA, 0xC9, 0x6A, 0xF7, 0x29, 0x91, 0xF0, 0x02, 0x18, 0x3A, 0x4E, 0x7C,
|
||||
];
|
||||
|
||||
const CRYPT_TAB3: [u8; 288] = [
|
||||
0x73, 0x51, 0x95, 0xE1, 0x12, 0xE4, 0xC0, 0x58, 0xEE, 0xF2, 0x08, 0x1B, 0xA9, 0xFA, 0x98, 0x4C,
|
||||
0xA7, 0x33, 0xE2, 0x1B, 0xA7, 0x6D, 0xF5, 0x30, 0x97, 0x1D, 0xF3, 0x02, 0x60, 0x5A, 0x82, 0x0F,
|
||||
0x91, 0xD0, 0x9C, 0x10, 0x39, 0x7A, 0x83, 0x85, 0x3B, 0xB2, 0xB8, 0xAE, 0x0C, 0x09, 0x52, 0xEA,
|
||||
0x1C, 0xE1, 0x8D, 0x66, 0x4F, 0xF3, 0xDA, 0x92, 0x29, 0xB9, 0xD5, 0xC5, 0x77, 0x47, 0x22, 0x53,
|
||||
0x14, 0xF7, 0xAF, 0x22, 0x64, 0xDF, 0xC6, 0x72, 0x12, 0xF3, 0x75, 0xDA, 0xD7, 0xD7, 0xE5, 0x02,
|
||||
0x9E, 0xED, 0xDA, 0xDB, 0x4C, 0x47, 0xCE, 0x91, 0x06, 0x06, 0x6D, 0x55, 0x8B, 0x19, 0xC9, 0xEF,
|
||||
0x8C, 0x80, 0x1A, 0x0E, 0xEE, 0x4B, 0xAB, 0xF2, 0x08, 0x5C, 0xE9, 0x37, 0x26, 0x5E, 0x9A, 0x90,
|
||||
0x00, 0xF3, 0x0D, 0xB2, 0xA6, 0xA3, 0xF7, 0x26, 0x17, 0x48, 0x88, 0xC9, 0x0E, 0x2C, 0xC9, 0x02,
|
||||
0xE7, 0x18, 0x05, 0x4B, 0xF3, 0x39, 0xE1, 0x20, 0x02, 0x0D, 0x40, 0xC7, 0xCA, 0xB9, 0x48, 0x30,
|
||||
0x57, 0x67, 0xCC, 0x06, 0xBF, 0xAC, 0x81, 0x08, 0x24, 0x7A, 0xD4, 0x8B, 0x19, 0x8E, 0xAC, 0xB4,
|
||||
0x5A, 0x0F, 0x73, 0x13, 0xAC, 0x9E, 0xDA, 0xB6, 0xB8, 0x96, 0x5B, 0x60, 0x88, 0xE1, 0x81, 0x3F,
|
||||
0x07, 0x86, 0x37, 0x2D, 0x79, 0x14, 0x52, 0xEA, 0x73, 0xDF, 0x3D, 0x09, 0xC8, 0x25, 0x48, 0xD8,
|
||||
0x75, 0x60, 0x9A, 0x08, 0x27, 0x4A, 0x2C, 0xB9, 0xA8, 0x8B, 0x8A, 0x73, 0x62, 0x37, 0x16, 0x02,
|
||||
0xBD, 0xC1, 0x0E, 0x56, 0x54, 0x3E, 0x14, 0x5F, 0x8C, 0x8F, 0x6E, 0x75, 0x1C, 0x07, 0x39, 0x7B,
|
||||
0x4B, 0xDB, 0xD3, 0x4B, 0x1E, 0xC8, 0x7E, 0xFE, 0x3E, 0x72, 0x16, 0x83, 0x7D, 0xEE, 0xF5, 0xCA,
|
||||
0xC5, 0x18, 0xF9, 0xD8, 0x68, 0xAB, 0x38, 0x85, 0xA8, 0xF0, 0xA1, 0x73, 0x9F, 0x5D, 0x19, 0x0B,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, 0x72, 0x39, 0x25, 0x67, 0x26, 0x6D, 0x71,
|
||||
0x36, 0x77, 0x3C, 0x20, 0x62, 0x23, 0x68, 0x74, 0xC3, 0x82, 0xC9, 0x15, 0x57, 0x16, 0x5D, 0x81,
|
||||
];
|
||||
|
||||
const VARIANTS: [u8; 32] = [
|
||||
0xB7, 0x74, 0x85, 0xD0, 0xCC, 0xDB, 0xCA, 0x73, 0x03, 0xFE, 0x31, 0x03, 0x52, 0xE0, 0xB7, 0x42,
|
||||
0x63, 0x16, 0xF2, 0x2A, 0x79, 0x52, 0xFF, 0x1B, 0x7A, 0x11, 0xCA, 0x1A, 0x9B, 0x40, 0xAD, 0x01,
|
||||
];
|
||||
|
||||
const SECRET: [u8; 5] = [0x55, 0xD6, 0xC4, 0xC5, 0x28];
|
||||
|
||||
const PERM_CHALLENGE: [[usize; 10]; 3] = [
|
||||
[1, 3, 0, 7, 5, 2, 9, 6, 4, 8],
|
||||
[6, 1, 9, 3, 8, 5, 7, 4, 0, 2],
|
||||
[4, 0, 3, 5, 7, 2, 8, 6, 1, 9],
|
||||
];
|
||||
|
||||
const PERM_VARIANT: [[u8; 32]; 2] = [
|
||||
[
|
||||
0x0A, 0x08, 0x0E, 0x0C, 0x0B, 0x09, 0x0F, 0x0D, 0x1A, 0x18, 0x1E, 0x1C, 0x1B, 0x19, 0x1F,
|
||||
0x1D, 0x02, 0x00, 0x06, 0x04, 0x03, 0x01, 0x07, 0x05, 0x12, 0x10, 0x16, 0x14, 0x13, 0x11,
|
||||
0x17, 0x15,
|
||||
],
|
||||
[
|
||||
0x12, 0x1A, 0x16, 0x1E, 0x02, 0x0A, 0x06, 0x0E, 0x10, 0x18, 0x14, 0x1C, 0x00, 0x08, 0x04,
|
||||
0x0C, 0x13, 0x1B, 0x17, 0x1F, 0x03, 0x0B, 0x07, 0x0F, 0x11, 0x19, 0x15, 0x1D, 0x01, 0x09,
|
||||
0x05, 0x0D,
|
||||
],
|
||||
];
|
||||
|
||||
// ── SCSI constants ────────────────────────────────────────────────────────
|
||||
|
||||
const SCSI_READ_DVD_STRUCTURE: u8 = 0xAD;
|
||||
|
||||
// ── Public API ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Perform CSS bus authentication only.
|
||||
pub fn authenticate(drive: &mut Drive) -> Result<()> {
|
||||
let (_, _) = bus_auth(drive)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full CSS key extraction: bus auth → disc key → title key.
|
||||
pub fn authenticate_and_read_title_key(drive: &mut Drive, lba: u32) -> Result<[u8; 5]> {
|
||||
// Session 1: bus auth → disc key (AGID consumed by READ_DVD_STRUCTURE)
|
||||
let (agid, bus_key) = bus_auth(drive)?;
|
||||
let disc_key = read_disc_key(drive, agid, &bus_key)?;
|
||||
|
||||
// Session 2: fresh bus auth → title key (needs separate AGID)
|
||||
let (agid2, bus_key2) = bus_auth(drive)?;
|
||||
let encrypted_title = read_raw_title_key(drive, agid2, lba)?;
|
||||
|
||||
// Decrypt title key: XOR with bus key, then decrypt with disc key
|
||||
let mut title_key = [0u8; 5];
|
||||
for i in 0..5 {
|
||||
title_key[i] = encrypted_title[i] ^ bus_key2[i];
|
||||
}
|
||||
|
||||
if title_key == [0u8; 5] {
|
||||
return Ok(title_key);
|
||||
}
|
||||
|
||||
let title_key = super::lfsr::decrypt_key(0xFF, &disc_key, &title_key);
|
||||
Ok(title_key)
|
||||
}
|
||||
|
||||
// ── Step 1: Bus Authentication ────────────────────────────────────────────
|
||||
|
||||
fn bus_auth(drive: &mut Drive) -> Result<(u8, [u8; 5])> {
|
||||
let scsi = drive.scsi_mut();
|
||||
|
||||
// Invalidate all AGIDs via REPORT KEY format 0x3F
|
||||
for agid in 0..4u8 {
|
||||
let mut cdb = [0u8; 12];
|
||||
cdb[0] = crate::scsi::SCSI_REPORT_KEY;
|
||||
// alloc_len = 0 (no data transfer)
|
||||
cdb[10] = (agid << 6) | 0x3F;
|
||||
let mut buf = [0u8; 8];
|
||||
let _ = scsi.execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
);
|
||||
}
|
||||
|
||||
// Allocate AGID
|
||||
let mut buf = [0u8; 8];
|
||||
scsi.execute(
|
||||
&report_key_cdb(0, 0x00, 8),
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
)
|
||||
.map_err(|_| Error::CssAuthFailed)?;
|
||||
let agid = (buf[7] >> 6) & 0x03;
|
||||
|
||||
// Host sends challenge
|
||||
let host_challenge: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
let mut hc_buf = [0u8; 16];
|
||||
hc_buf[0] = 0x00;
|
||||
hc_buf[1] = 0x0E;
|
||||
for i in 0..10 {
|
||||
hc_buf[4 + i] = host_challenge[9 - i];
|
||||
}
|
||||
scsi.execute(
|
||||
&send_key_cdb(agid, 0x01, 16),
|
||||
crate::scsi::DataDirection::ToDevice,
|
||||
&mut hc_buf,
|
||||
5_000,
|
||||
)
|
||||
.map_err(|_| Error::CssAuthFailed)?;
|
||||
|
||||
// Get Key1 from drive
|
||||
let mut dk_buf = [0u8; 12];
|
||||
scsi.execute(
|
||||
&report_key_cdb(agid, 0x02, 12),
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut dk_buf,
|
||||
5_000,
|
||||
)
|
||||
.map_err(|_| Error::CssAuthFailed)?;
|
||||
let mut key1 = [0u8; 5];
|
||||
for i in 0..5 {
|
||||
key1[i] = dk_buf[4 + (4 - i)];
|
||||
}
|
||||
|
||||
// Brute-force variant (0-31)
|
||||
let mut variant: Option<u8> = None;
|
||||
for v in 0..32u8 {
|
||||
if crypt_key(0, v, &host_challenge) == key1 {
|
||||
variant = Some(v);
|
||||
break;
|
||||
}
|
||||
}
|
||||
let variant = variant.ok_or(Error::CssAuthFailed)?;
|
||||
|
||||
// Get drive challenge
|
||||
let mut dc_buf = [0u8; 16];
|
||||
scsi.execute(
|
||||
&report_key_cdb(agid, 0x01, 16),
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut dc_buf,
|
||||
5_000,
|
||||
)
|
||||
.map_err(|_| Error::CssAuthFailed)?;
|
||||
let mut drive_challenge = [0u8; 10];
|
||||
for i in 0..10 {
|
||||
drive_challenge[i] = dc_buf[4 + (9 - i)];
|
||||
}
|
||||
|
||||
// Compute Key2 and send it
|
||||
let key2 = crypt_key(1, variant, &drive_challenge);
|
||||
let mut hk_buf = [0u8; 12];
|
||||
hk_buf[0] = 0x00;
|
||||
hk_buf[1] = 0x0A;
|
||||
for i in 0..5 {
|
||||
hk_buf[4 + i] = key2[4 - i];
|
||||
}
|
||||
scsi.execute(
|
||||
&send_key_cdb(agid, 0x03, 12),
|
||||
crate::scsi::DataDirection::ToDevice,
|
||||
&mut hk_buf,
|
||||
5_000,
|
||||
)
|
||||
.map_err(|_| Error::CssAuthFailed)?;
|
||||
|
||||
// Bus key = CryptKey(2, variant, key1 || key2)
|
||||
let mut combined = [0u8; 10];
|
||||
combined[..5].copy_from_slice(&key1);
|
||||
combined[5..].copy_from_slice(&key2);
|
||||
let bus_key = crypt_key(2, variant, &combined);
|
||||
|
||||
Ok((agid, bus_key))
|
||||
}
|
||||
|
||||
// ── Step 2: Disc Key ──────────────────────────────────────────────────────
|
||||
|
||||
fn read_disc_key(drive: &mut Drive, agid: u8, bus_key: &[u8; 5]) -> Result<[u8; 5]> {
|
||||
let scsi = drive.scsi_mut();
|
||||
|
||||
// READ DVD STRUCTURE, format 0x02 (disc key), 2048+4 bytes
|
||||
let alloc_len: u16 = 2048 + 4;
|
||||
let mut cdb = [0u8; 12];
|
||||
cdb[0] = SCSI_READ_DVD_STRUCTURE;
|
||||
// bytes 2-5: address = 0
|
||||
cdb[6] = 0; // layer
|
||||
cdb[7] = 0x02; // format = disc key
|
||||
cdb[8] = (alloc_len >> 8) as u8;
|
||||
cdb[9] = alloc_len as u8;
|
||||
cdb[10] = agid << 6;
|
||||
|
||||
let mut buf = vec![0u8; alloc_len as usize];
|
||||
let dvd_result = scsi.execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
);
|
||||
dvd_result.map_err(|_| Error::CssAuthFailed)?;
|
||||
|
||||
// Disc key block starts at offset 4 (skip 4-byte header)
|
||||
let disc_key_block = &mut buf[4..4 + 2048];
|
||||
|
||||
// XOR with reversed bus key (per libdvdcss)
|
||||
for (i, byte) in disc_key_block.iter_mut().enumerate() {
|
||||
*byte ^= bus_key[4 - (i % 5)];
|
||||
}
|
||||
|
||||
// Try each player key against each of 408 disc key entries.
|
||||
// Each entry in the block is the disc key encrypted with a specific player key.
|
||||
// We try all known player keys and verify by checking that two different
|
||||
// entries produce the same disc key.
|
||||
let mut candidates: Vec<([u8; 5], usize, usize)> = Vec::new(); // (disc_key, pk_idx, pos)
|
||||
|
||||
for (pk_idx, player_key) in PLAYER_KEYS.iter().enumerate() {
|
||||
for pos in 0..408 {
|
||||
let offset = pos * 5;
|
||||
if offset + 5 > disc_key_block.len() {
|
||||
break;
|
||||
}
|
||||
let mut enc = [0u8; 5];
|
||||
enc.copy_from_slice(&disc_key_block[offset..offset + 5]);
|
||||
let candidate = super::lfsr::decrypt_key(0x00, player_key, &enc);
|
||||
|
||||
// Check if any previous candidate matches (same disc key from different entry/pk)
|
||||
for (prev, _, _) in &candidates {
|
||||
if *prev == candidate {
|
||||
return Ok(candidate);
|
||||
}
|
||||
}
|
||||
candidates.push((candidate, pk_idx, pos));
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::CssAuthFailed)
|
||||
}
|
||||
|
||||
// ── Step 3: Title Key ─────────────────────────────────────────────────────
|
||||
|
||||
/// Read the raw (bus-encrypted) title key bytes from the drive.
|
||||
fn read_raw_title_key(drive: &mut Drive, agid: u8, lba: u32) -> Result<[u8; 5]> {
|
||||
let scsi = drive.scsi_mut();
|
||||
let mut cdb = [0u8; 12];
|
||||
cdb[0] = crate::scsi::SCSI_REPORT_KEY;
|
||||
cdb[2] = (lba >> 24) as u8;
|
||||
cdb[3] = (lba >> 16) as u8;
|
||||
cdb[4] = (lba >> 8) as u8;
|
||||
cdb[5] = lba as u8;
|
||||
cdb[8] = 0x00;
|
||||
cdb[9] = 0x0C;
|
||||
cdb[10] = (agid << 6) | 0x04;
|
||||
|
||||
let mut buf = [0u8; 12];
|
||||
let result = scsi.execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
);
|
||||
result.map_err(|_| Error::CssAuthFailed)?;
|
||||
|
||||
let mut key = [0u8; 5];
|
||||
for i in 0..5 {
|
||||
key[i] = buf[5 + (4 - i)];
|
||||
}
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn read_title_key(
|
||||
drive: &mut Drive,
|
||||
agid: u8,
|
||||
lba: u32,
|
||||
bus_key: &[u8; 5],
|
||||
disc_key: &[u8; 5],
|
||||
) -> Result<[u8; 5]> {
|
||||
let scsi = drive.scsi_mut();
|
||||
|
||||
let mut cdb = [0u8; 12];
|
||||
cdb[0] = crate::scsi::SCSI_REPORT_KEY;
|
||||
cdb[2] = (lba >> 24) as u8;
|
||||
cdb[3] = (lba >> 16) as u8;
|
||||
cdb[4] = (lba >> 8) as u8;
|
||||
cdb[5] = lba as u8;
|
||||
cdb[8] = 0x00;
|
||||
cdb[9] = 0x0C;
|
||||
cdb[10] = (agid << 6) | 0x04;
|
||||
|
||||
let mut buf = [0u8; 12];
|
||||
let tk_result = scsi.execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
);
|
||||
tk_result.map_err(|_| Error::CssAuthFailed)?;
|
||||
|
||||
// Title key at bytes 5..10, byte-reversed
|
||||
let mut title_key = [0u8; 5];
|
||||
for i in 0..5 {
|
||||
title_key[i] = buf[5 + (4 - i)];
|
||||
}
|
||||
|
||||
// XOR with reversed bus key (same pattern as disc key block)
|
||||
for i in 0..5 {
|
||||
title_key[i] ^= bus_key[4 - i];
|
||||
}
|
||||
|
||||
// Check for null key (title not encrypted)
|
||||
if title_key == [0u8; 5] {
|
||||
return Ok(title_key);
|
||||
}
|
||||
|
||||
// Decrypt with disc key (invert=0xFF for title keys)
|
||||
let title_key = super::lfsr::decrypt_key(0xFF, disc_key, &title_key);
|
||||
|
||||
Ok(title_key)
|
||||
}
|
||||
|
||||
// ── CSSCryptKey ───────────────────────────────────────────────────────────
|
||||
|
||||
/// Exposed for testing only.
|
||||
pub fn test_crypt_key(key_type: usize, variant: u8, challenge: &[u8; 10]) -> [u8; 5] {
|
||||
crypt_key(key_type, variant, challenge)
|
||||
}
|
||||
|
||||
fn crypt_key(key_type: usize, variant: u8, challenge: &[u8; 10]) -> [u8; 5] {
|
||||
let perm = &PERM_CHALLENGE[key_type];
|
||||
let mut scratch = [0u8; 10];
|
||||
for i in 0..10 {
|
||||
scratch[i] = challenge[perm[i]];
|
||||
}
|
||||
|
||||
let css_variant = match key_type {
|
||||
0 => variant as usize,
|
||||
1 => PERM_VARIANT[0][variant as usize] as usize,
|
||||
_ => PERM_VARIANT[1][variant as usize] as usize,
|
||||
};
|
||||
|
||||
let cse = VARIANTS[css_variant] ^ CRYPT_TAB2[css_variant];
|
||||
|
||||
let mut tmp1 = [0u8; 5];
|
||||
for i in 0..5 {
|
||||
tmp1[i] = scratch[5 + i] ^ SECRET[i] ^ CRYPT_TAB2[i];
|
||||
}
|
||||
|
||||
let mut lfsr0: u32 = ((tmp1[0] as u32) << 17)
|
||||
| ((tmp1[1] as u32) << 9)
|
||||
| (((tmp1[2] as u32) & !7) << 1)
|
||||
| 8
|
||||
| (tmp1[2] as u32 & 7);
|
||||
|
||||
let mut lfsr1: u32 = ((tmp1[3] as u32) << 9) | 0x100 | (tmp1[4] as u32);
|
||||
|
||||
let mut bits = [0u8; 30];
|
||||
let mut carry: u32 = 0;
|
||||
for idx in (0..30).rev() {
|
||||
let mut val: u8 = 0;
|
||||
for bit in 0..8u8 {
|
||||
let lfsr0_out = ((lfsr0 >> 24) ^ (lfsr0 >> 21) ^ (lfsr0 >> 20) ^ (lfsr0 >> 12)) & 1;
|
||||
lfsr0 = ((lfsr0 << 1) | lfsr0_out) & 0x1FFFFFF;
|
||||
|
||||
let lfsr1_out = ((lfsr1 >> 16) ^ (lfsr1 >> 2)) & 1;
|
||||
lfsr1 = ((lfsr1 << 1) | lfsr1_out) & 0x1FFFF;
|
||||
|
||||
let combined = ((!lfsr1_out) & 1) + carry + ((!lfsr0_out) & 1);
|
||||
carry = (combined >> 1) & 1;
|
||||
val |= ((combined & 1) as u8) << bit;
|
||||
}
|
||||
bits[idx] = val;
|
||||
}
|
||||
|
||||
let mut tmp1 = [scratch[0], scratch[1], scratch[2], scratch[3], scratch[4]];
|
||||
let mut tmp2 = [0u8; 5];
|
||||
|
||||
// Round 1: bits[25..29] ^ scratch -> tmp1 (term from original scratch)
|
||||
{
|
||||
let mut term: u8 = 0;
|
||||
for i in (0..5usize).rev() {
|
||||
let idx = (bits[25 + i] ^ tmp1[i]) as usize;
|
||||
let idx2 = (CRYPT_TAB1[idx] ^ (!CRYPT_TAB2[idx]) ^ cse) as usize;
|
||||
tmp1[i] = CRYPT_TAB2[idx2] ^ CRYPT_TAB3[idx2] ^ term;
|
||||
term = scratch[i]; // original challenge, NOT modified tmp1
|
||||
}
|
||||
tmp1[4] ^= tmp1[0];
|
||||
}
|
||||
|
||||
// Round 2
|
||||
{
|
||||
let mut term: u8 = 0;
|
||||
for i in (0..5usize).rev() {
|
||||
let idx = (bits[20 + i] ^ tmp1[i]) as usize;
|
||||
let idx2 = (CRYPT_TAB1[idx] ^ (!CRYPT_TAB2[idx]) ^ cse) as usize;
|
||||
tmp2[i] = CRYPT_TAB2[idx2] ^ CRYPT_TAB3[idx2] ^ term;
|
||||
term = tmp1[i];
|
||||
}
|
||||
tmp2[4] ^= tmp2[0];
|
||||
}
|
||||
|
||||
// Round 3 (uses CRYPT_TAB0)
|
||||
{
|
||||
let mut term: u8 = 0;
|
||||
for i in (0..5usize).rev() {
|
||||
let idx = (bits[15 + i] ^ tmp2[i]) as usize;
|
||||
let idx2 = (CRYPT_TAB1[idx] ^ (!CRYPT_TAB2[idx]) ^ cse) as usize;
|
||||
let idx3 = (CRYPT_TAB2[idx2] ^ CRYPT_TAB3[idx2] ^ term) as usize;
|
||||
tmp1[i] = CRYPT_TAB0[idx3] ^ CRYPT_TAB2[idx3];
|
||||
term = tmp2[i];
|
||||
}
|
||||
tmp1[4] ^= tmp1[0];
|
||||
}
|
||||
|
||||
// Round 4 (uses CRYPT_TAB0)
|
||||
{
|
||||
let mut term: u8 = 0;
|
||||
for i in (0..5usize).rev() {
|
||||
let idx = (bits[10 + i] ^ tmp1[i]) as usize;
|
||||
let idx2 = (CRYPT_TAB1[idx] ^ (!CRYPT_TAB2[idx]) ^ cse) as usize;
|
||||
let idx3 = (CRYPT_TAB2[idx2] ^ CRYPT_TAB3[idx2] ^ term) as usize;
|
||||
tmp2[i] = CRYPT_TAB0[idx3] ^ CRYPT_TAB2[idx3];
|
||||
term = tmp1[i];
|
||||
}
|
||||
tmp2[4] ^= tmp2[0];
|
||||
}
|
||||
|
||||
// Round 5
|
||||
{
|
||||
let mut term: u8 = 0;
|
||||
for i in (0..5usize).rev() {
|
||||
let idx = (bits[5 + i] ^ tmp2[i]) as usize;
|
||||
let idx2 = (CRYPT_TAB1[idx] ^ (!CRYPT_TAB2[idx]) ^ cse) as usize;
|
||||
tmp1[i] = CRYPT_TAB2[idx2] ^ CRYPT_TAB3[idx2] ^ term;
|
||||
term = tmp2[i];
|
||||
}
|
||||
tmp1[4] ^= tmp1[0];
|
||||
}
|
||||
|
||||
// Round 6
|
||||
let mut key = [0u8; 5];
|
||||
{
|
||||
let mut term: u8 = 0;
|
||||
for i in (0..5usize).rev() {
|
||||
let idx = (bits[i] ^ tmp1[i]) as usize;
|
||||
let idx2 = (CRYPT_TAB1[idx] ^ (!CRYPT_TAB2[idx]) ^ cse) as usize;
|
||||
key[i] = CRYPT_TAB2[idx2] ^ CRYPT_TAB3[idx2] ^ term;
|
||||
term = tmp1[i];
|
||||
}
|
||||
}
|
||||
|
||||
key
|
||||
}
|
||||
|
||||
// ── SCSI CDB builders ────────────────────────────────────────────────────
|
||||
|
||||
fn report_key_cdb(agid: u8, format: u8, alloc_len: u16) -> [u8; 12] {
|
||||
let mut cdb = [0u8; 12];
|
||||
cdb[0] = crate::scsi::SCSI_REPORT_KEY;
|
||||
cdb[8] = (alloc_len >> 8) as u8;
|
||||
cdb[9] = alloc_len as u8;
|
||||
cdb[10] = (agid << 6) | (format & 0x3F);
|
||||
cdb
|
||||
}
|
||||
|
||||
fn send_key_cdb(agid: u8, format: u8, param_len: u16) -> [u8; 12] {
|
||||
let mut cdb = [0u8; 12];
|
||||
cdb[0] = crate::scsi::SCSI_SEND_KEY;
|
||||
cdb[8] = (param_len >> 8) as u8;
|
||||
cdb[9] = param_len as u8;
|
||||
cdb[10] = (agid << 6) | (format & 0x3F);
|
||||
cdb
|
||||
}
|
||||
|
||||
// ── Tests ─────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn crypt_key_is_deterministic() {
|
||||
let challenge: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
for v in 0..32u8 {
|
||||
let r1 = crypt_key(0, v, &challenge);
|
||||
let r2 = crypt_key(0, v, &challenge);
|
||||
assert_eq!(r1, r2);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crypt_key_varies_by_variant() {
|
||||
let challenge: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
assert_ne!(crypt_key(0, 0, &challenge), crypt_key(0, 1, &challenge));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crypt_key_varies_by_type() {
|
||||
let challenge: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
assert_ne!(crypt_key(0, 5, &challenge), crypt_key(1, 5, &challenge));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crypt_key_nonzero() {
|
||||
let challenge: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
for v in 0..32u8 {
|
||||
assert_ne!(crypt_key(0, v, &challenge), [0u8; 5]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn player_keys_count() {
|
||||
assert_eq!(PLAYER_KEYS.len(), 31);
|
||||
}
|
||||
}
|
||||
+35
-24
@@ -197,37 +197,48 @@ pub fn crack_title_key(sector: &[u8]) -> Option<[u8; 5]> {
|
||||
//
|
||||
// We try multiple stream IDs and use zeros for unknown bytes (most common).
|
||||
|
||||
let stream_ids: &[u8] = &[
|
||||
0xE0, // video
|
||||
0xBD, // private stream 1 (AC3/DTS)
|
||||
0xC0, // MPEG audio
|
||||
0xBE, // padding
|
||||
];
|
||||
// Try many PES header patterns at byte 0x80.
|
||||
// Structure: 00 00 01 [stream_id] [len_hi] [len_lo] [flags1] [flags2] [hdr_len] [data]
|
||||
let mut patterns: Vec<[u8; 10]> = Vec::with_capacity(128);
|
||||
|
||||
for &sid in stream_ids {
|
||||
// Build candidate plaintext (10 bytes)
|
||||
// Bytes 0-2: PES start code 00 00 01
|
||||
// Byte 3: stream ID
|
||||
// Bytes 4-9: we try with zeros first (common for padding streams)
|
||||
// and with typical PES header bytes
|
||||
let patterns: &[[u8; 10]] = &[
|
||||
[0x00, 0x00, 0x01, sid, 0x00, 0x00, 0x80, 0x80, 0x05, 0x21],
|
||||
[0x00, 0x00, 0x01, sid, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00],
|
||||
[0x00, 0x00, 0x01, sid, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00],
|
||||
];
|
||||
// Padding stream (0xBE): payload is 0xFF bytes, various lengths
|
||||
for len_hi in 0u8..8 {
|
||||
for len_lo_top in [0x00u8, 0x80, 0xFF] {
|
||||
patterns.push([
|
||||
0x00, 0x00, 0x01, 0xBE, len_hi, len_lo_top, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
for pattern in patterns {
|
||||
if let Some(key) = recover_title_key(sector, pattern) {
|
||||
// Verify: the key should produce valid MPEG-2 when used to descramble
|
||||
let mut test = sector.to_vec();
|
||||
super::lfsr::descramble_sector(&key, &mut test);
|
||||
if test[0x80] == 0x00 && test[0x81] == 0x00 && test[0x82] == 0x01 {
|
||||
return Some(key);
|
||||
// Video (0xE0) and audio (0xBD, 0xC0) with typical PES headers
|
||||
for &sid in &[0xE0u8, 0xBD, 0xC0] {
|
||||
for &flags1 in &[0x80u8, 0x81, 0x84, 0x85, 0x8C, 0x8D] {
|
||||
for &flags2 in &[0x00u8, 0x05, 0x80, 0xC0] {
|
||||
let hdr_len = if flags2 & 0x80 != 0 { 0x05u8 } else { 0x00 };
|
||||
let pts0 = if flags2 & 0x80 != 0 { 0x21u8 } else { 0x00 };
|
||||
// Try with several PES lengths
|
||||
for &len_hi in &[0x00u8, 0x07] {
|
||||
patterns.push([
|
||||
0x00, 0x00, 0x01, sid, len_hi, 0x00, flags1, flags2, hdr_len, pts0,
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Navigation pack system header (0xBB)
|
||||
patterns.push([0x00, 0x00, 0x01, 0xBB, 0x00, 0x12, 0x80, 0xC4, 0xE1, 0x04]);
|
||||
|
||||
for pattern in &patterns {
|
||||
if let Some(key) = recover_title_key(sector, pattern) {
|
||||
let mut test = sector.to_vec();
|
||||
super::lfsr::descramble_sector(&key, &mut test);
|
||||
if test[0x80] == 0x00 && test[0x81] == 0x00 && test[0x82] == 0x01 {
|
||||
return Some(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
|
||||
+20
-54
@@ -10,7 +10,7 @@
|
||||
//! Algorithm: Frank A. Stevenson's divide-and-conquer attack (1999).
|
||||
//! Tables: CSS specification constants.
|
||||
|
||||
use super::tables::{TAB1, TAB2, TAB3, TAB4};
|
||||
use super::tables::{TAB1, TAB2, TAB3, TAB4, TAB5};
|
||||
|
||||
/// Descramble a CSS-encrypted DVD sector in place.
|
||||
///
|
||||
@@ -57,19 +57,16 @@ pub fn descramble_sector(title_key: &[u8; 5], sector: &mut [u8]) {
|
||||
let mut combined: u32 = 0;
|
||||
|
||||
// Generate 1920 keystream bytes (for sector bytes 128..2048)
|
||||
// Per libdvdcss css_unscramble: TAB1 permutation on ciphertext, no invert on LFSR0
|
||||
for byte in sector.iter_mut().take(2048).skip(128) {
|
||||
// Clock LFSR1
|
||||
let o_lfsr1 = TAB2[lfsr1_hi as usize] ^ TAB3[lfsr1_lo as usize];
|
||||
lfsr1_hi = lfsr1_lo >> 1;
|
||||
lfsr1_lo = ((lfsr1_lo & 1) << 8) ^ o_lfsr1 as u32;
|
||||
let o_lfsr1_perm = TAB4[o_lfsr1 as usize];
|
||||
|
||||
// Clock LFSR0
|
||||
let o_lfsr0 = (((((((lfsr0 >> 8) ^ lfsr0) >> 1) ^ lfsr0) >> 3) ^ lfsr0) >> 7) as u8;
|
||||
lfsr0 = (lfsr0 >> 8) | ((o_lfsr0 as u32) << 24);
|
||||
|
||||
// Combine with addition and carry
|
||||
combined += (o_lfsr0 ^ 0xFF) as u32 + o_lfsr1_perm as u32;
|
||||
combined += TAB5[o_lfsr1 as usize] as u32 + TAB4[o_lfsr0 as usize] as u32;
|
||||
*byte ^= (combined & 0xFF) as u8;
|
||||
combined >>= 8;
|
||||
}
|
||||
@@ -106,12 +103,12 @@ pub(crate) fn decrypt_key(invert: u8, p_key: &[u8; 5], p_crypted: &[u8]) -> [u8;
|
||||
let o_lfsr1 = TAB2[lfsr1_hi as usize] ^ TAB3[lfsr1_lo as usize];
|
||||
lfsr1_hi = lfsr1_lo >> 1;
|
||||
lfsr1_lo = ((lfsr1_lo & 1) << 8) ^ o_lfsr1 as u32;
|
||||
let o_lfsr1_perm = TAB4[o_lfsr1 as usize];
|
||||
|
||||
let o_lfsr0 = (((((((lfsr0 >> 8) ^ lfsr0) >> 1) ^ lfsr0) >> 3) ^ lfsr0) >> 7) as u8;
|
||||
lfsr0 = (lfsr0 >> 8) | ((o_lfsr0 as u32) << 24);
|
||||
|
||||
combined += (o_lfsr0 ^ invert) as u32 + o_lfsr1_perm as u32;
|
||||
// TAB5 for LFSR1 output, TAB4 for LFSR0^invert (per libdvdcss css_DecryptKey)
|
||||
combined += TAB5[o_lfsr1 as usize] as u32 + TAB4[(o_lfsr0 ^ invert) as usize] as u32;
|
||||
*byte = (combined & 0xFF) as u8;
|
||||
combined >>= 8;
|
||||
}
|
||||
@@ -152,7 +149,7 @@ mod tests {
|
||||
let key = [0x01, 0x02, 0x03, 0x04, 0x05];
|
||||
let mut sector = vec![0xAA; 2048];
|
||||
sector[0x14] = 0x30; // scramble flag set
|
||||
// Set a sector seed
|
||||
// Set a sector seed
|
||||
sector[0x54..0x59].copy_from_slice(&[0x11, 0x22, 0x33, 0x44, 0x55]);
|
||||
let original = sector.clone();
|
||||
descramble_sector(&key, &mut sector);
|
||||
@@ -237,58 +234,27 @@ mod tests {
|
||||
/// twice with the same key and restored scramble flag should roundtrip,
|
||||
/// since XOR is its own inverse.
|
||||
#[test]
|
||||
fn css_descramble_produces_valid_mpeg2() {
|
||||
fn css_descramble_modifies_encrypted_region() {
|
||||
let title_key = [0x42, 0x13, 0x37, 0xBE, 0xEF];
|
||||
|
||||
// Build a sector with MPEG-2 pack header and PES header
|
||||
let mut sector = vec![0x00u8; 2048];
|
||||
// Pack header at byte 0
|
||||
sector[0] = 0x00;
|
||||
sector[1] = 0x00;
|
||||
sector[2] = 0x01;
|
||||
sector[3] = 0xBA;
|
||||
// Scramble flag at byte 0x14
|
||||
sector[0x14] = 0x30;
|
||||
// Sector seed at bytes 0x54-0x58
|
||||
let mut sector = vec![0xAAu8; 2048];
|
||||
sector[0x14] = 0x30; // scramble flag
|
||||
sector[0x54..0x59].copy_from_slice(&[0xDE, 0xAD, 0xBE, 0xEF, 0x42]);
|
||||
// PES header at byte 128
|
||||
sector[0x80] = 0x00;
|
||||
sector[0x81] = 0x00;
|
||||
sector[0x82] = 0x01;
|
||||
sector[0x83] = 0xE0;
|
||||
// Fill some content in the encrypted region
|
||||
for (i, byte) in sector.iter_mut().enumerate().take(2048).skip(0x84) {
|
||||
*byte = (i & 0xFF) as u8;
|
||||
}
|
||||
|
||||
let original = sector.clone();
|
||||
|
||||
// First descramble: "encrypts" by XORing keystream
|
||||
descramble_sector(&title_key, &mut sector);
|
||||
// Flag should be cleared
|
||||
assert_eq!(
|
||||
sector[0x14] & 0x30,
|
||||
0x00,
|
||||
"scramble flag not cleared after first descramble"
|
||||
);
|
||||
// Encrypted region should differ
|
||||
assert_ne!(
|
||||
§or[0x80..0x84],
|
||||
&original[0x80..0x84],
|
||||
"encrypted region unchanged after descramble"
|
||||
);
|
||||
|
||||
// Restore the scramble flag and sector seed for second pass
|
||||
sector[0x14] = 0x30;
|
||||
|
||||
// Second descramble: XOR again = roundtrip
|
||||
descramble_sector(&title_key, &mut sector);
|
||||
// Now the encrypted region should match original
|
||||
assert_eq!(
|
||||
§or[0x80..2048],
|
||||
&original[0x80..2048],
|
||||
"double descramble did not roundtrip"
|
||||
);
|
||||
// Flag cleared
|
||||
assert_eq!(sector[0x14] & 0x30, 0x00);
|
||||
// Header (0..128) unchanged except flag byte
|
||||
for i in 0..128 {
|
||||
if i == 0x14 {
|
||||
continue;
|
||||
}
|
||||
assert_eq!(sector[i], original[i], "header byte {} changed", i);
|
||||
}
|
||||
// Encrypted region modified
|
||||
assert_ne!(§or[128..256], &original[128..256]);
|
||||
}
|
||||
|
||||
/// Test 4: css_tab1_relationship
|
||||
|
||||
+30
-20
@@ -10,12 +10,13 @@
|
||||
//! css::descramble_sector(&key, &mut sector);
|
||||
//! ```
|
||||
|
||||
pub mod auth;
|
||||
pub mod crack;
|
||||
pub mod lfsr;
|
||||
pub(crate) mod tables;
|
||||
|
||||
use crate::disc::Extent;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
|
||||
/// CSS decryption state for a DVD title.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -27,29 +28,38 @@ pub struct CssState {
|
||||
/// Crack the CSS title key by reading encrypted sectors and applying
|
||||
/// a known-plaintext attack on MPEG-2 headers.
|
||||
///
|
||||
/// Reads a few sectors from the first extent, finds one with the
|
||||
/// scramble flag set, and cracks the key.
|
||||
pub fn crack_key(reader: &mut dyn SectorReader, extents: &[Extent]) -> Option<CssState> {
|
||||
if extents.is_empty() {
|
||||
return None;
|
||||
}
|
||||
/// Crack the CSS title key by scanning scrambled sectors across extents.
|
||||
///
|
||||
/// The Stevenson attack needs a sector where a PES header starts at byte
|
||||
/// 0x80 (start of the encrypted region). This only happens when a new PES
|
||||
/// packet begins at exactly sector offset 128. We scan up to 50000
|
||||
/// scrambled sectors sequentially across all extents.
|
||||
pub fn crack_key(reader: &mut dyn SectorSource, extents: &[Extent]) -> Option<CssState> {
|
||||
let mut tried = 0u32;
|
||||
let max_tries = 50_000;
|
||||
|
||||
let ext = &extents[0];
|
||||
let mut sectors = Vec::new();
|
||||
|
||||
// Read first 10 sectors from the main extent
|
||||
let count = ext.sector_count.min(10);
|
||||
for i in 0..count {
|
||||
let mut buf = vec![0u8; 2048];
|
||||
if reader.read_sectors(ext.start_lba + i, 1, &mut buf).is_ok() {
|
||||
sectors.push(buf);
|
||||
for ext in extents {
|
||||
let mut i = 0;
|
||||
while i < ext.sector_count && tried < max_tries {
|
||||
let mut buf = vec![0u8; 2048];
|
||||
if reader
|
||||
.read_sectors(ext.start_lba + i, 1, &mut buf, true)
|
||||
.is_ok()
|
||||
&& is_scrambled(&buf)
|
||||
{
|
||||
if let Some(key) = crack::crack_title_key(&buf) {
|
||||
return Some(CssState { title_key: key });
|
||||
}
|
||||
tried += 1;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
if tried >= max_tries {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Try cracking from the collected sectors
|
||||
let key = crack::crack_from_sectors(§ors)?;
|
||||
|
||||
Some(CssState { title_key: key })
|
||||
None
|
||||
}
|
||||
|
||||
/// Descramble a single CSS-encrypted sector in place.
|
||||
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
//! Decrypt-on-read layer.
|
||||
//!
|
||||
//! Decrypts sectors in-place using resolved keys from disc scanning.
|
||||
//! Handles AACS 1.0, AACS 2.0, and CSS transparently.
|
||||
//! The caller never sees encrypted data unless explicitly bypassed.
|
||||
|
||||
use crate::aacs;
|
||||
use crate::css;
|
||||
|
||||
/// Resolved decryption state from disc scanning.
|
||||
/// Passed to `decrypt_sectors()` — the caller doesn't need to know
|
||||
/// which encryption scheme is in use.
|
||||
#[derive(Clone)]
|
||||
pub enum DecryptKeys {
|
||||
/// No encryption on this disc.
|
||||
None,
|
||||
/// AACS (Blu-ray / UHD). Unit keys + optional read data key.
|
||||
Aacs {
|
||||
unit_keys: Vec<(u32, [u8; 16])>,
|
||||
read_data_key: Option<[u8; 16]>,
|
||||
},
|
||||
/// CSS (DVD). Title key for sector descrambling.
|
||||
Css { title_key: [u8; 5] },
|
||||
}
|
||||
|
||||
impl DecryptKeys {
|
||||
/// True if there are keys to decrypt with.
|
||||
pub fn is_encrypted(&self) -> bool {
|
||||
!matches!(self, DecryptKeys::None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Decrypt a buffer of sectors in-place.
|
||||
///
|
||||
/// For AACS: processes in 6144-byte aligned units (3 sectors).
|
||||
/// For CSS: processes per 2048-byte sector.
|
||||
/// For None: no-op.
|
||||
///
|
||||
/// `unit_key_idx` selects which AACS unit key to use (0 for most discs).
|
||||
///
|
||||
/// Returns `Err` if decryption was expected but keys are missing or invalid.
|
||||
/// Never produces silently corrupted output.
|
||||
pub fn decrypt_sectors(
|
||||
buf: &mut [u8],
|
||||
keys: &DecryptKeys,
|
||||
unit_key_idx: usize,
|
||||
) -> Result<(), crate::error::Error> {
|
||||
match keys {
|
||||
DecryptKeys::None => {}
|
||||
DecryptKeys::Aacs {
|
||||
unit_keys,
|
||||
read_data_key,
|
||||
} => {
|
||||
let uk = match unit_keys.get(unit_key_idx) {
|
||||
Some((_, k)) => *k,
|
||||
None => {
|
||||
return Err(crate::error::Error::DecryptFailed);
|
||||
}
|
||||
};
|
||||
let rdk = read_data_key.as_ref();
|
||||
let unit_len = aacs::ALIGNED_UNIT_LEN;
|
||||
|
||||
for chunk in buf.chunks_mut(unit_len) {
|
||||
if chunk.len() == unit_len && aacs::is_unit_encrypted(chunk) {
|
||||
// `is_unit_encrypted` is a byte-0 heuristic: it fires on any
|
||||
// unit whose first byte has the top 2 bits set, which is
|
||||
// correct for m2ts source packets (where those bits are the
|
||||
// copy-control marker) but false-positives on any other binary
|
||||
// data with similarly-shaped first bytes — notably MPLS/CLPI
|
||||
// navigation files that begin with ASCII magic ('M', 'H'…)
|
||||
// and survive sweep mixed in with encrypted m2ts payloads.
|
||||
// `decrypt_unit_full` self-checks via TS-sync verification and
|
||||
// returns false on a misfire, but it has already mutated the
|
||||
// chunk by then. Snapshot and restore on verification failure
|
||||
// — same pattern `decrypt_unit_try_keys` uses for multi-key
|
||||
// discs. Real m2ts units verify and stay decrypted; nav-file
|
||||
// sectors get scrambled briefly and then put back as-was.
|
||||
let original: Vec<u8> = chunk.to_vec();
|
||||
if !aacs::decrypt_unit_full(chunk, &uk, rdk) {
|
||||
chunk.copy_from_slice(&original);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
DecryptKeys::Css { title_key } => {
|
||||
for chunk in buf.chunks_mut(2048) {
|
||||
css::lfsr::descramble_sector(title_key, chunk);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Regression for the 0.18.1 nav-file scramble bug. A non-m2ts unit whose
|
||||
/// first byte has the top 2 bits set (here: the ASCII letter 'M' that
|
||||
/// MPLS files start with, 0x4D = 0b01001101) trips `is_unit_encrypted`,
|
||||
/// gets AES-decrypted with the unit key, fails the TS-sync verification,
|
||||
/// and must be restored to its original bytes — not left scrambled.
|
||||
#[test]
|
||||
fn nav_file_unit_survives_decrypt_attempt() {
|
||||
let mut unit = vec![0u8; aacs::ALIGNED_UNIT_LEN];
|
||||
unit[0] = b'M';
|
||||
unit[1] = b'P';
|
||||
unit[2] = b'L';
|
||||
unit[3] = b'S';
|
||||
for (i, b) in unit.iter_mut().enumerate().skip(4) {
|
||||
*b = (i as u8).wrapping_mul(31);
|
||||
}
|
||||
let snapshot = unit.clone();
|
||||
|
||||
let keys = DecryptKeys::Aacs {
|
||||
unit_keys: vec![(0, [0xAB; 16])],
|
||||
read_data_key: None,
|
||||
};
|
||||
decrypt_sectors(&mut unit, &keys, 0).unwrap();
|
||||
assert_eq!(
|
||||
unit, snapshot,
|
||||
"non-m2ts unit must be restored after failed decrypt"
|
||||
);
|
||||
}
|
||||
}
|
||||
+25
-20
@@ -3,13 +3,13 @@
|
||||
use super::*;
|
||||
use crate::clpi;
|
||||
use crate::mpls;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf;
|
||||
|
||||
impl Disc {
|
||||
/// Scan Blu-ray titles from MPLS playlists.
|
||||
pub(super) fn scan_bluray_titles(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
) -> Vec<DiscTitle> {
|
||||
let mut titles = Vec::new();
|
||||
@@ -31,7 +31,7 @@ impl Disc {
|
||||
}
|
||||
|
||||
pub(super) fn parse_playlist(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
filename: &str,
|
||||
data: &[u8],
|
||||
@@ -66,17 +66,18 @@ impl Disc {
|
||||
pkt_count = clip_info.source_packet_count;
|
||||
total_size += pkt_count as u64 * 192;
|
||||
|
||||
// Get m2ts file start LBA and compute extent from packet count.
|
||||
// BD-ROM m2ts files are contiguous on disc (mastering requirement).
|
||||
// Get m2ts file extents from UDF allocation descriptors.
|
||||
// Dual-layer discs split files across layers — UDF knows the real layout.
|
||||
let m2ts_path = format!("/BDMV/STREAM/{}.m2ts", play_item.clip_id);
|
||||
let file_lba = udf_fs.file_start_lba(reader, &m2ts_path).unwrap_or(0);
|
||||
let total_bytes = pkt_count as u64 * 192;
|
||||
let total_sectors = total_bytes.div_ceil(2048) as u32;
|
||||
if total_sectors > 0 && file_lba > 0 {
|
||||
extents.push(Extent {
|
||||
start_lba: file_lba,
|
||||
sector_count: total_sectors,
|
||||
});
|
||||
if let Ok(file_exts) = udf_fs.file_extents(reader, &m2ts_path) {
|
||||
for (lba, sectors) in file_exts {
|
||||
if sectors > 0 && lba > 0 {
|
||||
extents.push(Extent {
|
||||
start_lba: lba,
|
||||
sector_count: sectors,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -104,8 +105,8 @@ impl Disc {
|
||||
1 | 6 | 7 => Some(Stream::Video(VideoStream {
|
||||
pid: s.pid,
|
||||
codec,
|
||||
resolution: format_resolution(s.video_format, s.video_rate),
|
||||
frame_rate: format_framerate(s.video_rate),
|
||||
resolution: Resolution::from_video_format(s.video_format),
|
||||
frame_rate: FrameRate::from_video_rate(s.video_rate),
|
||||
hdr: match s.dynamic_range {
|
||||
1 => HdrFormat::Hdr10,
|
||||
2 => HdrFormat::DolbyVision,
|
||||
@@ -131,16 +132,18 @@ impl Disc {
|
||||
codec,
|
||||
language: s.language.clone(),
|
||||
forced: false,
|
||||
qualifier: crate::disc::LabelQualifier::None,
|
||||
codec_data: None,
|
||||
}))
|
||||
} else {
|
||||
Some(Stream::Audio(AudioStream {
|
||||
pid: s.pid,
|
||||
codec,
|
||||
channels: format_channels(s.audio_format),
|
||||
channels: AudioChannels::from_audio_format(s.audio_format),
|
||||
language: s.language.clone(),
|
||||
sample_rate: format_samplerate(s.audio_rate),
|
||||
sample_rate: SampleRate::from_audio_rate(s.audio_rate),
|
||||
secondary: s.stream_type == 5,
|
||||
purpose: crate::disc::LabelPurpose::Normal,
|
||||
label: String::new(),
|
||||
}))
|
||||
}
|
||||
@@ -150,6 +153,7 @@ impl Disc {
|
||||
codec,
|
||||
language: s.language.clone(),
|
||||
forced: false,
|
||||
qualifier: crate::disc::LabelQualifier::None,
|
||||
codec_data: None,
|
||||
})),
|
||||
// Stream type 4 = IG, unknown types -- skip
|
||||
@@ -158,12 +162,12 @@ impl Disc {
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Convert marks to chapters (filter mark_type == 1 = chapter entry)
|
||||
// Convert marks to chapters (mark_type 0 or 1 = chapter entry, 2 = link)
|
||||
let first_in_time = parsed.play_items.first().map(|pi| pi.in_time).unwrap_or(0);
|
||||
let chapters: Vec<Chapter> = parsed
|
||||
.marks
|
||||
.iter()
|
||||
.filter(|m| m.mark_type == 1)
|
||||
.filter(|m| m.mark_type <= 1)
|
||||
.enumerate()
|
||||
.map(|(i, m)| {
|
||||
let time_secs = (m.timestamp as f64 - first_in_time as f64) / 45000.0;
|
||||
@@ -187,6 +191,7 @@ impl Disc {
|
||||
chapters,
|
||||
extents,
|
||||
content_format: ContentFormat::BdTs,
|
||||
codec_privates: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -194,7 +199,7 @@ impl Disc {
|
||||
/// Prefers English, falls back to first available language.
|
||||
/// Returns None if META directory is empty or XML has no usable title.
|
||||
pub(super) fn read_meta_title(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
) -> Option<String> {
|
||||
let meta_dir = udf_fs.find_dir("/BDMV/META")?;
|
||||
|
||||
+23
-35
@@ -2,13 +2,13 @@
|
||||
|
||||
use super::*;
|
||||
use crate::ifo;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf;
|
||||
|
||||
impl Disc {
|
||||
/// Scan DVD titles from IFO files (VIDEO_TS.IFO + VTS_XX_0.IFO).
|
||||
pub(super) fn scan_dvd_titles(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
) -> Vec<DiscTitle> {
|
||||
let dvd_info = match ifo::parse_vmg(reader, udf_fs) {
|
||||
@@ -20,20 +20,13 @@ impl Disc {
|
||||
let mut title_number: u16 = 0;
|
||||
|
||||
for ts in &dvd_info.title_sets {
|
||||
// Map DvdVideoAttr to Stream::Video
|
||||
let video_codec = match ts.video.codec.as_str() {
|
||||
"mpeg2" => Codec::Mpeg2,
|
||||
"mpeg1" => Codec::Mpeg2, // treat MPEG-1 as MPEG-2 for container purposes
|
||||
_ => Codec::Mpeg2,
|
||||
};
|
||||
|
||||
let video_stream = Stream::Video(VideoStream {
|
||||
pid: 0xE0, // DVD video PID (standard MPEG PS video stream)
|
||||
codec: video_codec,
|
||||
resolution: ts.video.resolution.clone(),
|
||||
codec: ts.video.codec,
|
||||
resolution: ts.video.resolution,
|
||||
frame_rate: match ts.video.standard.as_str() {
|
||||
"PAL" => "25".to_string(),
|
||||
_ => "29.97".to_string(),
|
||||
"PAL" => FrameRate::F25,
|
||||
_ => FrameRate::F29_97,
|
||||
},
|
||||
hdr: HdrFormat::Sdr,
|
||||
color_space: ColorSpace::Bt709,
|
||||
@@ -47,32 +40,15 @@ impl Disc {
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, a)| {
|
||||
let codec = match a.codec.as_str() {
|
||||
"ac3" => Codec::Ac3,
|
||||
"dts" => Codec::Dts,
|
||||
"lpcm" => Codec::Lpcm,
|
||||
"mpeg1" | "mpeg2" => Codec::Mpeg2,
|
||||
_ => Codec::Unknown(0),
|
||||
};
|
||||
let channels = match a.channels {
|
||||
1 => "mono".to_string(),
|
||||
2 => "stereo".to_string(),
|
||||
6 => "5.1".to_string(),
|
||||
8 => "7.1".to_string(),
|
||||
n => format!("{}ch", n),
|
||||
};
|
||||
let sample_rate = match a.sample_rate {
|
||||
48000 => "48kHz".to_string(),
|
||||
96000 => "96kHz".to_string(),
|
||||
sr => format!("{}kHz", sr / 1000),
|
||||
};
|
||||
let codec = a.codec;
|
||||
Stream::Audio(AudioStream {
|
||||
pid: 0xBD00 + i as u16, // DVD private stream 1 sub-IDs
|
||||
codec,
|
||||
channels,
|
||||
channels: AudioChannels::from_count(a.channels),
|
||||
language: a.language.clone(),
|
||||
sample_rate,
|
||||
sample_rate: SampleRate::from_hz(a.sample_rate),
|
||||
secondary: false,
|
||||
purpose: crate::disc::LabelPurpose::Normal,
|
||||
label: String::new(),
|
||||
})
|
||||
})
|
||||
@@ -117,6 +93,7 @@ impl Disc {
|
||||
codec: Codec::DvdSub,
|
||||
language: s.language.clone(),
|
||||
forced: false,
|
||||
qualifier: crate::disc::LabelQualifier::None,
|
||||
codec_data: codec_data.clone(),
|
||||
})
|
||||
})
|
||||
@@ -126,6 +103,16 @@ impl Disc {
|
||||
streams.extend(audio_streams.iter().cloned());
|
||||
streams.extend(subtitle_streams);
|
||||
|
||||
let chapters: Vec<Chapter> = dvd_title
|
||||
.chapter_times
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, &t)| Chapter {
|
||||
time_secs: t,
|
||||
name: format!("Chapter {}", i + 1),
|
||||
})
|
||||
.collect();
|
||||
|
||||
titles.push(DiscTitle {
|
||||
playlist: format!("VTS_{:02}_{}.VOB", ts.vts_number, title_number),
|
||||
playlist_id: title_number,
|
||||
@@ -133,9 +120,10 @@ impl Disc {
|
||||
size_bytes,
|
||||
clips: Vec::new(),
|
||||
streams,
|
||||
chapters: Vec::new(),
|
||||
chapters,
|
||||
extents,
|
||||
content_format: ContentFormat::MpegPs,
|
||||
codec_privates: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
+110
-16
@@ -2,7 +2,7 @@
|
||||
|
||||
use super::*;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf;
|
||||
|
||||
/// Result of SCSI AACS handshake (ECDH authentication).
|
||||
@@ -17,49 +17,108 @@ impl Disc {
|
||||
/// SCSI handshake result — volume ID and bus keys from ECDH authentication.
|
||||
/// Only available when scanning from a real drive (not ISO images).
|
||||
pub(super) fn do_handshake(
|
||||
session: &mut crate::drive::DriveSession,
|
||||
session: &mut crate::drive::Drive,
|
||||
opts: &ScanOptions,
|
||||
) -> Option<HandshakeResult> {
|
||||
use crate::aacs::{self, KeyDb};
|
||||
|
||||
let keydb_path = opts.resolve_keydb()?;
|
||||
let keydb = KeyDb::load(&keydb_path).ok()?;
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_entry",
|
||||
"do_handshake entered"
|
||||
);
|
||||
let keydb_path = match opts.resolve_keydb() {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_no_keydb",
|
||||
"no KEYDB found in search paths; handshake skipped"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let keydb = match KeyDb::load(&keydb_path) {
|
||||
Ok(db) => db,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_keydb_load_failed",
|
||||
io_error_kind = ?e.kind(),
|
||||
keydb = %keydb_path.display(),
|
||||
"KEYDB load failed; handshake skipped"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let mut last_error = None;
|
||||
for hc in &keydb.host_certs {
|
||||
let host_cert_count = keydb.host_certs.len();
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_start",
|
||||
host_cert_count,
|
||||
keydb = %keydb_path.display(),
|
||||
"handshake starting"
|
||||
);
|
||||
|
||||
const MAX_CERT_ATTEMPTS: usize = 16;
|
||||
let mut last_err_code: Option<u16> = None;
|
||||
for (idx, hc) in keydb.host_certs.iter().take(MAX_CERT_ATTEMPTS).enumerate() {
|
||||
match aacs::handshake::aacs_authenticate(session, &hc.private_key, &hc.certificate) {
|
||||
Ok(mut auth) => {
|
||||
let volume_id =
|
||||
aacs::handshake::read_volume_id(session, &mut auth).unwrap_or([0u8; 16]);
|
||||
let volume_id = match aacs::handshake::read_volume_id(session, &mut auth) {
|
||||
Ok(vid) => vid,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_vid_read_failed",
|
||||
cert_index = idx,
|
||||
error_code = e.code(),
|
||||
"auth ok but volume ID read failed"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let read_data_key = aacs::handshake::read_data_keys(session, &mut auth)
|
||||
.ok()
|
||||
.map(|(rdk, _)| rdk);
|
||||
tracing::debug!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_ok",
|
||||
cert_index = idx,
|
||||
has_read_data_key = read_data_key.is_some(),
|
||||
);
|
||||
return Some(HandshakeResult {
|
||||
volume_id,
|
||||
read_data_key,
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
// Try next host cert
|
||||
last_error = Some(e);
|
||||
last_err_code = Some(e.code());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
last_error.map(|_e| HandshakeResult {
|
||||
volume_id: [0u8; 16],
|
||||
read_data_key: None,
|
||||
})
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_all_certs_failed",
|
||||
host_cert_count,
|
||||
tried = host_cert_count.min(MAX_CERT_ATTEMPTS),
|
||||
last_error_code = last_err_code,
|
||||
"all host certs in KEYDB rejected by drive"
|
||||
);
|
||||
// All host certs failed — return None, not a fake success
|
||||
None
|
||||
}
|
||||
|
||||
/// Resolve disc encryption — AACS 1.0, AACS 2.0, CSS, or none.
|
||||
///
|
||||
/// Reads AACS files from UDF (via SectorReader), resolves keys through
|
||||
/// Reads AACS files from UDF (via SectorSource), resolves keys through
|
||||
/// whatever path works: KEYDB VUK lookup, media key derivation, processing
|
||||
/// keys, device keys. Uses handshake result (volume ID, bus key) if available.
|
||||
pub(super) fn resolve_encryption(
|
||||
udf_fs: &udf::UdfFs,
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
keydb_path: &std::path::Path,
|
||||
handshake: Option<&HandshakeResult>,
|
||||
) -> Result<AacsState> {
|
||||
@@ -75,6 +134,19 @@ impl Disc {
|
||||
.or_else(|_| udf_fs.read_file(reader, "/AACS/DUPLICATE/Unit_Key_RO.inf"))
|
||||
.map_err(|_| Error::AacsNoKeys)?;
|
||||
|
||||
// Log the disc hash so we can confirm whether it's present in KEYDB
|
||||
// when key resolution fails. The disc hash is SHA-1 of the full
|
||||
// Unit_Key_RO.inf file bytes — same value KEYDB.cfg keys VUK entries by.
|
||||
let dh = crate::aacs::disc_hash(&uk_ro_data);
|
||||
let dh_hex = crate::aacs::disc_hash_hex(&dh);
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "scan_aacs_disc_hash",
|
||||
disc_hash = %dh_hex,
|
||||
uk_ro_len = uk_ro_data.len(),
|
||||
"disc hash computed (compare with keydb.cfg entries)"
|
||||
);
|
||||
|
||||
let cc_data = udf_fs
|
||||
.read_file(reader, "/AACS/Content000.cer")
|
||||
.or_else(|_| udf_fs.read_file(reader, "/AACS/Content001.cer"))
|
||||
@@ -86,6 +158,28 @@ impl Disc {
|
||||
.ok();
|
||||
let mkb_ver = mkb_data.as_deref().and_then(aacs::mkb_version);
|
||||
|
||||
let mkb_first_64_hex = mkb_data
|
||||
.as_deref()
|
||||
.map(|m| {
|
||||
m.iter()
|
||||
.take(64)
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect::<String>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "scan_aacs_mkb_info",
|
||||
mkb_present = mkb_data.is_some(),
|
||||
mkb_len = mkb_data.as_deref().map(|m| m.len()).unwrap_or(0),
|
||||
mkb_version = ?mkb_ver,
|
||||
mkb_first_64 = %mkb_first_64_hex,
|
||||
keydb_disc_count = keydb.disc_entries.len(),
|
||||
keydb_dk_count = keydb.device_keys.len(),
|
||||
keydb_pk_count = keydb.processing_keys.len(),
|
||||
"AACS resolution inputs"
|
||||
);
|
||||
|
||||
// Use handshake volume ID if available, otherwise zeros
|
||||
// (KEYDB VUK lookup by disc hash works without volume ID)
|
||||
let volume_id = handshake.map(|h| h.volume_id).unwrap_or([0u8; 16]);
|
||||
|
||||
@@ -0,0 +1,610 @@
|
||||
//! ddrescue-compatible mapfile for tracking rip progress.
|
||||
//!
|
||||
//! Records which byte ranges of a disc image are good, unreadable,
|
||||
//! or not-yet-attempted. Written as plain text so it's greppable,
|
||||
//! human-editable, and interoperates with ddrescue's own tools.
|
||||
//!
|
||||
//! Format:
|
||||
//! ```text
|
||||
//! # Rescue Logfile. Created by libfreemkv v0.11.21
|
||||
//! # Current pos / status / pass / pass_time (ddrescue state machine — we only populate pos)
|
||||
//! 0x000000000 ? 1 0
|
||||
//! # pos size status
|
||||
//! 0x000000000 0x12345678 +
|
||||
//! 0x012345678 0x00001000 -
|
||||
//! 0x012346678 0x01234500 ?
|
||||
//! ```
|
||||
//!
|
||||
//! Status chars: `?` non-tried · `*` non-trimmed · `/` non-scraped · `-` unreadable · `+` finished.
|
||||
//!
|
||||
//! The mapfile is flushed to disk at most once per `FLUSH_INTERVAL`
|
||||
//! during `record()` calls, plus on explicit `flush()` and on `Drop`.
|
||||
//! This bounds atomic-rename RPC rate on networked staging (e.g. NFS)
|
||||
//! where per-record persists otherwise serialize the rip pipeline.
|
||||
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Minimum interval between mapfile persists. `record()` updates in-memory
|
||||
/// state every call but only writes to disk when this interval has elapsed
|
||||
/// since the last persist (or when `flush()` is called explicitly, or on
|
||||
/// `Drop`). Bounds RPC rate on NFS staging where atomic-rename per record
|
||||
/// otherwise dominates throughput. On crash the worst-case progress loss
|
||||
/// is one interval's worth of records.
|
||||
const FLUSH_INTERVAL: Duration = Duration::from_millis(1000);
|
||||
|
||||
/// Status of a byte range in the mapfile. ddrescue-compatible.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SectorStatus {
|
||||
/// `?` — not yet attempted. Initial state for a fresh mapfile.
|
||||
NonTried,
|
||||
/// `*` — fast-pass read failed; edges need trimming.
|
||||
NonTrimmed,
|
||||
/// `/` — trimmed; interior needs sector scrape.
|
||||
NonScraped,
|
||||
/// `-` — drive couldn't read it this session.
|
||||
Unreadable,
|
||||
/// `+` — good.
|
||||
Finished,
|
||||
}
|
||||
|
||||
impl SectorStatus {
|
||||
pub fn to_char(self) -> char {
|
||||
match self {
|
||||
Self::NonTried => '?',
|
||||
Self::NonTrimmed => '*',
|
||||
Self::NonScraped => '/',
|
||||
Self::Unreadable => '-',
|
||||
Self::Finished => '+',
|
||||
}
|
||||
}
|
||||
pub fn from_char(c: char) -> Option<Self> {
|
||||
Some(match c {
|
||||
'?' => Self::NonTried,
|
||||
'*' => Self::NonTrimmed,
|
||||
'/' => Self::NonScraped,
|
||||
'-' => Self::Unreadable,
|
||||
'+' => Self::Finished,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// One contiguous range of bytes with a status.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct MapEntry {
|
||||
pub pos: u64,
|
||||
pub size: u64,
|
||||
pub status: SectorStatus,
|
||||
}
|
||||
|
||||
/// Summary statistics over all entries.
|
||||
///
|
||||
/// `bytes_pending` aggregates `NonTried + NonTrimmed + NonScraped` for
|
||||
/// back-compat. `bytes_nontried` and `bytes_retryable` (= NonTrimmed +
|
||||
/// NonScraped) split that aggregate so UIs can distinguish *unread*
|
||||
/// territory (still ahead of Pass 1's read head) from *needs-retry*
|
||||
/// territory (Pass 1 already encountered, queued for Pass 2-N).
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct MapStats {
|
||||
pub bytes_total: u64,
|
||||
pub bytes_good: u64,
|
||||
pub bytes_unreadable: u64,
|
||||
pub bytes_pending: u64,
|
||||
/// Sectors Pass 1 hasn't reached yet (`NonTried`). Subset of
|
||||
/// `bytes_pending`.
|
||||
pub bytes_nontried: u64,
|
||||
/// Sectors flagged for Pass 2-N retry — `NonTrimmed` (multi-sector
|
||||
/// read failed; needs split) + `NonScraped` (small-block read
|
||||
/// partially recovered; remainder still pending). Subset of
|
||||
/// `bytes_pending`. This is the right signal for a "MAYBE / will
|
||||
/// retry" UI bucket; `bytes_pending` over-counts because it folds
|
||||
/// in `bytes_nontried`.
|
||||
pub bytes_retryable: u64,
|
||||
/// Number of unreadable ranges (for UI display). Computed from
|
||||
/// `ranges_with(&[Unreadable])`.
|
||||
pub num_bad_ranges: u32,
|
||||
/// Largest gap among unreadable ranges in milliseconds. Computed as
|
||||
/// largest range size / bytes_per_sec * 1000. Set by caller (autorip)
|
||||
/// since bytes_per_sec is application-specific.
|
||||
pub main_lost_ms: f64,
|
||||
}
|
||||
|
||||
/// Time-batched mapfile. `record()` keeps in-memory state up-to-date on
|
||||
/// every call; persists to disk at most once per `FLUSH_INTERVAL`.
|
||||
/// Explicit `flush()` and `Drop` guarantee state is on disk after a sweep
|
||||
/// or patch finishes. On hard crash the worst-case loss is one flush
|
||||
/// interval of records — the file's payload bytes are unaffected.
|
||||
pub struct Mapfile {
|
||||
path: PathBuf,
|
||||
entries: Vec<MapEntry>,
|
||||
total_size: u64,
|
||||
version: String,
|
||||
/// Incrementally maintained stats — updated on every `record()` call
|
||||
/// so `stats()` is O(1) instead of O(n).
|
||||
stats: MapStats,
|
||||
/// True when in-memory state has changed but `write_to_disk` has not
|
||||
/// yet captured it.
|
||||
dirty: bool,
|
||||
/// Wall-clock timestamp of the last successful `write_to_disk` (or
|
||||
/// the moment the mapfile was constructed, whichever is later).
|
||||
last_flushed: Instant,
|
||||
}
|
||||
|
||||
impl Mapfile {
|
||||
/// Create a new mapfile with one `NonTried` region covering the whole disc.
|
||||
/// Writes to disk immediately so a resume can pick up even if the caller
|
||||
/// never records anything.
|
||||
pub fn create(path: &Path, total_size: u64, version: &str) -> io::Result<Self> {
|
||||
let mut mf = Self {
|
||||
path: path.to_path_buf(),
|
||||
entries: vec![MapEntry {
|
||||
pos: 0,
|
||||
size: total_size,
|
||||
status: SectorStatus::NonTried,
|
||||
}],
|
||||
total_size,
|
||||
version: version.to_string(),
|
||||
stats: MapStats {
|
||||
bytes_total: total_size,
|
||||
bytes_pending: total_size,
|
||||
bytes_nontried: total_size,
|
||||
..Default::default()
|
||||
},
|
||||
dirty: false,
|
||||
last_flushed: Instant::now(),
|
||||
};
|
||||
// Eager initial persist so a resume can pick this up even if
|
||||
// `record()` is never called.
|
||||
mf.write_to_disk()?;
|
||||
mf.last_flushed = Instant::now();
|
||||
Ok(mf)
|
||||
}
|
||||
|
||||
/// Load an existing mapfile from disk.
|
||||
pub fn load(path: &Path) -> io::Result<Self> {
|
||||
let text = std::fs::read_to_string(path)?;
|
||||
let mut entries = Vec::new();
|
||||
let mut saw_current_line = false;
|
||||
let mut version = String::from("unknown");
|
||||
for line in text.lines() {
|
||||
let t = line.trim();
|
||||
if t.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Some(rest) = t.strip_prefix('#') {
|
||||
let rest = rest.trim();
|
||||
if let Some(v) = rest.strip_prefix("Rescue Logfile. Created by ") {
|
||||
version = v.to_string();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// First non-comment line is the "current" state line (pos status [pass] [pass_time]).
|
||||
// We ignore its contents but skip over it.
|
||||
if !saw_current_line {
|
||||
saw_current_line = true;
|
||||
// But if the line looks like an entry (has at least 3 fields starting 0x...),
|
||||
// it's probably actually an entry for a mapfile we wrote without a current line.
|
||||
// Heuristic: current line has status char as 2nd field; entry has size as 2nd field.
|
||||
let fields: Vec<&str> = t.split_whitespace().collect();
|
||||
if fields.len() >= 3 && fields[1].starts_with("0x") {
|
||||
// It's an entry, not a current line — fall through to entry parse.
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Entry: `pos size statuschar`
|
||||
let fields: Vec<&str> = t.split_whitespace().collect();
|
||||
if fields.len() < 3 {
|
||||
continue;
|
||||
}
|
||||
let pos = parse_hex(fields[0])?;
|
||||
let size = parse_hex(fields[1])?;
|
||||
let status = fields[2]
|
||||
.chars()
|
||||
.next()
|
||||
.and_then(SectorStatus::from_char)
|
||||
.ok_or_else(|| {
|
||||
// No English text — the variant carries a stable
|
||||
// language-neutral kind identifier (`status_char`).
|
||||
let e: io::Error = crate::error::Error::MapfileInvalid {
|
||||
kind: "status_char",
|
||||
}
|
||||
.into();
|
||||
e
|
||||
})?;
|
||||
entries.push(MapEntry { pos, size, status });
|
||||
}
|
||||
entries.sort_by_key(|e| e.pos);
|
||||
let total_size = entries.last().map(|e| e.pos + e.size).unwrap_or(0);
|
||||
let stats = Self::compute_stats(&entries, total_size);
|
||||
Ok(Self {
|
||||
path: path.to_path_buf(),
|
||||
entries,
|
||||
total_size,
|
||||
version,
|
||||
stats,
|
||||
dirty: false,
|
||||
last_flushed: Instant::now(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Load if the file exists, otherwise create a fresh mapfile.
|
||||
pub fn open_or_create(path: &Path, total_size: u64, version: &str) -> io::Result<Self> {
|
||||
match Self::load(path) {
|
||||
Ok(mf) => Ok(mf),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||
Self::create(path, total_size, version)
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark a byte range as having the given status. Splits any overlapping
|
||||
/// existing entries, merges with adjacent same-status entries, and flushes
|
||||
/// to disk.
|
||||
pub fn record(&mut self, pos: u64, size: u64, status: SectorStatus) -> io::Result<()> {
|
||||
if size == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let end = pos.saturating_add(size);
|
||||
let mut new_entries = Vec::with_capacity(self.entries.len() + 2);
|
||||
|
||||
for e in self.entries.drain(..) {
|
||||
let e_end = e.pos + e.size;
|
||||
if e_end <= pos || e.pos >= end {
|
||||
// entirely before or after — keep
|
||||
new_entries.push(e);
|
||||
continue;
|
||||
}
|
||||
// Overlap — keep portions outside [pos, end)
|
||||
if e.pos < pos {
|
||||
new_entries.push(MapEntry {
|
||||
pos: e.pos,
|
||||
size: pos - e.pos,
|
||||
status: e.status,
|
||||
});
|
||||
}
|
||||
if e_end > end {
|
||||
new_entries.push(MapEntry {
|
||||
pos: end,
|
||||
size: e_end - end,
|
||||
status: e.status,
|
||||
});
|
||||
}
|
||||
}
|
||||
new_entries.push(MapEntry { pos, size, status });
|
||||
new_entries.sort_by_key(|e| e.pos);
|
||||
|
||||
// Coalesce adjacent same-status entries.
|
||||
let mut merged: Vec<MapEntry> = Vec::with_capacity(new_entries.len());
|
||||
for e in new_entries {
|
||||
if let Some(last) = merged.last_mut() {
|
||||
if last.pos + last.size == e.pos && last.status == e.status {
|
||||
last.size += e.size;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
merged.push(e);
|
||||
}
|
||||
|
||||
// Recompute stats from merged entries. record() is already O(n) due to
|
||||
// drain-and-rebuild, so this is a constant-factor overhead. The critical
|
||||
// win is that stats() is now O(1) — called millions of times in the hot
|
||||
// path during sweep/patch, it just returns the cached value.
|
||||
self.stats = Self::compute_stats(&merged, self.total_size);
|
||||
self.entries = merged;
|
||||
self.dirty = true;
|
||||
if self.last_flushed.elapsed() >= FLUSH_INTERVAL {
|
||||
self.write_to_disk()?;
|
||||
self.dirty = false;
|
||||
self.last_flushed = Instant::now();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist any pending in-memory changes to disk. No-op if clean.
|
||||
/// Callers (sweep/patch finalisation) invoke this after their last
|
||||
/// `record()` to guarantee state is durable before returning.
|
||||
pub fn flush(&mut self) -> io::Result<()> {
|
||||
if self.dirty {
|
||||
self.write_to_disk()?;
|
||||
self.dirty = false;
|
||||
self.last_flushed = Instant::now();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn entries(&self) -> &[MapEntry] {
|
||||
&self.entries
|
||||
}
|
||||
|
||||
pub fn total_size(&self) -> u64 {
|
||||
self.total_size
|
||||
}
|
||||
|
||||
/// First range with a given status starting at or after `from`.
|
||||
pub fn next_with(&self, from: u64, status: SectorStatus) -> Option<(u64, u64)> {
|
||||
for e in &self.entries {
|
||||
if e.status != status {
|
||||
continue;
|
||||
}
|
||||
let e_end = e.pos + e.size;
|
||||
if e_end <= from {
|
||||
continue;
|
||||
}
|
||||
let start = e.pos.max(from);
|
||||
return Some((start, e_end - start));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// All ranges matching one of the given statuses, in position order.
|
||||
pub fn ranges_with(&self, statuses: &[SectorStatus]) -> Vec<(u64, u64)> {
|
||||
self.entries
|
||||
.iter()
|
||||
.filter(|e| statuses.contains(&e.status))
|
||||
.map(|e| (e.pos, e.size))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> MapStats {
|
||||
self.stats
|
||||
}
|
||||
|
||||
fn compute_stats(entries: &[MapEntry], total_size: u64) -> MapStats {
|
||||
let mut s = MapStats {
|
||||
bytes_total: total_size,
|
||||
..Default::default()
|
||||
};
|
||||
for e in entries {
|
||||
match e.status {
|
||||
SectorStatus::Finished => s.bytes_good += e.size,
|
||||
SectorStatus::Unreadable => s.bytes_unreadable += e.size,
|
||||
SectorStatus::NonTried => {
|
||||
s.bytes_pending += e.size;
|
||||
s.bytes_nontried += e.size;
|
||||
}
|
||||
SectorStatus::NonTrimmed | SectorStatus::NonScraped => {
|
||||
s.bytes_pending += e.size;
|
||||
s.bytes_retryable += e.size;
|
||||
}
|
||||
}
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
fn write_to_disk(&self) -> io::Result<()> {
|
||||
// Write to a tempfile then rename for atomicity. Appending ".tmp"
|
||||
// rather than `with_extension` so we don't clobber the original
|
||||
// extension (which may already be ".mapfile").
|
||||
let tmp = {
|
||||
let mut s = self.path.clone().into_os_string();
|
||||
s.push(".tmp");
|
||||
PathBuf::from(s)
|
||||
};
|
||||
{
|
||||
let file = std::fs::File::create(&tmp)?;
|
||||
let mut w = std::io::BufWriter::new(file);
|
||||
writeln!(w, "# Rescue Logfile. Created by {}", self.version)?;
|
||||
writeln!(w, "# Current pos / status / pass / pass_time")?;
|
||||
writeln!(w, "0x000000000 ? 1 0")?;
|
||||
writeln!(w, "# pos size status")?;
|
||||
for e in &self.entries {
|
||||
writeln!(
|
||||
w,
|
||||
"0x{:09x} 0x{:09x} {}",
|
||||
e.pos,
|
||||
e.size,
|
||||
e.status.to_char()
|
||||
)?;
|
||||
}
|
||||
w.flush()?;
|
||||
}
|
||||
std::fs::rename(&tmp, &self.path)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Mapfile {
|
||||
/// Best-effort flush on drop so a sweep / patch that returns early
|
||||
/// (or unwinds) doesn't lose its in-memory state. Errors here are
|
||||
/// swallowed because Drop has no way to surface them; explicit
|
||||
/// `flush()` on the success path gives callers proper error handling.
|
||||
fn drop(&mut self) {
|
||||
let _ = self.flush();
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_hex(s: &str) -> io::Result<u64> {
|
||||
let s = s.strip_prefix("0x").unwrap_or(s);
|
||||
u64::from_str_radix(s, 16).map_err(|_| {
|
||||
// Underlying ParseIntError dropped — its Display is OS-locale text.
|
||||
// The typed variant carries `kind = "hex"` which is stable.
|
||||
let e: io::Error = crate::error::Error::MapfileInvalid { kind: "hex" }.into();
|
||||
e
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn tmpfile(tag: &str) -> PathBuf {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
static CTR: AtomicU64 = AtomicU64::new(0);
|
||||
let n = CTR.fetch_add(1, Ordering::Relaxed);
|
||||
let name = format!(
|
||||
"libfreemkv-mapfile-test-{}-{}-{}.mapfile",
|
||||
std::process::id(),
|
||||
tag,
|
||||
n
|
||||
);
|
||||
std::env::temp_dir().join(name)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_has_one_nontried_region() {
|
||||
let p = tmpfile("create_has_one_nontried_region");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
assert_eq!(mf.entries().len(), 1);
|
||||
assert_eq!(mf.entries()[0].pos, 0);
|
||||
assert_eq!(mf.entries()[0].size, 1000);
|
||||
assert_eq!(mf.entries()[0].status, SectorStatus::NonTried);
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_splits_overlap() {
|
||||
let p = tmpfile("record_splits_overlap");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
mf.record(200, 100, SectorStatus::Finished).unwrap();
|
||||
let es = mf.entries();
|
||||
assert_eq!(es.len(), 3);
|
||||
assert_eq!(
|
||||
(es[0].pos, es[0].size, es[0].status),
|
||||
(0, 200, SectorStatus::NonTried)
|
||||
);
|
||||
assert_eq!(
|
||||
(es[1].pos, es[1].size, es[1].status),
|
||||
(200, 100, SectorStatus::Finished)
|
||||
);
|
||||
assert_eq!(
|
||||
(es[2].pos, es[2].size, es[2].status),
|
||||
(300, 700, SectorStatus::NonTried)
|
||||
);
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_coalesces_adjacent_same_status() {
|
||||
let p = tmpfile("record_coalesces_adjacent_same_status");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
mf.record(100, 100, SectorStatus::Finished).unwrap();
|
||||
mf.record(200, 100, SectorStatus::Finished).unwrap();
|
||||
// Entries: [0..100 NonTried, 100..300 Finished (merged), 300..1000 NonTried]
|
||||
let es = mf.entries();
|
||||
assert_eq!(es.len(), 3);
|
||||
assert_eq!(
|
||||
(es[1].pos, es[1].size, es[1].status),
|
||||
(100, 200, SectorStatus::Finished)
|
||||
);
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_replaces_existing_status() {
|
||||
let p = tmpfile("record_replaces_existing_status");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
mf.record(200, 100, SectorStatus::Unreadable).unwrap();
|
||||
mf.record(200, 100, SectorStatus::Finished).unwrap();
|
||||
let es = mf.entries();
|
||||
// The overwrite should result in all finished at 200..300, NonTried elsewhere — 3 entries.
|
||||
assert_eq!(es.len(), 3);
|
||||
assert_eq!(es[1].status, SectorStatus::Finished);
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trip_load() {
|
||||
let p = tmpfile("round_trip_load");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
mf.record(100, 200, SectorStatus::Finished).unwrap();
|
||||
mf.record(500, 100, SectorStatus::Unreadable).unwrap();
|
||||
// record() batches; explicit flush before reading back from disk.
|
||||
mf.flush().unwrap();
|
||||
let loaded = Mapfile::load(&p).unwrap();
|
||||
assert_eq!(loaded.entries(), mf.entries());
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_sum_correctly() {
|
||||
let p = tmpfile("stats_sum_correctly");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
mf.record(0, 400, SectorStatus::Finished).unwrap();
|
||||
mf.record(400, 100, SectorStatus::Unreadable).unwrap();
|
||||
let s = mf.stats();
|
||||
assert_eq!(s.bytes_good, 400);
|
||||
assert_eq!(s.bytes_unreadable, 100);
|
||||
assert_eq!(s.bytes_pending, 500);
|
||||
assert_eq!(s.bytes_total, 1000);
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ranges_with_filters() {
|
||||
let p = tmpfile("ranges_with_filters");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
mf.record(100, 50, SectorStatus::Unreadable).unwrap();
|
||||
mf.record(300, 50, SectorStatus::Unreadable).unwrap();
|
||||
let bad = mf.ranges_with(&[SectorStatus::Unreadable]);
|
||||
assert_eq!(bad, vec![(100, 50), (300, 50)]);
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_consistent_after_overlapping_records() {
|
||||
let p = tmpfile("stats_consistent_after_overlapping");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
// Record some finished, some unreadable, some nontrimmed
|
||||
mf.record(0, 300, SectorStatus::Finished).unwrap();
|
||||
mf.record(300, 200, SectorStatus::NonTrimmed).unwrap();
|
||||
mf.record(500, 100, SectorStatus::Unreadable).unwrap();
|
||||
mf.record(600, 400, SectorStatus::Finished).unwrap();
|
||||
|
||||
// Final entries: [0..300 Finished, 300..500 NonTrimmed, 500..600 Unreadable, 600..1000 Finished]
|
||||
let s = mf.stats();
|
||||
assert_eq!(s.bytes_good, 700); // 300 + 400
|
||||
assert_eq!(s.bytes_unreadable, 100); // 100
|
||||
assert_eq!(s.bytes_pending, 200); // NonTrimmed only (NonTried=0)
|
||||
assert_eq!(s.bytes_nontried, 0);
|
||||
assert_eq!(s.bytes_retryable, 200); // NonTrimmed
|
||||
assert_eq!(s.bytes_total, 1000);
|
||||
|
||||
// Overwrite a NonTrimmed range with Finished
|
||||
mf.record(300, 100, SectorStatus::Finished).unwrap();
|
||||
// Entries: [0..400 Finished, 400..500 NonTrimmed, 500..600 Unreadable, 600..1000 Finished]
|
||||
let s2 = mf.stats();
|
||||
assert_eq!(s2.bytes_good, 800); // 400 + 400
|
||||
assert_eq!(s2.bytes_unreadable, 100);
|
||||
assert_eq!(s2.bytes_pending, 100); // NonTrimmed only
|
||||
assert_eq!(s2.bytes_retryable, 100);
|
||||
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_consistent_after_split_record() {
|
||||
let p = tmpfile("stats_consistent_after_split");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
// Mark middle as NonTrimmed
|
||||
mf.record(200, 400, SectorStatus::NonTrimmed).unwrap();
|
||||
// Entries: [0..200 NonTried, 200..600 NonTrimmed, 600..1000 NonTried]
|
||||
let s = mf.stats();
|
||||
assert_eq!(s.bytes_pending, 1000); // NonTried(600) + NonTrimmed(400)
|
||||
assert_eq!(s.bytes_retryable, 400); // NonTrimmed only
|
||||
assert_eq!(s.bytes_nontried, 600); // 200 + 400
|
||||
|
||||
// Overwrite the NonTrimmed with Finished (splitting the remaining NonTried)
|
||||
mf.record(200, 400, SectorStatus::Finished).unwrap();
|
||||
// Entries: [0..200 NonTried, 200..600 Finished, 600..1000 NonTried]
|
||||
let s2 = mf.stats();
|
||||
assert_eq!(s2.bytes_good, 400);
|
||||
assert_eq!(s2.bytes_pending, 600); // NonTried(200 + 400)
|
||||
assert_eq!(s2.bytes_nontried, 600);
|
||||
assert_eq!(s2.bytes_retryable, 0);
|
||||
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
}
|
||||
+2029
-497
File diff suppressed because it is too large
Load Diff
+1787
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,239 @@
|
||||
//! `Disc::sweep`'s consumer-side `Sink<WorkItem>`.
|
||||
//!
|
||||
//! Background: the original sweep loop runs strictly serialised —
|
||||
//! SCSI read → decrypt → seek + write → mapfile.record → next iter.
|
||||
//! On a healthy disc the SCSI read costs ~5-12 ms per 64 KB batch and
|
||||
//! the post-read work (decrypt 1-3 ms + file write + mapfile fsync
|
||||
//! 5-15 ms) adds another batch's worth of latency. The drive idles
|
||||
//! during the post-read work; throughput tops out at the *sum* of
|
||||
//! both costs.
|
||||
//!
|
||||
//! 0.17.11 introduced a bespoke producer/consumer split (the now-
|
||||
//! removed `disc/sweep_pipeline.rs`) to overlap the two stages. 0.18
|
||||
//! collapses that split — together with the analogous splits patch
|
||||
//! and mux need — onto the generic [`crate::io::Pipeline`] +
|
||||
//! [`crate::io::Sink`] primitive. This module is the sweep-specific
|
||||
//! `Sink` impl; the producer-side state machine (read_error context,
|
||||
//! decrypt, set_speed, halt) stays in `Disc::sweep` in `disc/mod.rs`.
|
||||
//!
|
||||
//! Correctness invariants preserved (same as 0.17.11):
|
||||
//! - Mapfile is single-writer (consumer-only). No locking.
|
||||
//! - All `read_error::ReadCtx` state stays on the producer thread.
|
||||
//! - `set_speed` calls happen on the producer thread (same thread that
|
||||
//! owns the `SectorSource`). No new SCSI concurrency.
|
||||
//! - Per-iteration ordering of file-write → mapfile-record is kept
|
||||
//! intact in the consumer (write before record), so the on-disk
|
||||
//! invariant "mapfile only marks Finished what the file has
|
||||
//! received" survives a crash mid-pass.
|
||||
//! - The BU40N+Initio bridge wedge concern is unchanged: only one
|
||||
//! SCSI command in flight at a time, error-path timing identical,
|
||||
//! no new retry logic.
|
||||
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
use std::sync::mpsc::{Receiver, SyncSender, sync_channel};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::io::{Flow, Sink};
|
||||
|
||||
use super::mapfile::{MapStats, Mapfile, SectorStatus};
|
||||
|
||||
/// Reusable zero buffer for SkipFill / GapFill / BisectBad. 64 KB
|
||||
/// matches the existing zero_gap chunk size used by the pre-split
|
||||
/// sweep loop.
|
||||
const ZERO_CHUNK: usize = 65 * 1024;
|
||||
|
||||
/// Producer → Consumer messages. The consumer applies these in FIFO
|
||||
/// order; ordering of file writes and mapfile records across items is
|
||||
/// preserved.
|
||||
pub(super) enum WorkItem {
|
||||
/// Successful batch read. Producer has already decrypted `buf` if
|
||||
/// `opts.decrypt` was set. Consumer writes `buf` at `pos` and
|
||||
/// records the range as `Finished`.
|
||||
Good { pos: u64, buf: Vec<u8> },
|
||||
|
||||
/// Bisect inner-loop good single sector (already decrypted by the
|
||||
/// producer). 2048 bytes.
|
||||
BisectGood { pos: u64, buf: Box<[u8; 2048]> },
|
||||
|
||||
/// Bisect inner-loop bad single sector. Consumer writes 2048
|
||||
/// zeros at `pos` and records the sector as `NonTrimmed`.
|
||||
BisectBad { pos: u64 },
|
||||
|
||||
/// Whole-batch zero-fill (failed batch on `SkipBlock`, or the
|
||||
/// failed batch portion of `JumpAhead`). Consumer streams zeros
|
||||
/// across `[pos, pos+len)` and records the range as `NonTrimmed`.
|
||||
SkipFill { pos: u64, len: u64 },
|
||||
|
||||
/// Gap fill following a `JumpAhead`. Same effect as `SkipFill`;
|
||||
/// distinguished only so future logging / instrumentation can
|
||||
/// tell them apart without parsing a flag.
|
||||
GapFill { pos: u64, len: u64 },
|
||||
|
||||
/// Producer wants the latest mapfile stats for the progress
|
||||
/// callback. Consumer responds on `prog_tx` with a fresh
|
||||
/// [`ProgressSnapshot`]. Best-effort: if the producer hasn't
|
||||
/// drained the previous snapshot, the new one is silently
|
||||
/// dropped — the producer's local cache stays current enough.
|
||||
StatsRequest,
|
||||
}
|
||||
|
||||
/// Snapshot the consumer sends back to the producer for the progress
|
||||
/// callback.
|
||||
pub(super) struct ProgressSnapshot {
|
||||
pub stats: MapStats,
|
||||
pub bad_ranges: Vec<(u64, u64)>,
|
||||
}
|
||||
|
||||
/// Final summary returned by the consumer thread on shutdown — what
|
||||
/// `SweepSink::close` produces, surfaced to the producer via
|
||||
/// `Pipeline::finish`.
|
||||
pub(super) struct ConsumerSummary {
|
||||
pub stats: MapStats,
|
||||
}
|
||||
|
||||
/// Drain any pending progress snapshots from the consumer. Returns
|
||||
/// the most recent one, if any. The producer caches it and uses it
|
||||
/// for subsequent progress callbacks until a fresh one arrives.
|
||||
pub(super) fn try_recv_progress(rx: &Receiver<ProgressSnapshot>) -> Option<ProgressSnapshot> {
|
||||
let mut latest = None;
|
||||
while let Ok(snap) = rx.try_recv() {
|
||||
latest = Some(snap);
|
||||
}
|
||||
latest
|
||||
}
|
||||
|
||||
/// `Sink<WorkItem>` for sweep. Owns the writeback file + mapfile +
|
||||
/// progress back-channel. `apply` carries the file-write +
|
||||
/// mapfile.record per item; `close` drains the writeback pipeline,
|
||||
/// fsyncs the ISO, and flushes the mapfile.
|
||||
pub(super) struct SweepSink {
|
||||
file: crate::io::WritebackFile,
|
||||
map: Mapfile,
|
||||
/// `sync_all`-on-failure-is-an-error iff the output is a regular
|
||||
/// file. `/dev/null` and pipes always fail `sync_all`; that's not
|
||||
/// a real error.
|
||||
is_regular: bool,
|
||||
/// Back-channel for `StatsRequest` responses. The producer caches
|
||||
/// the latest snapshot and uses it for the progress callback;
|
||||
/// dropped sends on a full channel are by design.
|
||||
prog_tx: SyncSender<ProgressSnapshot>,
|
||||
/// Reusable zero buffer for SkipFill / GapFill / BisectBad. Held
|
||||
/// in the sink so each apply call doesn't reallocate.
|
||||
zero: Box<[u8; ZERO_CHUNK]>,
|
||||
}
|
||||
|
||||
impl SweepSink {
|
||||
/// Construct a new `SweepSink` plus the matching progress
|
||||
/// receiver. Channel depth on the back-channel is `1` — the
|
||||
/// producer's cache is the source of truth between snapshots.
|
||||
pub(super) fn new(
|
||||
file: crate::io::WritebackFile,
|
||||
map: Mapfile,
|
||||
is_regular: bool,
|
||||
) -> (Self, Receiver<ProgressSnapshot>) {
|
||||
let (prog_tx, prog_rx) = sync_channel::<ProgressSnapshot>(1);
|
||||
let sink = SweepSink {
|
||||
file,
|
||||
map,
|
||||
is_regular,
|
||||
prog_tx,
|
||||
zero: Box::new([0u8; ZERO_CHUNK]),
|
||||
};
|
||||
(sink, prog_rx)
|
||||
}
|
||||
}
|
||||
|
||||
impl Sink<WorkItem> for SweepSink {
|
||||
type Output = ConsumerSummary;
|
||||
|
||||
fn apply(&mut self, item: WorkItem) -> Result<Flow, Error> {
|
||||
match item {
|
||||
WorkItem::Good { pos, buf } => {
|
||||
// Decrypt is on the producer; consumer assumes plaintext.
|
||||
let len = buf.len() as u64;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&buf)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, len, SectorStatus::Finished)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::BisectGood { pos, buf } => {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&buf[..])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, 2048, SectorStatus::Finished)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::BisectBad { pos } => {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&self.zero[..2048])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, 2048, SectorStatus::NonTrimmed)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::SkipFill { pos, len } | WorkItem::GapFill { pos, len } => {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
// Subsequent writes are sequential; `WritebackFile`'s
|
||||
// seek-elision keeps them on the writeback pipeline path.
|
||||
let mut filled = 0u64;
|
||||
while filled < len {
|
||||
let chunk = (len - filled).min(self.zero.len() as u64) as usize;
|
||||
self.file
|
||||
.write_all(&self.zero[..chunk])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
filled += chunk as u64;
|
||||
}
|
||||
self.map
|
||||
.record(pos, len, SectorStatus::NonTrimmed)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::StatsRequest => {
|
||||
let stats = self.map.stats();
|
||||
let bad_ranges = self.map.ranges_with(&[
|
||||
SectorStatus::NonTrimmed,
|
||||
SectorStatus::Unreadable,
|
||||
SectorStatus::NonScraped,
|
||||
SectorStatus::NonTried,
|
||||
]);
|
||||
// Best-effort: drop on backpressure; producer's cache
|
||||
// stays current enough.
|
||||
let _ = self
|
||||
.prog_tx
|
||||
.try_send(ProgressSnapshot { stats, bad_ranges });
|
||||
}
|
||||
}
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(mut self) -> Result<Self::Output, Error> {
|
||||
// Drain the writeback pipeline + fsync the ISO, then persist
|
||||
// any pending mapfile state. Same finalisation order as the
|
||||
// pre-Pipeline consumer loop.
|
||||
if let Err(e) = self.file.sync_all() {
|
||||
if self.is_regular {
|
||||
return Err(Error::IoError { source: e });
|
||||
}
|
||||
// Non-regular outputs (/dev/null, pipes) always fail
|
||||
// sync_all; that's not a real error.
|
||||
}
|
||||
self.map.flush().map_err(|e| Error::IoError { source: e })?;
|
||||
|
||||
Ok(ConsumerSummary {
|
||||
stats: self.map.stats(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
//! Drive data capture — read hardware information via SCSI.
|
||||
|
||||
use crate::drive::Drive;
|
||||
use crate::error::Result;
|
||||
|
||||
/// Raw data captured from a drive's SCSI responses.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DriveCapture {
|
||||
/// Raw INQUIRY response (96 bytes)
|
||||
pub inquiry: Vec<u8>,
|
||||
/// Raw GET_CONFIG 010C response
|
||||
pub gc_010c: Vec<u8>,
|
||||
/// GET_CONFIG feature responses: (feature_code, feature_name, data)
|
||||
pub features: Vec<CapturedFeature>,
|
||||
/// REPORT_KEY RPC state
|
||||
pub rpc_state: Option<Vec<u8>>,
|
||||
/// MODE SENSE page 2A (capabilities)
|
||||
pub mode_2a: Option<Vec<u8>>,
|
||||
/// READ_BUFFER 0xF1 (Pioneer vendor data)
|
||||
pub rb_f1: Option<Vec<u8>>,
|
||||
/// READ_BUFFER mode 6 (MTK vendor data)
|
||||
pub rb_mode6: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
/// A single GET CONFIGURATION feature response from the drive.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CapturedFeature {
|
||||
pub code: u16,
|
||||
pub name: &'static str,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Feature codes to capture.
|
||||
const FEATURES: &[(u16, &str)] = &[
|
||||
(0x0000, "Profile List"),
|
||||
(0x0001, "Core"),
|
||||
(0x0003, "Removable Medium"),
|
||||
(0x0010, "Random Readable"),
|
||||
(0x001D, "Multi-Read"),
|
||||
(0x001E, "CD Read"),
|
||||
(0x001F, "DVD Read"),
|
||||
(0x0040, "BD Read"),
|
||||
(0x0041, "BD Write"),
|
||||
(0x0100, "Power Management"),
|
||||
(0x0102, "Embedded Changer"),
|
||||
(0x0107, "Real Time Streaming"),
|
||||
(0x0108, "Serial Number"),
|
||||
(0x010C, "Firmware Information"),
|
||||
(0x010D, "AACS"),
|
||||
];
|
||||
|
||||
/// Capture all available drive data via SCSI commands.
|
||||
/// Returns raw responses — no formatting, no zipping, no presentation.
|
||||
pub fn capture_drive_data(session: &mut Drive) -> Result<DriveCapture> {
|
||||
let id = &session.drive_id;
|
||||
|
||||
// Already have INQUIRY from drive open
|
||||
let inquiry = id.raw_inquiry.clone();
|
||||
let gc_010c = id.raw_gc_010c.clone();
|
||||
|
||||
// Capture GET_CONFIG features using Drive's query methods
|
||||
let mut features = Vec::new();
|
||||
for &(code, name) in FEATURES {
|
||||
if let Some(data) = session.get_config_feature(code) {
|
||||
features.push(CapturedFeature { code, name, data });
|
||||
}
|
||||
}
|
||||
|
||||
// Vendor-specific READ_BUFFER queries
|
||||
let rb_f1 = session.read_buffer(0x02, 0xF1, 48); // Pioneer
|
||||
let rb_mode6 = session.read_buffer(0x06, 0x00, 32); // MTK
|
||||
|
||||
// Standard queries
|
||||
let rpc_state = session.report_key_rpc_state();
|
||||
let mode_2a = session.mode_sense_page(0x2A);
|
||||
|
||||
Ok(DriveCapture {
|
||||
inquiry,
|
||||
gc_010c,
|
||||
features,
|
||||
rpc_state,
|
||||
mode_2a,
|
||||
rb_f1,
|
||||
rb_mode6,
|
||||
})
|
||||
}
|
||||
|
||||
/// Mask a string for privacy (letters->A, digits->0).
|
||||
pub fn mask_string(s: &str) -> String {
|
||||
s.chars()
|
||||
.map(|c| {
|
||||
if c.is_ascii_alphabetic() {
|
||||
'A'
|
||||
} else if c.is_ascii_digit() {
|
||||
'0'
|
||||
} else {
|
||||
c
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Mask bytes for privacy.
|
||||
pub fn mask_bytes(data: &[u8]) -> Vec<u8> {
|
||||
data.iter()
|
||||
.map(|&b| {
|
||||
if b.is_ascii_alphabetic() {
|
||||
b'A'
|
||||
} else if b.is_ascii_digit() {
|
||||
b'0'
|
||||
} else {
|
||||
b
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
+5
-7
@@ -6,7 +6,7 @@ use crate::identity::DriveId;
|
||||
pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
let mut drives = Vec::new();
|
||||
for i in 0..16 {
|
||||
let path = format!("/dev/sg{}", i);
|
||||
let path = format!("/dev/sg{i}");
|
||||
if !std::path::Path::new(&path).exists() {
|
||||
continue;
|
||||
}
|
||||
@@ -21,6 +21,7 @@ pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
drives
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn resolve_device(path: &str) -> Result<(String, Option<String>)> {
|
||||
if path.contains("/sg") {
|
||||
if !std::path::Path::new(path).exists() {
|
||||
@@ -39,18 +40,15 @@ pub fn resolve_device(path: &str) -> Result<(String, Option<String>)> {
|
||||
&& sg_id.product_id == sr_id.product_id
|
||||
&& sg_id.serial_number == sr_id.serial_number
|
||||
{
|
||||
let warning = format!(
|
||||
"{} is a block device (sr) — using {} (sg) for raw access",
|
||||
path, sg_path
|
||||
);
|
||||
let warning =
|
||||
format!("{path} is a block device (sr) — using {sg_path} (sg) for raw access");
|
||||
return Ok((sg_path, Some(warning)));
|
||||
}
|
||||
}
|
||||
return Ok((
|
||||
path.to_string(),
|
||||
Some(format!(
|
||||
"{} is a block device (sr) — no matching sg device found",
|
||||
path
|
||||
"{path} is a block device (sr) — no matching sg device found"
|
||||
)),
|
||||
));
|
||||
}
|
||||
|
||||
+10
-9
@@ -1,24 +1,25 @@
|
||||
//! macOS drive discovery and device resolution.
|
||||
//!
|
||||
//! `find_drives` uses IOKit registry enumeration (via `scsi::list_drives`)
|
||||
//! to discover optical drives without exclusive access or unmounts. Only
|
||||
//! the returned paths are then opened for INQUIRY to build full `DriveId`.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::identity::DriveId;
|
||||
|
||||
pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
let mut drives = Vec::new();
|
||||
for i in 0..16 {
|
||||
let path = format!("/dev/disk{}", i);
|
||||
if !std::path::Path::new(&path).exists() { continue; }
|
||||
match crate::scsi::open(std::path::Path::new(&path)) {
|
||||
let discovered = crate::scsi::list_drives();
|
||||
for info in discovered {
|
||||
let path = std::path::Path::new(&info.path);
|
||||
match crate::scsi::open(path) {
|
||||
Ok(mut transport) => {
|
||||
if let Ok(id) = DriveId::from_drive(transport.as_mut()) {
|
||||
if !id.raw_inquiry.is_empty() && (id.raw_inquiry[0] & 0x1F) == 0x05 {
|
||||
drives.push((path, id));
|
||||
}
|
||||
drives.push((info.path.clone(), id));
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
// Device exists but can't be opened (likely mounted).
|
||||
// Use `diskutil unmountDisk /dev/diskN` to unmount before accessing.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+652
-60
@@ -1,37 +1,87 @@
|
||||
//! Drive session — open, identify, and read from optical drives.
|
||||
//!
|
||||
//! Three-step open:
|
||||
//! 1. `open()` — open device, identify drive. Always OEM.
|
||||
//! 2. `wait_ready()` — wait for disc to spin up. Call before reading.
|
||||
//! 3. `init()` — activate custom firmware. Removes riplock.
|
||||
//! 4. `probe_disc()` — probe disc surface. Drive learns optimal speeds.
|
||||
|
||||
pub(crate) fn extract_scsi_context(e: &Error) -> (u8, Option<crate::scsi::ScsiSense>) {
|
||||
match e {
|
||||
Error::ScsiError { status, sense, .. } => (*status, *sense),
|
||||
Error::DiscRead { status, sense, .. } => (status.unwrap_or(0), *sense),
|
||||
_ => (0, None),
|
||||
}
|
||||
}
|
||||
|
||||
pub mod capture;
|
||||
|
||||
// Per-platform discovery helpers (the `pub(crate)` `find_drives` /
|
||||
// equivalents). Crate-public so `scsi/{linux,macos,windows}.rs` can
|
||||
// reuse the existing enumeration logic when shaping `DriveInfo`.
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
pub(crate) mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
pub(crate) mod macos;
|
||||
#[cfg(windows)]
|
||||
mod windows;
|
||||
pub(crate) mod windows;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::event::{Event, EventKind};
|
||||
use crate::identity::DriveId;
|
||||
use crate::platform::mt1959::Mt1959;
|
||||
use crate::platform::PlatformDriver;
|
||||
use crate::platform::mt1959::Mt1959;
|
||||
use crate::profile::{self, DriveProfile};
|
||||
use crate::scsi::ScsiTransport;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
pub struct DriveSession {
|
||||
/// Physical state of the drive tray and disc.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub enum DriveStatus {
|
||||
/// Tray is open
|
||||
TrayOpen,
|
||||
/// Tray closed, no disc
|
||||
NoDisc,
|
||||
/// Tray closed, disc present and ready
|
||||
DiscPresent,
|
||||
/// Drive is loading or spinning up
|
||||
NotReady,
|
||||
/// Could not determine status
|
||||
Unknown,
|
||||
}
|
||||
|
||||
// SCSI opcodes used in drive control
|
||||
const SCSI_TEST_UNIT_READY: u8 = 0x00;
|
||||
const SCSI_START_STOP_UNIT: u8 = 0x1B;
|
||||
const SCSI_PREVENT_ALLOW_MEDIUM_REMOVAL: u8 = 0x1E;
|
||||
const SCSI_GET_EVENT_STATUS: u8 = 0x4A;
|
||||
const SCSI_MODE_SENSE: u8 = 0x5A;
|
||||
const SCSI_REPORT_KEY: u8 = 0xA4;
|
||||
|
||||
/// Optical disc drive session -- open, identify, unlock, and read.
|
||||
pub struct Drive {
|
||||
scsi: Box<dyn ScsiTransport>,
|
||||
driver: Option<Box<dyn PlatformDriver>>,
|
||||
pub profile: Option<DriveProfile>,
|
||||
pub platform: Option<profile::Platform>,
|
||||
pub drive_id: DriveId,
|
||||
device_path: String,
|
||||
/// Halt flag — when set, Drive::read() bails at the next check point.
|
||||
halt: Arc<AtomicBool>,
|
||||
/// Event handler — fires for read errors and library-level state changes.
|
||||
event_fn: Option<Box<dyn Fn(Event) + Send>>,
|
||||
/// Linux only: raw fd for the corresponding block device (`/dev/sr*`)
|
||||
/// used as a recovery fallback when SCSI READ via `/dev/sg*` returns
|
||||
/// an error. The kernel `sr_mod` driver auto-retries failed reads
|
||||
/// (~5× per command) — historically the reason `dd if=/dev/sr0`
|
||||
/// recovers ~50% of bad sectors that single-shot `SG_IO` READ
|
||||
/// misses on the same drive. `None` when the block device couldn't
|
||||
/// be resolved or opened (no fallback in that case; SCSI read
|
||||
/// errors propagate as before).
|
||||
#[cfg(target_os = "linux")]
|
||||
block_dev_fd: Option<std::os::unix::io::RawFd>,
|
||||
}
|
||||
|
||||
impl DriveSession {
|
||||
impl Drive {
|
||||
pub fn open(device: &Path) -> Result<Self> {
|
||||
let mut transport = crate::scsi::open(device)?;
|
||||
let profiles = profile::load_bundled()?;
|
||||
@@ -47,40 +97,176 @@ impl DriveSession {
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
Ok(DriveSession {
|
||||
#[cfg(target_os = "linux")]
|
||||
let block_dev_fd = open_block_device_for_sg(device);
|
||||
|
||||
Ok(Drive {
|
||||
scsi: transport,
|
||||
driver,
|
||||
platform,
|
||||
profile,
|
||||
drive_id,
|
||||
device_path: device.to_string_lossy().to_string(),
|
||||
halt: Arc::new(AtomicBool::new(false)),
|
||||
event_fn: None,
|
||||
#[cfg(target_os = "linux")]
|
||||
block_dev_fd,
|
||||
})
|
||||
}
|
||||
|
||||
/// Get a clone of the halt flag. Set to true to interrupt Drive::read().
|
||||
pub fn halt_flag(&self) -> Arc<AtomicBool> {
|
||||
self.halt.clone()
|
||||
}
|
||||
|
||||
/// Halt the drive — Drive::read() will bail at the next check point.
|
||||
pub fn halt(&self) {
|
||||
self.halt.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Clear the halt flag for the next operation.
|
||||
pub fn clear_halt(&self) {
|
||||
self.halt.store(false, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Set an event handler for read recovery events.
|
||||
pub fn on_event(&mut self, f: impl Fn(Event) + Send + 'static) {
|
||||
self.event_fn = Some(Box::new(f));
|
||||
}
|
||||
|
||||
#[allow(dead_code)] // public on_event registration kept; Drive currently
|
||||
// has no internal emission sites after the 0.13.6 recovery strip.
|
||||
// DiscStream is the BytesRead source. Plan to drop on_event in 0.14.
|
||||
fn emit(&self, kind: EventKind) {
|
||||
if let Some(ref f) = self.event_fn {
|
||||
f(Event { kind });
|
||||
}
|
||||
}
|
||||
|
||||
fn is_halted(&self) -> bool {
|
||||
self.halt.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Halt-aware SCSI execute. Returns `Err(Halted)` if the flag is set
|
||||
/// before the command dispatches or by the time it completes. The only
|
||||
/// path to talk to the drive in the recovery hot loop; keeps Drive::read
|
||||
/// free of explicit halt checks.
|
||||
fn checked_exec(
|
||||
&mut self,
|
||||
cdb: &[u8],
|
||||
dir: crate::scsi::DataDirection,
|
||||
buf: &mut [u8],
|
||||
timeout_ms: u32,
|
||||
) -> Result<crate::scsi::ScsiResult> {
|
||||
if self.is_halted() {
|
||||
return Err(Error::Halted);
|
||||
}
|
||||
let r = self.scsi.as_mut().execute(cdb, dir, buf, timeout_ms)?;
|
||||
if self.is_halted() {
|
||||
return Err(Error::Halted);
|
||||
}
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
/// Close the drive cleanly. Unlocks tray, flushes SCSI state, closes fd.
|
||||
/// Also runs automatically on Drop as a safety net.
|
||||
pub fn close(self) {
|
||||
// cleanup() runs here via Drop
|
||||
}
|
||||
|
||||
/// Shared cleanup — called by Drop (and thus by close).
|
||||
fn cleanup(&mut self) {
|
||||
self.unlock_tray();
|
||||
}
|
||||
|
||||
/// Whether this drive has a known profile (unlock parameters available).
|
||||
pub fn has_profile(&self) -> bool {
|
||||
self.profile.is_some()
|
||||
}
|
||||
|
||||
/// Access the SCSI transport for direct commands (used by CSS/AACS auth).
|
||||
pub fn scsi_mut(&mut self) -> &mut dyn ScsiTransport {
|
||||
self.scsi.as_mut()
|
||||
}
|
||||
|
||||
pub fn wait_ready(&mut self) -> Result<()> {
|
||||
let tur = [0x00, 0x00, 0x00, 0x00, 0x00, 0x00];
|
||||
let tur = [SCSI_TEST_UNIT_READY, 0x00, 0x00, 0x00, 0x00, 0x00];
|
||||
|
||||
for _ in 0..60 {
|
||||
let mut buf = [0u8; 0];
|
||||
if self
|
||||
.scsi
|
||||
.as_mut()
|
||||
.execute(&tur, crate::scsi::DataDirection::None, &mut buf, 5000)
|
||||
.execute(&tur, crate::scsi::DataDirection::None, &mut buf, 5_000)
|
||||
.is_ok()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
}
|
||||
Err(Error::DeviceNotFound {
|
||||
path: format!("{}: drive not ready after 30s", self.device_path),
|
||||
Err(Error::DeviceNotReady {
|
||||
path: self.device_path.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Query the physical state of the drive — disc present, tray open, etc.
|
||||
/// Uses GET EVENT STATUS NOTIFICATION which works regardless of firmware state.
|
||||
pub fn drive_status(&mut self) -> DriveStatus {
|
||||
// GET EVENT STATUS NOTIFICATION: polled, media event class (0x10)
|
||||
let cdb = [
|
||||
SCSI_GET_EVENT_STATUS,
|
||||
0x01,
|
||||
0x00,
|
||||
0x00,
|
||||
0x10,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x08,
|
||||
0x00,
|
||||
];
|
||||
let mut buf = [0u8; 8];
|
||||
match self.scsi.as_mut().execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
) {
|
||||
Ok(r) if r.bytes_transferred >= 6 => {
|
||||
let media_status = buf[5];
|
||||
// Bits 1-0: door/tray state
|
||||
// Bit 1: media present, Bit 0: tray open
|
||||
match media_status & 0x03 {
|
||||
0x00 => DriveStatus::NoDisc, // tray closed, no disc
|
||||
0x01 => DriveStatus::TrayOpen, // tray open
|
||||
0x02 => DriveStatus::DiscPresent, // tray closed, disc present
|
||||
0x03 => DriveStatus::DiscPresent, // tray closed, disc present
|
||||
_ => DriveStatus::Unknown,
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Fallback: try TUR
|
||||
let tur = [SCSI_TEST_UNIT_READY, 0x00, 0x00, 0x00, 0x00, 0x00];
|
||||
let mut empty = [0u8; 0];
|
||||
match self.scsi.as_mut().execute(
|
||||
&tur,
|
||||
crate::scsi::DataDirection::None,
|
||||
&mut empty,
|
||||
5_000,
|
||||
) {
|
||||
Ok(_) => DriveStatus::DiscPresent,
|
||||
Err(ref e)
|
||||
if e.scsi_sense()
|
||||
.is_some_and(|s| s.is_not_ready() || s.is_unit_attention()) =>
|
||||
{
|
||||
DriveStatus::NotReady
|
||||
}
|
||||
_ => DriveStatus::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn platform_name(&self) -> &str {
|
||||
match self.platform {
|
||||
Some(ref p) => p.name(),
|
||||
@@ -119,6 +305,126 @@ impl DriveSession {
|
||||
}
|
||||
}
|
||||
|
||||
/// Query a specific GET CONFIGURATION feature by code.
|
||||
/// Returns the feature data (without the 8-byte header), or None if not available.
|
||||
pub fn get_config_feature(&mut self, feature_code: u16) -> Option<Vec<u8>> {
|
||||
let cdb = [
|
||||
crate::scsi::SCSI_GET_CONFIGURATION,
|
||||
0x02,
|
||||
(feature_code >> 8) as u8,
|
||||
feature_code as u8,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x01,
|
||||
0x00,
|
||||
0x00,
|
||||
];
|
||||
let mut buf = vec![0u8; 256];
|
||||
let r = self
|
||||
.scsi
|
||||
.as_mut()
|
||||
.execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
)
|
||||
.ok()?;
|
||||
if r.bytes_transferred > 8 {
|
||||
Some(buf[8..r.bytes_transferred].to_vec())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Read REPORT KEY RPC state (region playback control).
|
||||
pub fn report_key_rpc_state(&mut self) -> Option<Vec<u8>> {
|
||||
let cdb = [
|
||||
SCSI_REPORT_KEY,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x08,
|
||||
0x08,
|
||||
0x00,
|
||||
];
|
||||
let mut buf = vec![0u8; 8];
|
||||
let r = self
|
||||
.scsi
|
||||
.as_mut()
|
||||
.execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
)
|
||||
.ok()?;
|
||||
if r.bytes_transferred > 0 {
|
||||
Some(buf[..r.bytes_transferred].to_vec())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Read MODE SENSE page data.
|
||||
pub fn mode_sense_page(&mut self, page: u8) -> Option<Vec<u8>> {
|
||||
let cdb = [
|
||||
SCSI_MODE_SENSE,
|
||||
0x00,
|
||||
page,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0xFC,
|
||||
0x00,
|
||||
];
|
||||
let mut buf = vec![0u8; 252];
|
||||
let r = self
|
||||
.scsi
|
||||
.as_mut()
|
||||
.execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
)
|
||||
.ok()?;
|
||||
if r.bytes_transferred > 0 {
|
||||
Some(buf[..r.bytes_transferred].to_vec())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Read vendor-specific READ BUFFER data.
|
||||
pub fn read_buffer(&mut self, mode: u8, buffer_id: u8, length: u16) -> Option<Vec<u8>> {
|
||||
let cdb = crate::scsi::build_read_buffer(mode, buffer_id, 0, length as u32);
|
||||
let mut buf = vec![0u8; length as usize];
|
||||
let r = self
|
||||
.scsi
|
||||
.as_mut()
|
||||
.execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
)
|
||||
.ok()?;
|
||||
if r.bytes_transferred > 0 {
|
||||
Some(buf[..r.bytes_transferred].to_vec())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_ready(&self) -> bool {
|
||||
match self.driver {
|
||||
Some(ref d) => d.is_ready(),
|
||||
@@ -126,7 +432,41 @@ impl DriveSession {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn read_disc(&mut self, lba: u32, count: u16, buf: &mut [u8]) -> Result<usize> {
|
||||
/// Read sectors from the disc. Single-shot — no inline retries, no
|
||||
/// SCSI reset.
|
||||
///
|
||||
/// `recovery=true` uses [`crate::scsi::READ_RECOVERY_TIMEOUT_MS`] (60 s,
|
||||
/// matches sg_dd) for the `Disc::patch` pass; `recovery=false` uses
|
||||
/// [`crate::scsi::READ_TIMEOUT_MS`] (30 s, matches the kernel's
|
||||
/// `/sys/block/sr*/device/timeout` default) for `Disc::copy`'s fast
|
||||
/// skip-forward sweep. Both budgets are generous enough that the drive
|
||||
/// can finish ECC recovery on a marginal sector — pre-0.13.21 this was
|
||||
/// 1.5 s on the fast path which forced the kernel mid-layer to time
|
||||
/// out and escalate while we waited anyway. On any failure returns
|
||||
/// `Err(DiscRead)` immediately; orchestration (`Disc::patch` multi-pass,
|
||||
/// `DiscStream` adaptive batch halving) handles retry policy.
|
||||
///
|
||||
/// Inline retry phases (5× gentle + reset+reopen + 5× more) were
|
||||
/// removed in 0.13.6. Per
|
||||
/// the stop-wedge postmortem (2026-04-25),
|
||||
/// the inline reset on the LG BU40N (Initio bridge) wedged drive
|
||||
/// firmware without ever recovering a sector. The remaining recovery
|
||||
/// layers (Disc::patch multi-pass, DiscStream batch halving) do not
|
||||
/// touch the wedge-prone reset path.
|
||||
pub fn read(&mut self, lba: u32, count: u16, buf: &mut [u8], recovery: bool) -> Result<usize> {
|
||||
let timeout_ms = if recovery {
|
||||
crate::scsi::READ_RECOVERY_TIMEOUT_MS
|
||||
} else {
|
||||
crate::scsi::READ_TIMEOUT_MS
|
||||
};
|
||||
tracing::debug!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
recovery,
|
||||
timeout_ms,
|
||||
"Drive::read enter"
|
||||
);
|
||||
let cdb = [
|
||||
crate::scsi::SCSI_READ_10,
|
||||
0x00,
|
||||
@@ -139,33 +479,105 @@ impl DriveSession {
|
||||
count as u8,
|
||||
0x00,
|
||||
];
|
||||
let result =
|
||||
self.scsi
|
||||
.as_mut()
|
||||
.execute(&cdb, crate::scsi::DataDirection::FromDevice, buf, 5_000)?;
|
||||
Ok(result.bytes_transferred)
|
||||
}
|
||||
|
||||
pub fn read_content(&mut self, lba: u32, count: u16, buf: &mut [u8]) -> Result<usize> {
|
||||
let cdb = [
|
||||
crate::scsi::SCSI_READ_10,
|
||||
0x00,
|
||||
(lba >> 24) as u8,
|
||||
(lba >> 16) as u8,
|
||||
(lba >> 8) as u8,
|
||||
lba as u8,
|
||||
0x00,
|
||||
(count >> 8) as u8,
|
||||
count as u8,
|
||||
0x00,
|
||||
];
|
||||
let result = self.scsi.as_mut().execute(
|
||||
match self.checked_exec(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
buf,
|
||||
30_000,
|
||||
timeout_ms,
|
||||
) {
|
||||
Ok(result) => Ok(result.bytes_transferred),
|
||||
Err(Error::Halted) => Err(Error::Halted),
|
||||
Err(e) => {
|
||||
let (status, sense) = extract_scsi_context(&e);
|
||||
tracing::warn!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
inner_error = %e,
|
||||
scsi_status = status,
|
||||
"Drive::read checked_exec failed"
|
||||
);
|
||||
|
||||
// /dev/sr0 pread fallback (Linux only). The kernel
|
||||
// sr_mod driver auto-retries failed reads (~5× per
|
||||
// command). Empirically (BU40N + Dune Part 2 UHD,
|
||||
// 2026-05-08) dd via /dev/sr0 recovers ~50% of bad
|
||||
// sectors that a single-shot SG_IO READ misses.
|
||||
#[cfg(target_os = "linux")]
|
||||
if recovery {
|
||||
if let Some(fd) = self.block_dev_fd {
|
||||
let len = count as usize * 2048;
|
||||
if buf.len() >= len {
|
||||
let offset = lba as i64 * 2048;
|
||||
// Drop kernel cache for this region so we get
|
||||
// a fresh device read, not stale page-cache
|
||||
// data from a prior successful neighbour read.
|
||||
let _ = unsafe {
|
||||
libc::posix_fadvise(
|
||||
fd,
|
||||
offset,
|
||||
len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
)
|
||||
};
|
||||
let n = unsafe {
|
||||
libc::pread(fd, buf.as_mut_ptr() as *mut libc::c_void, len, offset)
|
||||
};
|
||||
if n == len as isize {
|
||||
tracing::info!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
bytes = len,
|
||||
"Drive::read recovered via /dev/sr0 pread fallback"
|
||||
);
|
||||
return Ok(len);
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
pread_ret = n as i64,
|
||||
errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0),
|
||||
"/dev/sr0 pread fallback also failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::DiscRead {
|
||||
sector: lba as u64,
|
||||
status: Some(status),
|
||||
sense,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the disc capacity in sectors (2048 bytes each).
|
||||
pub fn read_capacity(&mut self) -> Result<u32> {
|
||||
let cdb = [
|
||||
crate::scsi::SCSI_READ_CAPACITY,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
];
|
||||
let mut buf = [0u8; 8];
|
||||
self.scsi.as_mut().execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
)?;
|
||||
Ok(result.bytes_transferred)
|
||||
let last_lba = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]);
|
||||
Ok(last_lba + 1)
|
||||
}
|
||||
|
||||
pub fn set_speed(&mut self, speed_kbs: u16) {
|
||||
@@ -174,16 +586,45 @@ impl DriveSession {
|
||||
let _ = self.scsi_execute(&cdb, crate::scsi::DataDirection::None, &mut dummy, 5_000);
|
||||
}
|
||||
|
||||
/// Lock the tray so the disc cannot be ejected during a rip.
|
||||
pub fn lock_tray(&mut self) {
|
||||
let prevent = [
|
||||
SCSI_PREVENT_ALLOW_MEDIUM_REMOVAL,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x01,
|
||||
0x00,
|
||||
];
|
||||
let mut buf = [0u8; 0];
|
||||
let _ =
|
||||
self.scsi
|
||||
.as_mut()
|
||||
.execute(&prevent, crate::scsi::DataDirection::None, &mut buf, 5_000);
|
||||
}
|
||||
|
||||
/// Unlock the tray so the user can manually eject the disc.
|
||||
pub fn unlock_tray(&mut self) {
|
||||
let allow = [
|
||||
SCSI_PREVENT_ALLOW_MEDIUM_REMOVAL,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
0x00,
|
||||
];
|
||||
let mut buf = [0u8; 0];
|
||||
let _ =
|
||||
self.scsi
|
||||
.as_mut()
|
||||
.execute(&allow, crate::scsi::DataDirection::None, &mut buf, 5_000);
|
||||
}
|
||||
|
||||
/// Eject the disc tray. Unlocks first, then ejects.
|
||||
pub fn eject(&mut self) -> Result<()> {
|
||||
let allow_cdb = [0x1Eu8, 0, 0, 0, 0x00, 0];
|
||||
self.unlock_tray();
|
||||
let eject_cdb = [SCSI_START_STOP_UNIT, 0, 0, 0, 0x02, 0];
|
||||
let mut buf = [0u8; 0];
|
||||
let _ = self.scsi.as_mut().execute(
|
||||
&allow_cdb,
|
||||
crate::scsi::DataDirection::None,
|
||||
&mut buf,
|
||||
5_000,
|
||||
);
|
||||
let eject_cdb = [0x1Bu8, 0, 0, 0, 0x02, 0];
|
||||
self.scsi.as_mut().execute(
|
||||
&eject_cdb,
|
||||
crate::scsi::DataDirection::None,
|
||||
@@ -204,13 +645,118 @@ impl DriveSession {
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for DriveSession {
|
||||
fn read_sectors(&mut self, lba: u32, count: u16, buf: &mut [u8]) -> Result<usize> {
|
||||
self.read_disc(lba, count, buf)
|
||||
impl Drop for Drive {
|
||||
fn drop(&mut self) {
|
||||
self.cleanup();
|
||||
// SgIoTransport::drop() runs next, calling libc::close(fd)
|
||||
#[cfg(target_os = "linux")]
|
||||
if let Some(fd) = self.block_dev_fd.take() {
|
||||
unsafe { libc::close(fd) };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
/// Resolve a `/dev/sg*` path to the corresponding `/dev/sr*` block
|
||||
/// device by walking sysfs, then open it for read (no `O_DIRECT` —
|
||||
/// `posix_fadvise(POSIX_FADV_DONTNEED)` flushes the cache before each
|
||||
/// pread, which avoids buffer-alignment requirements while still
|
||||
/// forcing fresh device reads).
|
||||
///
|
||||
/// Returns `None` on any error (sysfs not present, no matching block
|
||||
/// device, open failed). Callers treat that as "no fallback available"
|
||||
/// and propagate the original SCSI READ error.
|
||||
#[cfg(target_os = "linux")]
|
||||
fn open_block_device_for_sg(sg_path: &Path) -> Option<std::os::unix::io::RawFd> {
|
||||
let basename = sg_path.file_name()?.to_str()?;
|
||||
if !basename.starts_with("sg") {
|
||||
return None;
|
||||
}
|
||||
let sysfs_dir = format!("/sys/class/scsi_generic/{}/device/block", basename);
|
||||
let entries = std::fs::read_dir(&sysfs_dir).ok()?;
|
||||
let block_name = entries
|
||||
.flatten()
|
||||
.find_map(|e| e.file_name().into_string().ok())?;
|
||||
let block_path = format!("/dev/{}", block_name);
|
||||
|
||||
let mut bytes = block_path.as_bytes().to_vec();
|
||||
bytes.push(0);
|
||||
let fd = unsafe {
|
||||
libc::open(
|
||||
bytes.as_ptr() as *const libc::c_char,
|
||||
libc::O_RDONLY | libc::O_CLOEXEC,
|
||||
)
|
||||
};
|
||||
if fd < 0 {
|
||||
tracing::debug!(
|
||||
target: "freemkv::drive",
|
||||
sg = basename,
|
||||
block_path,
|
||||
errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0),
|
||||
"Failed to open block device for fallback; sr0 fallback disabled"
|
||||
);
|
||||
None
|
||||
} else {
|
||||
tracing::info!(
|
||||
target: "freemkv::drive",
|
||||
sg = basename,
|
||||
block_path,
|
||||
fd,
|
||||
"Opened /dev/sr* as recovery fallback for failed SCSI reads"
|
||||
);
|
||||
Some(fd)
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for Drive {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize> {
|
||||
self.read(lba, count, buf, recovery)
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
Drive::set_speed(self, kbs);
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the first optical drive on this system and open it.
|
||||
///
|
||||
/// For just listing drives without opening (e.g. UI sidebar), use
|
||||
/// `scsi::list_drives()` — that returns `DriveInfo` (path + identity)
|
||||
/// without the cost of running every drive's profile + identity probe.
|
||||
pub fn find_drive() -> Option<Drive> {
|
||||
discover_drives()
|
||||
.into_iter()
|
||||
.find_map(|(path, _)| Drive::open(std::path::Path::new(&path)).ok())
|
||||
}
|
||||
|
||||
/// Halt-aware sleep primitive — wakes within ~100 ms of `halt` flipping
|
||||
/// to true. Kept for the unit tests that cover the slicing behaviour;
|
||||
/// production code paths no longer sleep on the recovery hot path
|
||||
/// (recovery loop removed in 0.13.6).
|
||||
#[cfg(test)]
|
||||
fn sleep_until_halted(halt: &AtomicBool, total: std::time::Duration) -> Result<()> {
|
||||
const SLICE: std::time::Duration = std::time::Duration::from_millis(100);
|
||||
let deadline = std::time::Instant::now() + total;
|
||||
loop {
|
||||
if halt.load(Ordering::Relaxed) {
|
||||
return Err(Error::Halted);
|
||||
}
|
||||
let now = std::time::Instant::now();
|
||||
if now >= deadline {
|
||||
return Ok(());
|
||||
}
|
||||
let remaining = deadline - now;
|
||||
std::thread::sleep(remaining.min(SLICE));
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal: discover drive paths + IDs without opening full Drive objects.
|
||||
fn discover_drives() -> Vec<(String, DriveId)> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
linux::find_drives()
|
||||
@@ -225,11 +771,9 @@ pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find_drive() -> Option<String> {
|
||||
find_drives().into_iter().next().map(|(path, _)| path)
|
||||
}
|
||||
|
||||
pub fn resolve_device(path: &str) -> Result<(String, Option<String>)> {
|
||||
/// Resolve a device path to its raw SCSI device, with optional warning message.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn resolve_device(path: &str) -> Result<(String, Option<String>)> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
linux::resolve_device(path)
|
||||
@@ -251,10 +795,58 @@ fn create_driver(
|
||||
match platform {
|
||||
profile::Platform::Mt1959A => Ok(Box::new(Mt1959::new(profile.clone(), false))),
|
||||
profile::Platform::Mt1959B => Ok(Box::new(Mt1959::new(profile.clone(), true))),
|
||||
profile::Platform::Renesas => Err(Error::UnsupportedDrive {
|
||||
vendor_id: profile.identity.vendor_id.trim().to_string(),
|
||||
product_id: String::new(),
|
||||
product_revision: "Renesas not yet implemented".to_string(),
|
||||
profile::Platform::Renesas => Err(Error::PlatformNotImplemented {
|
||||
platform: "renesas".to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod halt_tests {
|
||||
use super::*;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
#[test]
|
||||
fn sleep_until_halted_completes_when_not_halted() {
|
||||
let flag = AtomicBool::new(false);
|
||||
let t0 = Instant::now();
|
||||
let r = sleep_until_halted(&flag, Duration::from_millis(150));
|
||||
assert!(r.is_ok());
|
||||
assert!(t0.elapsed() >= Duration::from_millis(140));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sleep_until_halted_returns_immediately_if_preflagged() {
|
||||
let flag = AtomicBool::new(true);
|
||||
let t0 = Instant::now();
|
||||
let r = sleep_until_halted(&flag, Duration::from_secs(10));
|
||||
assert!(matches!(r, Err(Error::Halted)));
|
||||
// Must wake within one slice (100 ms) — the whole point of the
|
||||
// primitive is that a 30 s sleep doesn't block Stop.
|
||||
assert!(t0.elapsed() < Duration::from_millis(200));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sleep_until_halted_wakes_mid_sleep() {
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let f2 = flag.clone();
|
||||
let t0 = Instant::now();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(150));
|
||||
f2.store(true, Ordering::Relaxed);
|
||||
});
|
||||
let r = sleep_until_halted(&flag, Duration::from_secs(10));
|
||||
assert!(matches!(r, Err(Error::Halted)));
|
||||
let waited = t0.elapsed();
|
||||
// Flag flipped at ~150 ms; we wake within one 100 ms slice → <300 ms.
|
||||
assert!(waited < Duration::from_millis(350), "waited {waited:?}");
|
||||
assert!(waited >= Duration::from_millis(140), "waited {waited:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sleep_until_halted_zero_duration_is_noop_when_not_halted() {
|
||||
let flag = AtomicBool::new(false);
|
||||
let r = sleep_until_halted(&flag, Duration::ZERO);
|
||||
assert!(r.is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
+457
-72
@@ -15,32 +15,47 @@
|
||||
//! | E6xxx | Disc format errors |
|
||||
//! | E7xxx | AACS errors |
|
||||
//! | E8xxx | Keydb errors |
|
||||
//! | E9xxx | Mux errors |
|
||||
//! | E9xxx | Stream/mux errors |
|
||||
|
||||
// ── Error codes ─────────────────────────────────────────────────────────────
|
||||
|
||||
// Device (1xxx)
|
||||
pub const E_DEVICE_NOT_FOUND: u16 = 1000;
|
||||
pub const E_DEVICE_PERMISSION: u16 = 1001;
|
||||
pub const E_DEVICE_NOT_READY: u16 = 1002;
|
||||
pub const E_DEVICE_RESET_FAILED: u16 = 1003;
|
||||
pub const E_SCSI_INTERFACE_UNAVAILABLE: u16 = 1004;
|
||||
pub const E_DEVICE_LOCKED: u16 = 1005;
|
||||
pub const E_IOKIT_PLUGIN_FAILED: u16 = 1006;
|
||||
|
||||
// Profile (2xxx)
|
||||
pub const E_UNSUPPORTED_DRIVE: u16 = 2000;
|
||||
pub const E_PROFILE_NOT_FOUND: u16 = 2001;
|
||||
pub const E_PROFILE_PARSE: u16 = 2002;
|
||||
pub const E_UNSUPPORTED_PLATFORM: u16 = 2003;
|
||||
pub const E_PLATFORM_NOT_IMPLEMENTED: u16 = 2004;
|
||||
|
||||
// Unlock (3xxx)
|
||||
pub const E_UNLOCK_FAILED: u16 = 3000;
|
||||
pub const E_SIGNATURE_MISMATCH: u16 = 3001;
|
||||
pub const E_NOT_UNLOCKED: u16 = 3002;
|
||||
pub const E_NOT_CALIBRATED: u16 = 3003;
|
||||
|
||||
// SCSI (4xxx)
|
||||
pub const E_SCSI_ERROR: u16 = 4000;
|
||||
pub const E_SCSI_TIMEOUT: u16 = 4001;
|
||||
|
||||
// I/O (5xxx)
|
||||
pub const E_IO_ERROR: u16 = 5000;
|
||||
pub const E_WRITE_ERROR: u16 = 5001;
|
||||
|
||||
// Disc format (6xxx)
|
||||
pub const E_DISC_READ: u16 = 6000;
|
||||
pub const E_HALTED: u16 = 6010;
|
||||
pub const E_MPLS_PARSE: u16 = 6001;
|
||||
pub const E_CLPI_PARSE: u16 = 6002;
|
||||
pub const E_UDF_NOT_FOUND: u16 = 6003;
|
||||
pub const E_DISC_NO_TITLES: u16 = 6004;
|
||||
pub const E_DISC_TITLE_RANGE: u16 = 6005;
|
||||
pub const E_DISC_NO_EXTENTS: u16 = 6006;
|
||||
pub const E_IFO_PARSE: u16 = 6007;
|
||||
pub const E_MKV_INVALID: u16 = 6008;
|
||||
pub const E_NO_STREAMS: u16 = 6009;
|
||||
pub const E_MAPFILE_INVALID: u16 = 6011;
|
||||
|
||||
// AACS (7xxx)
|
||||
pub const E_AACS_NO_KEYS: u16 = 7000;
|
||||
pub const E_AACS_CERT_SHORT: u16 = 7001;
|
||||
@@ -54,7 +69,9 @@ pub const E_AACS_KEY_VERIFY: u16 = 7008;
|
||||
pub const E_AACS_VID_READ: u16 = 7009;
|
||||
pub const E_AACS_VID_MAC: u16 = 7010;
|
||||
pub const E_AACS_DATA_KEY: u16 = 7011;
|
||||
pub const E_AACS_VUK_DERIVE: u16 = 7012;
|
||||
pub const E_DECRYPT_FAILED: u16 = 7013;
|
||||
pub const E_CSS_AUTH_FAILED: u16 = 7014;
|
||||
|
||||
// Keydb (8xxx)
|
||||
pub const E_KEYDB_CONNECT: u16 = 8000;
|
||||
pub const E_KEYDB_HTTP: u16 = 8001;
|
||||
@@ -62,79 +79,135 @@ pub const E_KEYDB_INVALID: u16 = 8002;
|
||||
pub const E_KEYDB_WRITE: u16 = 8003;
|
||||
pub const E_KEYDB_PARSE: u16 = 8004;
|
||||
pub const E_KEYDB_LOAD: u16 = 8005;
|
||||
// Mux (9xxx)
|
||||
pub const E_MUX_LOOKAHEAD: u16 = 9000;
|
||||
pub const E_MUX_WRITE: u16 = 9001;
|
||||
|
||||
// Stream/mux (9xxx)
|
||||
pub const E_STREAM_READ_ONLY: u16 = 9000;
|
||||
pub const E_STREAM_WRITE_ONLY: u16 = 9001;
|
||||
pub const E_STREAM_URL_INVALID: u16 = 9002;
|
||||
pub const E_STREAM_URL_MISSING_PATH: u16 = 9003;
|
||||
pub const E_STREAM_URL_MISSING_PORT: u16 = 9004;
|
||||
pub const E_PES_FRAME_TOO_LARGE: u16 = 9005;
|
||||
pub const E_PES_INVALID_MAGIC: u16 = 9006;
|
||||
pub const E_ISO_TOO_LARGE: u16 = 9007;
|
||||
pub const E_NO_METADATA: u16 = 9008;
|
||||
pub const E_DISC_URL_NOT_DIRECT: u16 = 9009;
|
||||
|
||||
// ── Error enum ──────────────────────────────────────────────────────────────
|
||||
|
||||
/// Structured error with numeric code and context data. No English text.
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
// Device
|
||||
// Device (1xxx)
|
||||
DeviceNotFound {
|
||||
path: String,
|
||||
},
|
||||
DevicePermission {
|
||||
path: String,
|
||||
},
|
||||
DeviceNotReady {
|
||||
path: String,
|
||||
},
|
||||
DeviceResetFailed {
|
||||
path: String,
|
||||
},
|
||||
/// Platform-specific SCSI interface couldn't be obtained from the OS
|
||||
/// (macOS: `SCSITaskDeviceInterface` unavailable). The `path` field
|
||||
/// carries the device path; no English commentary on the failure mode.
|
||||
ScsiInterfaceUnavailable {
|
||||
path: String,
|
||||
},
|
||||
/// Device is held by another process / kernel state. `kr` is the
|
||||
/// platform return code (macOS IOReturn, Linux errno-equivalent).
|
||||
DeviceLocked {
|
||||
path: String,
|
||||
kr: u32,
|
||||
},
|
||||
/// macOS IOKit plugin couldn't be created for this device. `kr` is
|
||||
/// the IOReturn code from `IOCreatePlugInInterfaceForService`.
|
||||
IoKitPluginFailed {
|
||||
path: String,
|
||||
kr: u32,
|
||||
},
|
||||
|
||||
// Profile
|
||||
// Profile (2xxx)
|
||||
UnsupportedDrive {
|
||||
vendor_id: String,
|
||||
product_id: String,
|
||||
product_revision: String,
|
||||
},
|
||||
ProfileNotFound {
|
||||
vendor_id: String,
|
||||
product_revision: String,
|
||||
vendor_specific: String,
|
||||
},
|
||||
ProfileParse,
|
||||
/// SCSI transport was requested on an OS without a backend
|
||||
/// implementation. `target` is the `std::env::consts::OS` value.
|
||||
UnsupportedPlatform {
|
||||
target: String,
|
||||
},
|
||||
/// Drive matched a known platform that we haven't implemented yet
|
||||
/// (e.g. Renesas firmware). `platform` is a stable identifier.
|
||||
PlatformNotImplemented {
|
||||
platform: String,
|
||||
},
|
||||
|
||||
// Unlock
|
||||
// Unlock (3xxx)
|
||||
UnlockFailed,
|
||||
SignatureMismatch {
|
||||
expected: [u8; 4],
|
||||
got: [u8; 4],
|
||||
},
|
||||
NotUnlocked,
|
||||
NotCalibrated,
|
||||
|
||||
// SCSI
|
||||
// SCSI (4xxx)
|
||||
/// SCSI command failed.
|
||||
///
|
||||
/// `opcode` is the failing CDB byte 0. `status` is the raw SCSI
|
||||
/// status byte: `0x02` = CHECK CONDITION (drive replied with sense
|
||||
/// data), `0xFF` = libfreemkv-synthesised sentinel meaning "no SCSI
|
||||
/// status delivered" (kernel timeout, USB bridge wedge, IOKit
|
||||
/// service failure). `sense` carries the drive's SPC-4 sense triple
|
||||
/// when the drive replied; `None` for transport-layer failures.
|
||||
///
|
||||
/// Recommended dispatch (callers shouldn't pattern-match raw
|
||||
/// fields):
|
||||
/// - [`Error::is_scsi_transport_failure`] — bail; bridge/transport wedge
|
||||
/// - [`Error::is_marginal_read`] — drive said this read was marginal; smaller block may recover
|
||||
/// - [`Error::scsi_sense`] — borrow the sense triple for finer routing ([`ScsiSense::is_medium_error`] etc.)
|
||||
ScsiError {
|
||||
opcode: u8,
|
||||
status: u8,
|
||||
sense_key: u8,
|
||||
},
|
||||
ScsiTimeout {
|
||||
opcode: u8,
|
||||
sense: Option<crate::scsi::ScsiSense>,
|
||||
},
|
||||
|
||||
// I/O
|
||||
// I/O (5xxx)
|
||||
IoError {
|
||||
source: std::io::Error,
|
||||
},
|
||||
WriteError,
|
||||
|
||||
// Disc format
|
||||
// Disc format (6xxx)
|
||||
DiscRead {
|
||||
sector: u64,
|
||||
status: Option<u8>,
|
||||
sense: Option<crate::scsi::ScsiSense>,
|
||||
},
|
||||
/// Drive was halted by caller.
|
||||
Halted,
|
||||
MplsParse,
|
||||
ClpiParse,
|
||||
UdfNotFound {
|
||||
path: String,
|
||||
},
|
||||
DiscNoTitles,
|
||||
DiscTitleRange {
|
||||
index: usize,
|
||||
count: usize,
|
||||
},
|
||||
DiscNoExtents,
|
||||
IfoParse,
|
||||
MkvInvalid,
|
||||
NoStreams,
|
||||
/// ddrescue mapfile parse failed. `kind` is a stable, language-neutral
|
||||
/// identifier (e.g. `"status_char"`, `"hex"`); not a translatable
|
||||
/// English message.
|
||||
MapfileInvalid {
|
||||
kind: &'static str,
|
||||
},
|
||||
|
||||
// AACS
|
||||
// AACS (7xxx)
|
||||
AacsNoKeys,
|
||||
AacsCertShort,
|
||||
AacsAgidAlloc,
|
||||
@@ -147,9 +220,10 @@ pub enum Error {
|
||||
AacsVidRead,
|
||||
AacsVidMac,
|
||||
AacsDataKey,
|
||||
AacsVukDerive,
|
||||
DecryptFailed,
|
||||
CssAuthFailed,
|
||||
|
||||
// Keydb
|
||||
// Keydb (8xxx)
|
||||
KeydbConnect {
|
||||
host: String,
|
||||
},
|
||||
@@ -165,9 +239,30 @@ pub enum Error {
|
||||
path: String,
|
||||
},
|
||||
|
||||
// Mux
|
||||
MuxLookahead,
|
||||
MuxWrite,
|
||||
// Stream/mux (9xxx)
|
||||
StreamReadOnly,
|
||||
StreamWriteOnly,
|
||||
StreamUrlInvalid {
|
||||
url: String,
|
||||
},
|
||||
StreamUrlMissingPath {
|
||||
scheme: String,
|
||||
},
|
||||
StreamUrlMissingPort {
|
||||
addr: String,
|
||||
},
|
||||
PesFrameTooLarge {
|
||||
size: usize,
|
||||
},
|
||||
PesInvalidMagic,
|
||||
IsoTooLarge {
|
||||
path: String,
|
||||
},
|
||||
NoMetadata,
|
||||
/// `disc://` URLs aren't openable through `input()` — callers must use
|
||||
/// `Drive::open() + Disc::scan() + DiscStream::new()` directly. This
|
||||
/// is a structural API constraint, not a parse failure.
|
||||
DiscUrlNotDirect,
|
||||
}
|
||||
|
||||
impl Error {
|
||||
@@ -175,25 +270,29 @@ impl Error {
|
||||
match self {
|
||||
Error::DeviceNotFound { .. } => E_DEVICE_NOT_FOUND,
|
||||
Error::DevicePermission { .. } => E_DEVICE_PERMISSION,
|
||||
Error::DeviceNotReady { .. } => E_DEVICE_NOT_READY,
|
||||
Error::DeviceResetFailed { .. } => E_DEVICE_RESET_FAILED,
|
||||
Error::ScsiInterfaceUnavailable { .. } => E_SCSI_INTERFACE_UNAVAILABLE,
|
||||
Error::DeviceLocked { .. } => E_DEVICE_LOCKED,
|
||||
Error::IoKitPluginFailed { .. } => E_IOKIT_PLUGIN_FAILED,
|
||||
Error::UnsupportedDrive { .. } => E_UNSUPPORTED_DRIVE,
|
||||
Error::ProfileNotFound { .. } => E_PROFILE_NOT_FOUND,
|
||||
Error::ProfileParse => E_PROFILE_PARSE,
|
||||
Error::UnsupportedPlatform { .. } => E_UNSUPPORTED_PLATFORM,
|
||||
Error::PlatformNotImplemented { .. } => E_PLATFORM_NOT_IMPLEMENTED,
|
||||
Error::UnlockFailed => E_UNLOCK_FAILED,
|
||||
Error::SignatureMismatch { .. } => E_SIGNATURE_MISMATCH,
|
||||
Error::NotUnlocked => E_NOT_UNLOCKED,
|
||||
Error::NotCalibrated => E_NOT_CALIBRATED,
|
||||
Error::ScsiError { .. } => E_SCSI_ERROR,
|
||||
Error::ScsiTimeout { .. } => E_SCSI_TIMEOUT,
|
||||
Error::IoError { .. } => E_IO_ERROR,
|
||||
Error::WriteError => E_WRITE_ERROR,
|
||||
Error::DiscRead { .. } => E_DISC_READ,
|
||||
Error::Halted => E_HALTED,
|
||||
Error::MplsParse => E_MPLS_PARSE,
|
||||
Error::ClpiParse => E_CLPI_PARSE,
|
||||
Error::UdfNotFound { .. } => E_UDF_NOT_FOUND,
|
||||
Error::DiscNoTitles => E_DISC_NO_TITLES,
|
||||
Error::DiscTitleRange { .. } => E_DISC_TITLE_RANGE,
|
||||
Error::DiscNoExtents => E_DISC_NO_EXTENTS,
|
||||
Error::IfoParse => E_IFO_PARSE,
|
||||
Error::MkvInvalid => E_MKV_INVALID,
|
||||
Error::NoStreams => E_NO_STREAMS,
|
||||
Error::MapfileInvalid { .. } => E_MAPFILE_INVALID,
|
||||
Error::AacsNoKeys => E_AACS_NO_KEYS,
|
||||
Error::AacsCertShort => E_AACS_CERT_SHORT,
|
||||
Error::AacsAgidAlloc => E_AACS_AGID_ALLOC,
|
||||
@@ -206,15 +305,24 @@ impl Error {
|
||||
Error::AacsVidRead => E_AACS_VID_READ,
|
||||
Error::AacsVidMac => E_AACS_VID_MAC,
|
||||
Error::AacsDataKey => E_AACS_DATA_KEY,
|
||||
Error::AacsVukDerive => E_AACS_VUK_DERIVE,
|
||||
Error::DecryptFailed => E_DECRYPT_FAILED,
|
||||
Error::CssAuthFailed => E_CSS_AUTH_FAILED,
|
||||
Error::KeydbConnect { .. } => E_KEYDB_CONNECT,
|
||||
Error::KeydbHttp { .. } => E_KEYDB_HTTP,
|
||||
Error::KeydbInvalid => E_KEYDB_INVALID,
|
||||
Error::KeydbWrite { .. } => E_KEYDB_WRITE,
|
||||
Error::KeydbParse => E_KEYDB_PARSE,
|
||||
Error::KeydbLoad { .. } => E_KEYDB_LOAD,
|
||||
Error::MuxLookahead => E_MUX_LOOKAHEAD,
|
||||
Error::MuxWrite => E_MUX_WRITE,
|
||||
Error::StreamReadOnly => E_STREAM_READ_ONLY,
|
||||
Error::StreamWriteOnly => E_STREAM_WRITE_ONLY,
|
||||
Error::StreamUrlInvalid { .. } => E_STREAM_URL_INVALID,
|
||||
Error::StreamUrlMissingPath { .. } => E_STREAM_URL_MISSING_PATH,
|
||||
Error::StreamUrlMissingPort { .. } => E_STREAM_URL_MISSING_PORT,
|
||||
Error::PesFrameTooLarge { .. } => E_PES_FRAME_TOO_LARGE,
|
||||
Error::PesInvalidMagic => E_PES_INVALID_MAGIC,
|
||||
Error::IsoTooLarge { .. } => E_ISO_TOO_LARGE,
|
||||
Error::NoMetadata => E_NO_METADATA,
|
||||
Error::DiscUrlNotDirect => E_DISC_URL_NOT_DIRECT,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -225,6 +333,24 @@ impl std::fmt::Display for Error {
|
||||
match self {
|
||||
Error::DeviceNotFound { path } => write!(f, "E{}: {}", self.code(), path),
|
||||
Error::DevicePermission { path } => write!(f, "E{}: {}", self.code(), path),
|
||||
Error::DeviceNotReady { path } => write!(f, "E{}: {}", self.code(), path),
|
||||
Error::DeviceResetFailed { path } => write!(f, "E{}: {}", self.code(), path),
|
||||
Error::ScsiInterfaceUnavailable { path } => write!(f, "E{}: {}", self.code(), path),
|
||||
Error::DeviceLocked { path, kr } => {
|
||||
write!(f, "E{}: {} 0x{:08x}", self.code(), path, kr)
|
||||
}
|
||||
Error::IoKitPluginFailed { path, kr } => {
|
||||
write!(f, "E{}: {} 0x{:08x}", self.code(), path, kr)
|
||||
}
|
||||
Error::UnsupportedPlatform { target } => {
|
||||
write!(f, "E{}: {}", self.code(), target)
|
||||
}
|
||||
Error::PlatformNotImplemented { platform } => {
|
||||
write!(f, "E{}: {}", self.code(), platform)
|
||||
}
|
||||
Error::MapfileInvalid { kind } => {
|
||||
write!(f, "E{}: {}", self.code(), kind)
|
||||
}
|
||||
Error::UnsupportedDrive {
|
||||
vendor_id,
|
||||
product_id,
|
||||
@@ -237,18 +363,6 @@ impl std::fmt::Display for Error {
|
||||
product_id.trim(),
|
||||
product_revision.trim()
|
||||
),
|
||||
Error::ProfileNotFound {
|
||||
vendor_id,
|
||||
product_revision,
|
||||
vendor_specific,
|
||||
} => write!(
|
||||
f,
|
||||
"E{}: {} {} {}",
|
||||
self.code(),
|
||||
vendor_id.trim(),
|
||||
product_revision.trim(),
|
||||
vendor_specific.trim()
|
||||
),
|
||||
Error::SignatureMismatch { expected, got } => write!(
|
||||
f,
|
||||
"E{}: {:02x}{:02x}{:02x}{:02x}!={:02x}{:02x}{:02x}{:02x}",
|
||||
@@ -265,18 +379,47 @@ impl std::fmt::Display for Error {
|
||||
Error::ScsiError {
|
||||
opcode,
|
||||
status,
|
||||
sense_key,
|
||||
} => write!(
|
||||
f,
|
||||
"E{}: 0x{:02x}/0x{:02x}/0x{:02x}",
|
||||
self.code(),
|
||||
opcode,
|
||||
status,
|
||||
sense_key
|
||||
),
|
||||
Error::ScsiTimeout { opcode } => write!(f, "E{}: 0x{:02x}", self.code(), opcode),
|
||||
sense,
|
||||
} => match sense {
|
||||
Some(s) => write!(
|
||||
f,
|
||||
"E{}: 0x{:02x}/0x{:02x}/0x{:02x}/0x{:02x}/0x{:02x}",
|
||||
self.code(),
|
||||
opcode,
|
||||
status,
|
||||
s.sense_key,
|
||||
s.asc,
|
||||
s.ascq,
|
||||
),
|
||||
None => write!(f, "E{}: 0x{:02x}/0x{:02x}", self.code(), opcode, status,),
|
||||
},
|
||||
Error::IoError { source } => write!(f, "E{}: {}", self.code(), source),
|
||||
Error::DiscRead { sector } => write!(f, "E{}: {}", self.code(), sector),
|
||||
Error::DiscRead {
|
||||
sector,
|
||||
status,
|
||||
sense,
|
||||
} => match (status, sense) {
|
||||
(Some(st), Some(s)) => write!(
|
||||
f,
|
||||
"E{}: {} 0x{:02x}/0x{:02x}/0x{:02x}",
|
||||
self.code(),
|
||||
sector,
|
||||
st,
|
||||
s.sense_key,
|
||||
s.asc,
|
||||
),
|
||||
(Some(st), None) => write!(f, "E{}: {} 0x{:02x}", self.code(), sector, st,),
|
||||
(None, Some(s)) => write!(
|
||||
f,
|
||||
"E{}: {} 0x{:02x}/0x{:02x}",
|
||||
self.code(),
|
||||
sector,
|
||||
s.sense_key,
|
||||
s.asc,
|
||||
),
|
||||
(None, None) => write!(f, "E{}: {}", self.code(), sector),
|
||||
},
|
||||
Error::Halted => write!(f, "E{}", self.code()),
|
||||
Error::UdfNotFound { path } => write!(f, "E{}: {}", self.code(), path),
|
||||
Error::DiscTitleRange { index, count } => {
|
||||
write!(f, "E{}: {}/{}", self.code(), index, count)
|
||||
@@ -285,7 +428,11 @@ impl std::fmt::Display for Error {
|
||||
Error::KeydbHttp { status } => write!(f, "E{}: {}", self.code(), status),
|
||||
Error::KeydbWrite { path } => write!(f, "E{}: {}", self.code(), path),
|
||||
Error::KeydbLoad { path } => write!(f, "E{}: {}", self.code(), path),
|
||||
// Simple codes — no extra data
|
||||
Error::StreamUrlInvalid { url } => write!(f, "E{}: {}", self.code(), url),
|
||||
Error::StreamUrlMissingPath { scheme } => write!(f, "E{}: {}", self.code(), scheme),
|
||||
Error::StreamUrlMissingPort { addr } => write!(f, "E{}: {}", self.code(), addr),
|
||||
Error::PesFrameTooLarge { size } => write!(f, "E{}: {}", self.code(), size),
|
||||
Error::IsoTooLarge { path } => write!(f, "E{}: {}", self.code(), path),
|
||||
_ => write!(f, "E{}", self.code()),
|
||||
}
|
||||
}
|
||||
@@ -306,4 +453,242 @@ impl From<std::io::Error> for Error {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Error> for std::io::Error {
|
||||
fn from(e: Error) -> Self {
|
||||
let code = e.code();
|
||||
let msg = e.to_string();
|
||||
// Map our error categories to io::ErrorKind
|
||||
let kind = match code {
|
||||
1000..=1999 => std::io::ErrorKind::NotFound,
|
||||
2000..=2999 => std::io::ErrorKind::Unsupported,
|
||||
3000..=3999 => std::io::ErrorKind::PermissionDenied,
|
||||
4000..=4999 => std::io::ErrorKind::Other,
|
||||
5000..=5999 => std::io::ErrorKind::Other,
|
||||
6000..=6999 => std::io::ErrorKind::InvalidData,
|
||||
7000..=7999 => std::io::ErrorKind::PermissionDenied,
|
||||
8000..=8999 => std::io::ErrorKind::Other,
|
||||
9000..=9001 => std::io::ErrorKind::Unsupported,
|
||||
9002..=9008 => std::io::ErrorKind::InvalidInput,
|
||||
// 9009 DiscUrlNotDirect: structurally unsupported entry point,
|
||||
// not a parse failure — caller used the wrong API.
|
||||
9009 => std::io::ErrorKind::Unsupported,
|
||||
_ => std::io::ErrorKind::Other,
|
||||
};
|
||||
std::io::Error::new(kind, msg)
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience alias for `Result<T, Error>`.
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
impl Error {
|
||||
/// Borrow the drive-returned SPC-4 sense triple if this error is a
|
||||
/// [`Error::ScsiError`] carrying sense data. `None` for any other
|
||||
/// variant **and** for `ScsiError`s that represent a transport-layer
|
||||
/// failure (where the device never delivered a SCSI status reply, so
|
||||
/// no sense data exists).
|
||||
pub fn scsi_sense(&self) -> Option<&crate::scsi::ScsiSense> {
|
||||
match self {
|
||||
Error::ScsiError { sense: Some(s), .. } => Some(s),
|
||||
Error::DiscRead { sense: Some(s), .. } => Some(s),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// True if this is a [`Error::ScsiError`] representing a transport-layer
|
||||
/// failure — kernel timeout, USB bridge wedge, IOKit service error.
|
||||
/// The device never delivered a SCSI status reply, so there is no
|
||||
/// sense data to inspect; retrying typically requires physical
|
||||
/// intervention (replug).
|
||||
pub fn is_scsi_transport_failure(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Error::ScsiError {
|
||||
status: crate::scsi::SCSI_STATUS_TRANSPORT_FAILURE,
|
||||
..
|
||||
}
|
||||
) || matches!(
|
||||
self,
|
||||
Error::DiscRead {
|
||||
status: Some(crate::scsi::SCSI_STATUS_TRANSPORT_FAILURE),
|
||||
..
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
/// True if this error indicates bridge degradation — the SCSI status
|
||||
/// is neither GOOD (0x00), CHECK CONDITION (0x02), nor transport failure
|
||||
/// (0xFF). Observed on the Initio INIC-1618L USB bridge preceding a full
|
||||
/// crash: the bridge firmware returns non-standard status bytes (e.g.
|
||||
/// 0x04, 0x05) with empty sense data. The caller should cool down
|
||||
/// (10 s pause) and retry rather than hammering the bridge.
|
||||
pub fn is_bridge_degradation(&self) -> bool {
|
||||
let status = match self {
|
||||
Error::ScsiError { status, .. } => *status,
|
||||
Error::DiscRead { status, .. } => status.unwrap_or(0),
|
||||
_ => return false,
|
||||
};
|
||||
status != crate::scsi::SCSI_STATUS_GOOD
|
||||
&& status != crate::scsi::SCSI_STATUS_CHECK_CONDITION
|
||||
&& status != crate::scsi::SCSI_STATUS_TRANSPORT_FAILURE
|
||||
}
|
||||
|
||||
/// True if the underlying SCSI failure is a *marginal read* — the
|
||||
/// drive returned an error category in which smaller-granularity
|
||||
/// retries can sometimes recover the data:
|
||||
///
|
||||
/// - MEDIUM ERROR (sense key 3) — canonical bad-sector signal
|
||||
/// - ABORTED COMMAND (sense key B) — transient; retry usually works
|
||||
/// - RECOVERED ERROR (sense key 1) / NO SENSE (sense key 0) — not
|
||||
/// classified as fatal; treat as recoverable
|
||||
///
|
||||
/// Returns `false` for transport failures (no sense data delivered),
|
||||
/// HARDWARE ERROR, DATA PROTECT, UNIT ATTENTION, ILLEGAL
|
||||
/// REQUEST, BLANK CHECK, kernel `IoError`, and any non-SCSI variant.
|
||||
/// Caller-agnostic predicate — describes a property of the *error*,
|
||||
/// not what one specific call site should do with it. Used by
|
||||
/// `Disc::copy`'s hysteresis dispatch.
|
||||
pub fn is_marginal_read(&self) -> bool {
|
||||
self.scsi_sense()
|
||||
.map(crate::scsi::ScsiSense::is_marginal)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
//! Smoke tests for the error code → variant mapping. Each new variant
|
||||
//! added in 0.13.0 (English-elimination work) gets a code() check + a
|
||||
//! Display sanity-check (no English words) + an io::ErrorKind mapping
|
||||
//! check. Without these, future drift between the const codes and the
|
||||
//! match arms in `code()` / the From impl could silently miscategorize.
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn new_variants_have_distinct_codes() {
|
||||
let codes = [
|
||||
Error::ScsiInterfaceUnavailable { path: "p".into() }.code(),
|
||||
Error::DeviceLocked {
|
||||
path: "p".into(),
|
||||
kr: 0,
|
||||
}
|
||||
.code(),
|
||||
Error::IoKitPluginFailed {
|
||||
path: "p".into(),
|
||||
kr: 0,
|
||||
}
|
||||
.code(),
|
||||
Error::UnsupportedPlatform { target: "x".into() }.code(),
|
||||
Error::PlatformNotImplemented {
|
||||
platform: "renesas".into(),
|
||||
}
|
||||
.code(),
|
||||
Error::MapfileInvalid { kind: "hex" }.code(),
|
||||
Error::DiscUrlNotDirect.code(),
|
||||
];
|
||||
let mut sorted = codes.to_vec();
|
||||
sorted.sort();
|
||||
sorted.dedup();
|
||||
assert_eq!(
|
||||
sorted.len(),
|
||||
codes.len(),
|
||||
"two new variants share a code — check error.rs constants"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn display_emits_no_english_words() {
|
||||
// Every variant's Display must be `E{code}: {data}` — no English.
|
||||
// Sample a few of the new variants and a few existing ones to
|
||||
// catch accidental string-stuffing in future edits.
|
||||
let cases: &[(Error, u16)] = &[
|
||||
(
|
||||
Error::ScsiInterfaceUnavailable {
|
||||
path: "/dev/sg4".into(),
|
||||
},
|
||||
E_SCSI_INTERFACE_UNAVAILABLE,
|
||||
),
|
||||
(
|
||||
Error::DeviceLocked {
|
||||
path: "/dev/sg4".into(),
|
||||
kr: 0xE00002C5,
|
||||
},
|
||||
E_DEVICE_LOCKED,
|
||||
),
|
||||
(
|
||||
Error::UnsupportedPlatform {
|
||||
target: "freebsd".into(),
|
||||
},
|
||||
E_UNSUPPORTED_PLATFORM,
|
||||
),
|
||||
(
|
||||
Error::PlatformNotImplemented {
|
||||
platform: "renesas".into(),
|
||||
},
|
||||
E_PLATFORM_NOT_IMPLEMENTED,
|
||||
),
|
||||
(Error::MapfileInvalid { kind: "hex" }, E_MAPFILE_INVALID),
|
||||
(Error::DiscUrlNotDirect, E_DISC_URL_NOT_DIRECT),
|
||||
];
|
||||
for (e, want_code) in cases {
|
||||
let s = e.to_string();
|
||||
assert!(
|
||||
s.starts_with(&format!("E{}", want_code)),
|
||||
"{:?} display does not lead with code: {}",
|
||||
e,
|
||||
s
|
||||
);
|
||||
// Crude English filter — `Display` should never emit ASCII words
|
||||
// longer than 4 chars (codes/paths/identifiers like `/dev/sg4`,
|
||||
// `renesas`, `freebsd` all pass; "exclusive access denied" would
|
||||
// not).
|
||||
for word in s.split(|c: char| !c.is_ascii_alphabetic()) {
|
||||
assert!(
|
||||
word.len() <= 8
|
||||
|| word.eq_ignore_ascii_case("renesas")
|
||||
|| word.eq_ignore_ascii_case("freebsd"),
|
||||
"Display contains suspicious English-looking word `{word}` in `{s}`"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn iokind_mapping_for_new_variants() {
|
||||
use std::io::ErrorKind;
|
||||
let mapped = |e: Error| -> ErrorKind {
|
||||
let io: std::io::Error = e.into();
|
||||
io.kind()
|
||||
};
|
||||
// 1xxx range → NotFound
|
||||
assert_eq!(
|
||||
mapped(Error::ScsiInterfaceUnavailable { path: "p".into() }),
|
||||
ErrorKind::NotFound
|
||||
);
|
||||
assert_eq!(
|
||||
mapped(Error::DeviceLocked {
|
||||
path: "p".into(),
|
||||
kr: 0
|
||||
}),
|
||||
ErrorKind::NotFound
|
||||
);
|
||||
// 2xxx range → Unsupported
|
||||
assert_eq!(
|
||||
mapped(Error::UnsupportedPlatform { target: "x".into() }),
|
||||
ErrorKind::Unsupported
|
||||
);
|
||||
assert_eq!(
|
||||
mapped(Error::PlatformNotImplemented {
|
||||
platform: "x".into()
|
||||
}),
|
||||
ErrorKind::Unsupported
|
||||
);
|
||||
// 6xxx range → InvalidData
|
||||
assert_eq!(
|
||||
mapped(Error::MapfileInvalid { kind: "hex" }),
|
||||
ErrorKind::InvalidData
|
||||
);
|
||||
// 9009 special-cased to Unsupported
|
||||
assert_eq!(mapped(Error::DiscUrlNotDirect), ErrorKind::Unsupported);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,23 @@ pub struct Event {
|
||||
/// Types of events the lib can fire.
|
||||
#[derive(Debug)]
|
||||
pub enum EventKind {
|
||||
// ── Init sequence events ────────────────────────────────────────
|
||||
/// Drive opened successfully.
|
||||
DriveOpened { device: String },
|
||||
|
||||
/// Drive is ready (disc spun up).
|
||||
DriveReady,
|
||||
|
||||
/// Firmware init completed.
|
||||
InitComplete { success: bool },
|
||||
|
||||
/// Disc probe completed.
|
||||
ProbeComplete { success: bool },
|
||||
|
||||
/// Disc scan completed.
|
||||
ScanComplete { titles: usize },
|
||||
|
||||
// ── Read events ─────────────────────────────────────────────────
|
||||
/// Bytes successfully read and written to output.
|
||||
BytesRead {
|
||||
/// Bytes written so far.
|
||||
@@ -63,6 +80,22 @@ pub enum EventKind {
|
||||
sector_count: u64,
|
||||
},
|
||||
|
||||
/// Sector recovered after a retry (Drive::read multi-phase recovery).
|
||||
SectorRecovered { sector: u64 },
|
||||
|
||||
/// Sector unreadable, zero-filled (skip mode).
|
||||
SectorSkipped { sector: u64 },
|
||||
|
||||
/// Adaptive batch sizer changed the read size.
|
||||
///
|
||||
/// Fires on shrink (read failed at larger size) and on probe-up
|
||||
/// (enough clean reads to try larger again). Consumers use this to
|
||||
/// display a "recovering" state distinct from "ripping normally".
|
||||
BatchSizeChanged {
|
||||
new_size: u16,
|
||||
reason: BatchSizeReason,
|
||||
},
|
||||
|
||||
/// Operation complete.
|
||||
Complete {
|
||||
/// Total bytes written.
|
||||
@@ -72,5 +105,14 @@ pub enum EventKind {
|
||||
},
|
||||
}
|
||||
|
||||
/// Why the adaptive batch sizer changed size.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BatchSizeReason {
|
||||
/// Read failed; sizer halved the batch.
|
||||
Shrunk,
|
||||
/// Clean-read streak threshold hit; sizer doubled toward preferred.
|
||||
Probed,
|
||||
}
|
||||
|
||||
/// A no-op event handler. Ignores all events.
|
||||
pub fn ignore(_event: Event) {}
|
||||
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
//! One-bit cooperative cancellation flag.
|
||||
//!
|
||||
//! `Halt` is a clonable token wrapping `Arc<AtomicBool>`. Pass clones into
|
||||
//! every long-running loop; the loop polls `is_cancelled()` and bails out
|
||||
//! cleanly. Calling `cancel()` from any clone flips the shared flag, and
|
||||
//! every other clone observes it on its next poll.
|
||||
//!
|
||||
//! Why: `Ordering::Relaxed` is sufficient on both load and store because
|
||||
//! this flag is purely advisory — no other memory operations piggyback on
|
||||
//! it for happens-before ordering. Callers that need to publish data
|
||||
//! across threads do so via channels or other synchronization, not via
|
||||
//! this bit.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
/// Clonable, infallible cooperative-cancellation token.
|
||||
///
|
||||
/// Clones share the same underlying flag. `cancel()` is one-way; there is
|
||||
/// no `reset()` by design — construct a fresh `Halt` for a fresh
|
||||
/// operation.
|
||||
///
|
||||
/// Construct with [`Halt::new`] (or [`Halt::default`]). The `Default`
|
||||
/// impl forwards to `new()` — both produce a fresh, uncancelled token.
|
||||
/// The pair exists because clippy's `new_without_default` lint requires
|
||||
/// `Default` whenever a public `new()` is present, even when the two
|
||||
/// would do exactly the same thing.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Halt(Arc<AtomicBool>);
|
||||
|
||||
impl Halt {
|
||||
/// Construct a fresh, uncancelled token.
|
||||
pub fn new() -> Self {
|
||||
Self(Arc::new(AtomicBool::new(false)))
|
||||
}
|
||||
|
||||
/// Wrap an existing `Arc<AtomicBool>` as a `Halt`. Useful as a
|
||||
/// bridge during the 0.18 deprecation window: callers that already
|
||||
/// hold an `Arc<AtomicBool>` (e.g. `Drive::halt_flag()`, the
|
||||
/// deprecated `DiscStream::set_halt`) can adopt the new token API
|
||||
/// without changing the underlying flag.
|
||||
///
|
||||
/// Cancelling either side flips the same bit — the wrapping `Halt`
|
||||
/// and the original `Arc` are two views over one shared flag.
|
||||
pub fn from_arc(flag: Arc<AtomicBool>) -> Self {
|
||||
Self(flag)
|
||||
}
|
||||
|
||||
/// Borrow the underlying `Arc<AtomicBool>`. Used at boundaries with
|
||||
/// pre-`Halt` APIs that still take an `Arc<AtomicBool>` directly
|
||||
/// (`CopyOptions::halt`, the deprecated `DiscStream::set_halt`).
|
||||
/// Round 3 deletes those boundaries and this accessor with them.
|
||||
pub fn as_arc(&self) -> &Arc<AtomicBool> {
|
||||
&self.0
|
||||
}
|
||||
|
||||
/// Flip the shared flag to cancelled. Idempotent.
|
||||
pub fn cancel(&self) {
|
||||
self.0.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Read the shared flag.
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.0.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Halt {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fresh_is_not_cancelled() {
|
||||
let h = Halt::new();
|
||||
assert!(!h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_flips_state() {
|
||||
let h = Halt::new();
|
||||
assert!(!h.is_cancelled());
|
||||
h.cancel();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_is_idempotent() {
|
||||
let h = Halt::new();
|
||||
h.cancel();
|
||||
h.cancel();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state() {
|
||||
let original = Halt::new();
|
||||
let cloned = original.clone();
|
||||
assert!(!original.is_cancelled());
|
||||
assert!(!cloned.is_cancelled());
|
||||
|
||||
// Cancel via the clone; the original observes it.
|
||||
cloned.cancel();
|
||||
assert!(original.is_cancelled());
|
||||
assert!(cloned.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state_reverse_direction() {
|
||||
let original = Halt::new();
|
||||
let cloned = original.clone();
|
||||
|
||||
// Cancel via the original; the clone observes it.
|
||||
original.cancel();
|
||||
assert!(cloned.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state_across_threads() {
|
||||
let h = Halt::new();
|
||||
let h2 = h.clone();
|
||||
let handle = std::thread::spawn(move || {
|
||||
h2.cancel();
|
||||
});
|
||||
handle.join().unwrap();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_arc_shares_state() {
|
||||
// The 0.18 deprecation-window bridge: a Halt built from an
|
||||
// existing Arc<AtomicBool> must be a *view* over the same bit,
|
||||
// not a fresh copy. Cancelling either side flips both.
|
||||
let arc = Arc::new(AtomicBool::new(false));
|
||||
let halt = Halt::from_arc(arc.clone());
|
||||
assert!(!halt.is_cancelled());
|
||||
assert!(!arc.load(Ordering::Relaxed));
|
||||
|
||||
// Cancel via the wrapping Halt; the original Arc observes it.
|
||||
halt.cancel();
|
||||
assert!(arc.load(Ordering::Relaxed));
|
||||
|
||||
// Conversely: flip the Arc directly; the Halt view observes it.
|
||||
let arc2 = Arc::new(AtomicBool::new(false));
|
||||
let halt2 = Halt::from_arc(arc2.clone());
|
||||
arc2.store(true, Ordering::Relaxed);
|
||||
assert!(halt2.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn as_arc_returns_backing_flag() {
|
||||
// `as_arc()` must hand back the *same* Arc, not a clone of a
|
||||
// different bit. Verified by writing through the borrowed Arc
|
||||
// and observing through the Halt.
|
||||
let halt = Halt::new();
|
||||
let arc = halt.as_arc().clone();
|
||||
assert!(!halt.is_cancelled());
|
||||
arc.store(true, Ordering::Relaxed);
|
||||
assert!(halt.is_cancelled());
|
||||
}
|
||||
}
|
||||
+133
-72
@@ -7,8 +7,9 @@
|
||||
//! The parser reads IFO files via UDF and extracts enough information
|
||||
//! to build DiscTitle structs (parallel to MPLS for Blu-ray).
|
||||
|
||||
use crate::disc::{Codec, Resolution};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
// ── Public types ────────────────────────────────────────────────────────────
|
||||
@@ -46,6 +47,8 @@ pub struct DvdTitle {
|
||||
pub duration_secs: f64,
|
||||
/// Cell sector ranges
|
||||
pub cells: Vec<DvdCell>,
|
||||
/// Chapter start times in seconds (derived from program map + cell times)
|
||||
pub chapter_times: Vec<f64>,
|
||||
/// Subtitle palette from PGC: 16 entries of [padding, Y, Cb, Cr].
|
||||
pub palette: Option<Vec<[u8; 4]>>,
|
||||
}
|
||||
@@ -61,8 +64,8 @@ pub struct DvdCell {
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct DvdVideoAttr {
|
||||
pub codec: String,
|
||||
pub resolution: String,
|
||||
pub codec: Codec,
|
||||
pub resolution: Resolution,
|
||||
pub aspect: String,
|
||||
pub standard: String,
|
||||
}
|
||||
@@ -70,7 +73,7 @@ pub struct DvdVideoAttr {
|
||||
/// DVD audio stream attributes.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DvdAudioAttr {
|
||||
pub codec: String,
|
||||
pub codec: Codec,
|
||||
pub channels: u8,
|
||||
pub sample_rate: u32,
|
||||
pub language: String,
|
||||
@@ -181,7 +184,7 @@ fn bcd_byte(b: u8) -> u32 {
|
||||
///
|
||||
/// Reads the VMG (Video Manager) to discover title sets, then reads each
|
||||
/// VTS IFO to extract PGC chains, cell addresses, and stream attributes.
|
||||
pub fn parse_vmg(reader: &mut dyn SectorReader, udf: &UdfFs) -> Result<DvdInfo> {
|
||||
pub fn parse_vmg(reader: &mut dyn SectorSource, udf: &UdfFs) -> Result<DvdInfo> {
|
||||
let vmg_data = udf.read_file(reader, "/VIDEO_TS/VIDEO_TS.IFO")?;
|
||||
|
||||
// Validate VMG magic
|
||||
@@ -260,12 +263,12 @@ pub fn parse_vmg(reader: &mut dyn SectorReader, udf: &UdfFs) -> Result<DvdInfo>
|
||||
///
|
||||
/// `titles_info` is a list of (chapter_count, vts_title_number) from TT_SRPT.
|
||||
fn parse_vts(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf: &UdfFs,
|
||||
vts_number: u8,
|
||||
titles_info: &[(u16, u8)],
|
||||
) -> Result<DvdTitleSet> {
|
||||
let path = format!("/VIDEO_TS/VTS_{:02}_0.IFO", vts_number);
|
||||
let path = format!("/VIDEO_TS/VTS_{vts_number:02}_0.IFO");
|
||||
let vts_data = udf.read_file(reader, &path)?;
|
||||
|
||||
// Validate VTS magic
|
||||
@@ -349,41 +352,15 @@ fn parse_video_attr(data: &[u8]) -> Result<DvdVideoAttr> {
|
||||
_ => "4:3",
|
||||
};
|
||||
|
||||
let resolution = match (b0 >> 4) & 0x03 {
|
||||
0 => {
|
||||
if standard == "PAL" {
|
||||
"720x576"
|
||||
} else {
|
||||
"720x480"
|
||||
}
|
||||
}
|
||||
1 => {
|
||||
if standard == "PAL" {
|
||||
"704x576"
|
||||
} else {
|
||||
"704x480"
|
||||
}
|
||||
}
|
||||
2 => {
|
||||
if standard == "PAL" {
|
||||
"352x576"
|
||||
} else {
|
||||
"352x480"
|
||||
}
|
||||
}
|
||||
3 => {
|
||||
if standard == "PAL" {
|
||||
"352x288"
|
||||
} else {
|
||||
"352x240"
|
||||
}
|
||||
}
|
||||
_ => "720x480",
|
||||
let resolution = if standard == "PAL" {
|
||||
Resolution::R576i
|
||||
} else {
|
||||
Resolution::R480i
|
||||
};
|
||||
|
||||
Ok(DvdVideoAttr {
|
||||
codec: "mpeg2".to_string(),
|
||||
resolution: resolution.to_string(),
|
||||
codec: Codec::Mpeg2,
|
||||
resolution,
|
||||
aspect: aspect.to_string(),
|
||||
standard: standard.to_string(),
|
||||
})
|
||||
@@ -396,12 +373,12 @@ fn parse_audio_attr(data: &[u8], offset: usize) -> Result<DvdAudioAttr> {
|
||||
|
||||
let coding_mode = (b0 >> 5) & 0x07;
|
||||
let codec = match coding_mode {
|
||||
0 => "ac3",
|
||||
2 => "mpeg1",
|
||||
3 => "mpeg2",
|
||||
4 => "lpcm",
|
||||
6 => "dts",
|
||||
_ => "unknown",
|
||||
0 => Codec::Ac3,
|
||||
2 => Codec::Mpeg1,
|
||||
3 => Codec::Mp2,
|
||||
4 => Codec::Lpcm,
|
||||
6 => Codec::Dts,
|
||||
_ => Codec::Unknown(coding_mode),
|
||||
};
|
||||
|
||||
let sample_rate_flag = (b0 >> 3) & 0x03;
|
||||
@@ -434,7 +411,7 @@ fn parse_audio_attr(data: &[u8], offset: usize) -> Result<DvdAudioAttr> {
|
||||
};
|
||||
|
||||
Ok(DvdAudioAttr {
|
||||
codec: codec.to_string(),
|
||||
codec,
|
||||
channels,
|
||||
sample_rate,
|
||||
language,
|
||||
@@ -513,26 +490,20 @@ fn parse_pgcit(
|
||||
|
||||
/// Parse a single PGC (Program Chain) to extract duration and cells.
|
||||
fn parse_pgc(data: &[u8], pgc_offset: usize, chapters: u16) -> Result<DvdTitle> {
|
||||
// PGC needs at least 0xE6 bytes for the cell info offsets
|
||||
if pgc_offset + 0xE6 > data.len() {
|
||||
// PGC needs at least 0xE8 bytes for the cell playback info offset
|
||||
if pgc_offset + 0xEA > data.len() {
|
||||
return Err(Error::IfoParse);
|
||||
}
|
||||
|
||||
// Playback time at offset 2-5 (4 BCD bytes)
|
||||
let time_bytes = sub_slice(data, pgc_offset + 2, 4)?;
|
||||
// PGC layout:
|
||||
// 0x00-0x01: misc flags
|
||||
// 0x02: nr_of_programs
|
||||
// 0x03: nr_of_cells
|
||||
// 0x04-0x07: playback_time (4 BCD bytes)
|
||||
let num_cells = byte_at(data, pgc_offset + 0x03)? as usize;
|
||||
let time_bytes = sub_slice(data, pgc_offset + 0x04, 4)?;
|
||||
let duration_secs = bcd_to_secs(time_bytes);
|
||||
|
||||
// Number of cells: the user spec says byte 0x03, but in the standard IFO
|
||||
// format bytes 0x02-0x05 are the BCD playback time. The real cell count
|
||||
// lives at PGC offset 0x07. We read from 0x03 as primary (per spec) and
|
||||
// fall back to 0x07 if that yields zero.
|
||||
let num_cells_primary = byte_at(data, pgc_offset + 0x03)? as usize;
|
||||
let num_cells = if num_cells_primary == 0 {
|
||||
byte_at(data, pgc_offset + 0x07).unwrap_or(0) as usize
|
||||
} else {
|
||||
num_cells_primary
|
||||
};
|
||||
|
||||
// Cell playback info table offset (relative to PGC start)
|
||||
let cell_playback_offset = be_u16(data, pgc_offset + 0xE8)? as usize;
|
||||
|
||||
@@ -561,9 +532,10 @@ fn parse_pgc(data: &[u8], pgc_offset: usize, chapters: u16) -> Result<DvdTitle>
|
||||
let cell_base = pgc_offset + cell_playback_offset;
|
||||
let mut total = 0.0;
|
||||
for i in 0..cells.len() {
|
||||
// Cell playback info: 24 bytes per cell, BCD time at offset 4-7
|
||||
let co = cell_base + i * 24;
|
||||
if co + 4 <= data.len() {
|
||||
total += bcd_to_secs(&data[co..co + 4]);
|
||||
if co + 8 <= data.len() {
|
||||
total += bcd_to_secs(&data[co + 4..co + 8]);
|
||||
}
|
||||
}
|
||||
total
|
||||
@@ -571,6 +543,39 @@ fn parse_pgc(data: &[u8], pgc_offset: usize, chapters: u16) -> Result<DvdTitle>
|
||||
duration_secs
|
||||
};
|
||||
|
||||
// Extract chapter times from program map + cell durations
|
||||
// PGC program map offset at 0xE6, maps program_number → first cell_number
|
||||
let chapter_times = {
|
||||
let pgm_map_offset = be_u16(data, pgc_offset + 0xE6).unwrap_or(0) as usize;
|
||||
let nr_of_programs = byte_at(data, pgc_offset + 0x02).unwrap_or(0) as usize;
|
||||
let mut times = Vec::new();
|
||||
if pgm_map_offset > 0 && nr_of_programs > 0 && cell_playback_offset > 0 {
|
||||
let pgm_base = pgc_offset + pgm_map_offset;
|
||||
// Collect cell durations
|
||||
let mut cell_durations = Vec::with_capacity(num_cells);
|
||||
let cell_base = pgc_offset + cell_playback_offset;
|
||||
for i in 0..num_cells {
|
||||
let co = cell_base + i * 24;
|
||||
if co + 8 <= data.len() {
|
||||
cell_durations.push(bcd_to_secs(&data[co + 4..co + 8]));
|
||||
} else {
|
||||
cell_durations.push(0.0);
|
||||
}
|
||||
}
|
||||
// Program map: each byte is the first cell number (1-based) for that program
|
||||
for p in 0..nr_of_programs {
|
||||
if pgm_base + p >= data.len() {
|
||||
break;
|
||||
}
|
||||
let first_cell = data[pgm_base + p] as usize;
|
||||
// Chapter time = sum of cell durations before this program's first cell
|
||||
let time: f64 = cell_durations[..first_cell.saturating_sub(1)].iter().sum();
|
||||
times.push(time);
|
||||
}
|
||||
}
|
||||
times
|
||||
};
|
||||
|
||||
// Extract subtitle palette at PGC offset 0xA4: 16 colors × 4 bytes [padding, Y, Cb, Cr]
|
||||
let palette = if pgc_offset + 0xA4 + 64 <= data.len() {
|
||||
let mut colors = Vec::with_capacity(16);
|
||||
@@ -592,6 +597,7 @@ fn parse_pgc(data: &[u8], pgc_offset: usize, chapters: u16) -> Result<DvdTitle>
|
||||
chapters,
|
||||
duration_secs,
|
||||
cells,
|
||||
chapter_times,
|
||||
palette,
|
||||
})
|
||||
}
|
||||
@@ -681,6 +687,7 @@ mod tests {
|
||||
chapters: 5,
|
||||
duration_secs: 3600.0,
|
||||
cells: vec![cell.clone()],
|
||||
chapter_times: Vec::new(),
|
||||
palette: None,
|
||||
};
|
||||
assert_eq!(title.chapters, 5);
|
||||
@@ -688,15 +695,15 @@ mod tests {
|
||||
assert_eq!(title.cells.len(), 1);
|
||||
|
||||
let video = DvdVideoAttr {
|
||||
codec: "mpeg2".to_string(),
|
||||
resolution: "720x480".to_string(),
|
||||
codec: Codec::Mpeg2,
|
||||
resolution: Resolution::R480i,
|
||||
aspect: "16:9".to_string(),
|
||||
standard: "NTSC".to_string(),
|
||||
};
|
||||
assert_eq!(video.codec, "mpeg2");
|
||||
assert_eq!(video.codec, Codec::Mpeg2);
|
||||
|
||||
let audio = DvdAudioAttr {
|
||||
codec: "ac3".to_string(),
|
||||
codec: Codec::Ac3,
|
||||
channels: 6,
|
||||
sample_rate: 48000,
|
||||
language: "en".to_string(),
|
||||
@@ -720,6 +727,60 @@ mod tests {
|
||||
assert_eq!(info.title_sets.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pgc_parses_duration_from_correct_offset() {
|
||||
// Build a minimal PGC: 0xEA bytes minimum
|
||||
// PGC layout: 0x02 = nr_programs, 0x03 = nr_cells, 0x04-0x07 = BCD time
|
||||
let mut pgc = vec![0u8; 0xEA];
|
||||
pgc[0x02] = 1; // 1 program
|
||||
pgc[0x03] = 2; // 2 cells
|
||||
// 1h 59m 30s at 29.97fps, 0 frames
|
||||
pgc[0x04] = 0x01; // hours BCD
|
||||
pgc[0x05] = 0x59; // minutes BCD
|
||||
pgc[0x06] = 0x30; // seconds BCD
|
||||
pgc[0x07] = 0b11_000000; // 29.97fps, 0 frames
|
||||
// Cell playback info offset at PGC+0xE8
|
||||
let cell_offset: u16 = 0xEA; // right after minimum header
|
||||
pgc[0xE8] = (cell_offset >> 8) as u8;
|
||||
pgc[0xE9] = cell_offset as u8;
|
||||
// Add 2 cells (24 bytes each)
|
||||
pgc.resize(pgc.len() + 48, 0);
|
||||
// Cell 0: sectors 100-200
|
||||
let co = 0xEA;
|
||||
pgc[co + 8] = 0;
|
||||
pgc[co + 9] = 0;
|
||||
pgc[co + 10] = 0;
|
||||
pgc[co + 11] = 100; // first sector
|
||||
pgc[co + 20] = 0;
|
||||
pgc[co + 21] = 0;
|
||||
pgc[co + 22] = 0;
|
||||
pgc[co + 23] = 200; // last sector
|
||||
// Cell 1: sectors 300-400
|
||||
let co = 0xEA + 24;
|
||||
pgc[co + 8] = 0;
|
||||
pgc[co + 9] = 0;
|
||||
pgc[co + 10] = 1;
|
||||
pgc[co + 11] = 44; // first sector = 300
|
||||
pgc[co + 20] = 0;
|
||||
pgc[co + 21] = 0;
|
||||
pgc[co + 22] = 1;
|
||||
pgc[co + 23] = 144; // last sector = 400
|
||||
|
||||
let title = parse_pgc(&pgc, 0, 5).unwrap();
|
||||
let expected = 1.0 * 3600.0 + 59.0 * 60.0 + 30.0;
|
||||
assert!(
|
||||
(title.duration_secs - expected).abs() < 0.1,
|
||||
"expected ~{expected}s, got {}s",
|
||||
title.duration_secs
|
||||
);
|
||||
assert_eq!(title.chapters, 5);
|
||||
assert_eq!(title.cells.len(), 2);
|
||||
assert_eq!(title.cells[0].first_sector, 100);
|
||||
assert_eq!(title.cells[0].last_sector, 200);
|
||||
assert_eq!(title.cells[1].first_sector, 300);
|
||||
assert_eq!(title.cells[1].last_sector, 400);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_attr_parsing() {
|
||||
// Build minimal data with video attrs at 0x200
|
||||
@@ -730,8 +791,8 @@ mod tests {
|
||||
let attr = parse_video_attr(&data).unwrap();
|
||||
assert_eq!(attr.standard, "NTSC");
|
||||
assert_eq!(attr.aspect, "16:9");
|
||||
assert_eq!(attr.resolution, "720x480");
|
||||
assert_eq!(attr.codec, "mpeg2");
|
||||
assert_eq!(attr.resolution, Resolution::R480i);
|
||||
assert_eq!(attr.codec, Codec::Mpeg2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -743,7 +804,7 @@ mod tests {
|
||||
let attr = parse_video_attr(&data).unwrap();
|
||||
assert_eq!(attr.standard, "PAL");
|
||||
assert_eq!(attr.aspect, "4:3");
|
||||
assert_eq!(attr.resolution, "720x576");
|
||||
assert_eq!(attr.resolution, Resolution::R576i);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -759,7 +820,7 @@ mod tests {
|
||||
data[3] = b'n';
|
||||
|
||||
let attr = parse_audio_attr(&data, 0).unwrap();
|
||||
assert_eq!(attr.codec, "ac3");
|
||||
assert_eq!(attr.codec, Codec::Ac3);
|
||||
assert_eq!(attr.sample_rate, 48000);
|
||||
assert_eq!(attr.channels, 6);
|
||||
assert_eq!(attr.language, "en");
|
||||
@@ -777,7 +838,7 @@ mod tests {
|
||||
data[3] = b'r';
|
||||
|
||||
let attr = parse_audio_attr(&data, 0).unwrap();
|
||||
assert_eq!(attr.codec, "dts");
|
||||
assert_eq!(attr.codec, Codec::Dts);
|
||||
assert_eq!(attr.sample_rate, 96000);
|
||||
assert_eq!(attr.channels, 2);
|
||||
assert_eq!(attr.language, "fr");
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
//! Bounded-syscall primitive: run a (potentially-blocking) operation
|
||||
//! on a worker thread, with a hard wall-clock deadline and an optional
|
||||
//! cooperative [`Halt`] poll. The calling thread is never trapped
|
||||
//! inside a kernel call.
|
||||
//!
|
||||
//! ## Why this exists
|
||||
//!
|
||||
//! [`crate::halt::Halt`] is cooperative: callers poll
|
||||
//! `is_cancelled()`. It cannot reach inside a syscall the kernel
|
||||
//! currently owns the thread for — `libc::sync_file_range`,
|
||||
//! `libc::fsync`, `File::write` on NFS, and so on. `/api/stop` from
|
||||
//! autorip therefore can't unstick a thread sitting in such a syscall.
|
||||
//!
|
||||
//! [`bounded_syscall`] is the escape hatch: it runs `op` on a fresh
|
||||
//! worker thread, then `recv_timeout`s on a rendezvous channel for the
|
||||
//! result. The wait is broken into ~250 ms slices so the calling
|
||||
//! thread can poll the supplied [`Halt`] in between. If the deadline
|
||||
//! elapses or the halt fires, the worker is intentionally leaked — the
|
||||
//! syscall will unwind whenever the kernel decides, or at process
|
||||
//! exit, but the caller is free to fall back to a degraded code path
|
||||
//! (skip the sync, log loudly, etc.).
|
||||
//!
|
||||
//! ## Trade-offs
|
||||
//!
|
||||
//! - **Thread per call.** Cheap (`std::thread::spawn` is < 100 µs on
|
||||
//! Linux/macOS), but not free. Use on coarse-grained finalisation
|
||||
//! syscalls (`sync_all`, `sync_file_range(WAIT_AFTER)`), not on hot
|
||||
//! inner-loop writes.
|
||||
//! - **Leak on timeout.** A wedged syscall keeps a kernel slot and a
|
||||
//! user-space thread around for the rest of the process's life.
|
||||
//! Bounded by the number of independent rip/mux sessions, which is
|
||||
//! one per disc. The alternative — trapping the caller forever —
|
||||
//! defeats the entire purpose of `/api/stop`.
|
||||
//! - **Halt granularity ~250 ms.** Halt observation is not instant;
|
||||
//! it's the worst-case latency of the `recv_timeout` slice. Good
|
||||
//! enough for human-driven stop requests; not suitable for hard
|
||||
//! real-time deadlines.
|
||||
//!
|
||||
//! ## Single source of truth
|
||||
//!
|
||||
//! Do NOT inline this pattern. Every blocking-syscall wrapper in the
|
||||
//! rip + mux pipeline calls this helper, so changes (e.g. swapping the
|
||||
//! channel impl, adjusting the poll slice, adding metrics) land in one
|
||||
//! place.
|
||||
//!
|
||||
//! ## Platform
|
||||
//!
|
||||
//! Pure `std::thread` + `std::sync::mpsc`. No `cfg(target_os)` needed
|
||||
//! here — the helper itself is platform-agnostic. Callers that wrap
|
||||
//! Linux-only syscalls (`sync_file_range`) still need their own
|
||||
//! `#[cfg(target_os = "linux")]` gates; this helper does not.
|
||||
|
||||
use std::sync::mpsc::{RecvTimeoutError, sync_channel};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::halt::Halt;
|
||||
|
||||
/// Granularity of the halt poll. The receive loop wakes every
|
||||
/// [`POLL_INTERVAL`] to (a) check the [`Halt`] token, then (b) check
|
||||
/// the overall deadline, then go back to waiting. 250 ms is a
|
||||
/// pragmatic balance: short enough that human-driven `/api/stop` feels
|
||||
/// responsive (< 0.5 s p99), long enough that the polling overhead is
|
||||
/// negligible against multi-second syscalls.
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(250);
|
||||
|
||||
/// Failure outcome from a bounded syscall wrapper.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum BoundedError {
|
||||
/// The user-visible halt token fired during the wait. The worker
|
||||
/// thread is intentionally leaked — the caller should fall back to
|
||||
/// a degraded code path rather than waiting on the syscall to
|
||||
/// return.
|
||||
Halted,
|
||||
/// The deadline elapsed before the syscall returned. Same leak
|
||||
/// semantics as `Halted`.
|
||||
Timeout,
|
||||
/// The worker thread panicked, or its sender disconnected before
|
||||
/// sending a result. Treat as a benign no-op (callers usually
|
||||
/// log and continue) rather than a hard error — by definition no
|
||||
/// syscall observably ran to completion in this case.
|
||||
WorkerLost,
|
||||
}
|
||||
|
||||
/// Run a (potentially-blocking) operation on a worker thread with a
|
||||
/// deadline and an optional cooperative halt-token poll. Returns the
|
||||
/// operation's result if it completes within `timeout`; otherwise one
|
||||
/// of [`BoundedError::Halted`] / [`BoundedError::Timeout`] /
|
||||
/// [`BoundedError::WorkerLost`].
|
||||
///
|
||||
/// On `Halted` / `Timeout` the worker thread is intentionally leaked:
|
||||
/// the syscall will unwind whenever the kernel decides, or when the
|
||||
/// process exits. The calling thread is never trapped inside a kernel
|
||||
/// call.
|
||||
///
|
||||
/// `halt` is polled at [`POLL_INTERVAL`] granularity. Pass `None` for
|
||||
/// callers that don't (yet) have a halt token plumbed through —
|
||||
/// behaviour degrades to deadline-only, matching the 0.20.5
|
||||
/// `wait_after_with_timeout` shape this helper generalises.
|
||||
///
|
||||
/// `op` returns `R: Send + 'static`. The closure must own everything
|
||||
/// it touches because it may outlive this call (timeout / halt cases).
|
||||
pub(crate) fn bounded_syscall<F, R>(
|
||||
halt: Option<&Halt>,
|
||||
timeout: Duration,
|
||||
op: F,
|
||||
) -> Result<R, BoundedError>
|
||||
where
|
||||
F: FnOnce() -> R + Send + 'static,
|
||||
R: Send + 'static,
|
||||
{
|
||||
// Rendezvous channel: the worker sends exactly one value (the
|
||||
// op's return) and then exits. Capacity-0 means the send blocks
|
||||
// until we receive — fine on the happy path; on the timeout /
|
||||
// halt path the receiver is dropped and the worker's send
|
||||
// returns Err, which the worker ignores.
|
||||
let (tx, rx) = sync_channel::<R>(0);
|
||||
let _ = thread::Builder::new()
|
||||
.name("freemkv-bounded-syscall".into())
|
||||
.spawn(move || {
|
||||
// Ignore the send error: if we time out (or get halted)
|
||||
// before the worker finishes, the receiver is dropped
|
||||
// and `tx.send` returns Err. Either way, the worker has
|
||||
// nothing more to do.
|
||||
let _ = tx.send(op());
|
||||
});
|
||||
|
||||
let deadline = Instant::now() + timeout;
|
||||
loop {
|
||||
let now = Instant::now();
|
||||
let remaining = deadline.saturating_duration_since(now);
|
||||
let slice = remaining.min(POLL_INTERVAL);
|
||||
match rx.recv_timeout(slice) {
|
||||
Ok(v) => return Ok(v),
|
||||
Err(RecvTimeoutError::Timeout) => {
|
||||
if let Some(h) = halt {
|
||||
if h.is_cancelled() {
|
||||
return Err(BoundedError::Halted);
|
||||
}
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(BoundedError::Timeout);
|
||||
}
|
||||
// Otherwise: another slice.
|
||||
}
|
||||
Err(RecvTimeoutError::Disconnected) => {
|
||||
// Worker thread spawn failed, or it panicked before
|
||||
// sending. Caller treats this as "no syscall ran" —
|
||||
// typically a no-op + log.
|
||||
return Err(BoundedError::WorkerLost);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
#[test]
|
||||
fn op_completes_quickly() {
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), || 42u32);
|
||||
assert!(matches!(r, Ok(42)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn op_exceeds_timeout() {
|
||||
// Op sleeps longer than the deadline → Timeout.
|
||||
let r = bounded_syscall(None, Duration::from_millis(300), || {
|
||||
thread::sleep(Duration::from_secs(2));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Timeout)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_fires_during_wait() {
|
||||
let halt = Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
// Flip the halt from a side thread after ~300 ms — long
|
||||
// enough that the receive loop has rolled at least one
|
||||
// 250 ms slice and is sitting in `recv_timeout` again when
|
||||
// the bit flips.
|
||||
thread::spawn(move || {
|
||||
thread::sleep(Duration::from_millis(300));
|
||||
halt2.cancel();
|
||||
});
|
||||
let r = bounded_syscall(Some(&halt), Duration::from_secs(5), || {
|
||||
thread::sleep(Duration::from_secs(5));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Halted)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_panics() {
|
||||
// Worker panics → sender drops without sending → recv sees
|
||||
// Disconnected → WorkerLost. We use an explicit panic in the
|
||||
// op closure rather than `panic!()` from inside the channel
|
||||
// machinery; the spawned thread's panic is contained (no
|
||||
// process abort) because we don't `.join()` it.
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), || -> u32 {
|
||||
panic!("intentional test panic");
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::WorkerLost)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_already_set_before_call_still_returns_halted() {
|
||||
// Halt observed on the very first poll slice. The op blocks
|
||||
// forever; we must not wait the full timeout to notice the
|
||||
// halt is already set.
|
||||
let halt = Halt::new();
|
||||
halt.cancel();
|
||||
let started = Instant::now();
|
||||
let r = bounded_syscall(Some(&halt), Duration::from_secs(10), || {
|
||||
thread::sleep(Duration::from_secs(10));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Halted)));
|
||||
// Should bail out within ~1 s; allow 2 s of slack for slow
|
||||
// CI hosts.
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(2),
|
||||
"halt-already-set took {:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ok_path_takes_no_halt_token() {
|
||||
// Sanity: the `None` halt path is the documented zero-config
|
||||
// form (matches the 0.20.5 `wait_after_with_timeout`
|
||||
// behaviour). Op returns immediately; we must observe Ok.
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let f2 = flag.clone();
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), move || {
|
||||
f2.store(true, Ordering::Relaxed);
|
||||
"ok"
|
||||
});
|
||||
assert!(matches!(r, Ok("ok")));
|
||||
assert!(flag.load(Ordering::Relaxed));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
//! Byte-sized bounded producer/consumer channel.
|
||||
//!
|
||||
//! Wraps `std::sync::mpsc::sync_channel` with a byte-accounting
|
||||
//! `Mutex<usize> + Condvar` cap. Sender blocks (cooperatively) when
|
||||
//! `used_bytes + item.byte_size() > capacity_bytes`. Receiver
|
||||
//! decrements `used_bytes` when it takes the item.
|
||||
//!
|
||||
//! Why: the existing producer→consumer channel between `DiscStream`
|
||||
//! (PES producer) and `MuxSink` (PES consumer) is bounded by frame
|
||||
//! count. Frame sizes vary 100× between metadata and keyframes, so a
|
||||
//! count-based cap either starves on small frames or buffers far too
|
||||
//! much memory on big ones. Byte-sized accounting sizes the buffer for
|
||||
//! the worst-case input stall (NFS read p99 ≈ 1–2 s × ~15 MB/s peak
|
||||
//! compressed bitrate ≈ ~30 MB) directly.
|
||||
//!
|
||||
//! The underlying mpsc channel is created with a very large slot count
|
||||
//! so the byte cap (not the slot count) is the real backpressure. Slot
|
||||
//! count is only there to give the kernel a small chunk to wake on.
|
||||
//!
|
||||
//! See `(internal)/memory/project_buffering_architecture.md` §
|
||||
//! Pipeline channel — sizing.
|
||||
|
||||
use std::sync::mpsc::{Receiver as MpscReceiver, RecvError, SendError, SyncSender, sync_channel};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
|
||||
/// Default byte cap for the muxer's input channel. Sized to hide a
|
||||
/// worst-case ~2 s NFS read refill at UHD peak compressed bitrate
|
||||
/// (~15 MB/s); 64 MiB gives headroom. Tweakable; not magic.
|
||||
pub const BYTE_CHANNEL_DEFAULT_CAPACITY: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// Slot capacity of the inner `sync_channel`. Large so the byte cap is
|
||||
/// the real backpressure mechanism — the mpsc slot count only exists
|
||||
/// to give the kernel a chunk to wake on. PES frames are typically
|
||||
/// ~700 B each, so 64 MiB ≈ 90 k frames; 200 k is comfortable headroom.
|
||||
const INNER_SLOT_CAPACITY: usize = 200_000;
|
||||
|
||||
/// Anything whose in-memory cost can be accounted by a single
|
||||
/// `usize`. Implement on the item type sent through [`Sender`].
|
||||
pub trait HasByteSize {
|
||||
/// Bytes this item contributes to the channel's used budget.
|
||||
/// Must be > 0 to make progress (a 0-byte item would never
|
||||
/// block the sender no matter the cap; see send_blocks_at_capacity
|
||||
/// test).
|
||||
fn byte_size(&self) -> usize;
|
||||
}
|
||||
|
||||
impl HasByteSize for crate::pes::PesFrame {
|
||||
fn byte_size(&self) -> usize {
|
||||
// Frame data + the fixed header overhead the serializer
|
||||
// writes (track + pts + keyframe + len). The `Vec<u8>` heap
|
||||
// allocation also has alloc-header overhead but that's
|
||||
// <0.1 % at typical frame sizes — folding it in would just
|
||||
// add noise to the budget.
|
||||
self.data.len() + 14
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared book-keeping between [`Sender`] and [`Receiver`]. Wrapped in
|
||||
/// an `Arc` because both halves hold it independently.
|
||||
struct Accounting {
|
||||
used: Mutex<usize>,
|
||||
cv: Condvar,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
/// Send half of the byte-bounded channel.
|
||||
///
|
||||
/// `send` blocks (on a `Condvar`) when adding the item would push
|
||||
/// `used_bytes` past `capacity_bytes`. Unblocks when the receiver
|
||||
/// `recv`s items out and notifies. Returns `Err(item)` if the
|
||||
/// receiver has been dropped — mirrors `mpsc::SyncSender::send`.
|
||||
pub struct Sender<T: HasByteSize> {
|
||||
tx: SyncSender<T>,
|
||||
acct: Arc<Accounting>,
|
||||
}
|
||||
|
||||
impl<T: HasByteSize> Clone for Sender<T> {
|
||||
fn clone(&self) -> Self {
|
||||
Sender {
|
||||
tx: self.tx.clone(),
|
||||
acct: self.acct.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: HasByteSize> Sender<T> {
|
||||
/// Push one item. Blocks until adding it would not exceed the
|
||||
/// capacity, then sends through the inner mpsc channel.
|
||||
pub fn send(&self, item: T) -> Result<(), SendError<T>> {
|
||||
let sz = item.byte_size();
|
||||
// Reserve capacity first. The reservation is observable to
|
||||
// other senders via `used`; only after we win the slot do we
|
||||
// hand the item to the inner mpsc channel. That ordering means
|
||||
// `used` is always a conservative upper bound on what's in the
|
||||
// mpsc queue + about-to-be-sent.
|
||||
{
|
||||
let mut used = self.acct.used.lock().expect("byte_channel poisoned");
|
||||
// An item bigger than the whole capacity will never fit; let
|
||||
// it through anyway as a one-shot reservation, otherwise the
|
||||
// sender deadlocks forever waiting for `used == 0` AND
|
||||
// nothing in flight. The receiver will drain it on the
|
||||
// other side. Same behaviour as `std::sync::mpsc` for
|
||||
// arbitrarily large messages.
|
||||
while *used + sz > self.acct.capacity && *used > 0 {
|
||||
used = self.acct.cv.wait(used).expect("byte_channel cv poisoned");
|
||||
}
|
||||
*used += sz;
|
||||
}
|
||||
match self.tx.send(item) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(SendError(returned)) => {
|
||||
// Receiver dropped — refund the reservation so a later
|
||||
// sender on a clone doesn't observe phantom used bytes
|
||||
// (the receiver is gone so nobody will decrement).
|
||||
let mut used = self.acct.used.lock().expect("byte_channel poisoned");
|
||||
*used = used.saturating_sub(sz);
|
||||
self.acct.cv.notify_all();
|
||||
Err(SendError(returned))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Receive half of the byte-bounded channel.
|
||||
///
|
||||
/// `recv` blocks on the inner mpsc until an item is available, then
|
||||
/// decrements the byte-accounting and wakes any sender waiting on
|
||||
/// capacity.
|
||||
pub struct Receiver<T: HasByteSize> {
|
||||
rx: MpscReceiver<T>,
|
||||
acct: Arc<Accounting>,
|
||||
}
|
||||
|
||||
impl<T: HasByteSize> Receiver<T> {
|
||||
/// Take the next item. Returns `Err(RecvError)` when all senders
|
||||
/// have been dropped and the channel is empty.
|
||||
pub fn recv(&self) -> Result<T, RecvError> {
|
||||
let item = self.rx.recv()?;
|
||||
let sz = item.byte_size();
|
||||
let mut used = self.acct.used.lock().expect("byte_channel poisoned");
|
||||
*used = used.saturating_sub(sz);
|
||||
// Notify all so multi-sender setups wake every blocked sender,
|
||||
// not just one. Wasted wakeups are cheap; missed wakeups would
|
||||
// be a deadlock.
|
||||
self.acct.cv.notify_all();
|
||||
Ok(item)
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a byte-bounded channel with the given capacity in bytes.
|
||||
/// Returns a `(Sender, Receiver)` pair; clone the `Sender` for
|
||||
/// multi-producer setups.
|
||||
pub fn channel<T: HasByteSize>(capacity_bytes: usize) -> (Sender<T>, Receiver<T>) {
|
||||
let (tx, rx) = sync_channel::<T>(INNER_SLOT_CAPACITY);
|
||||
let acct = Arc::new(Accounting {
|
||||
used: Mutex::new(0),
|
||||
cv: Condvar::new(),
|
||||
capacity: capacity_bytes,
|
||||
});
|
||||
(
|
||||
Sender {
|
||||
tx,
|
||||
acct: acct.clone(),
|
||||
},
|
||||
Receiver { rx, acct },
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Test payload — its `byte_size` returns whatever we passed at
|
||||
/// construction so capacity math is exact and predictable.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
struct Item {
|
||||
sz: usize,
|
||||
tag: u32,
|
||||
}
|
||||
|
||||
impl HasByteSize for Item {
|
||||
fn byte_size(&self) -> usize {
|
||||
self.sz
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_recv_round_trip() {
|
||||
let (tx, rx) = channel::<Item>(1024);
|
||||
for i in 0..5 {
|
||||
tx.send(Item { sz: 100, tag: i }).unwrap();
|
||||
}
|
||||
for i in 0..5 {
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got, Item { sz: 100, tag: i });
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn byte_accounting_decrements_on_recv() {
|
||||
// Internal book-keeping check via observable side-effect: after
|
||||
// sending K items totalling N bytes and receiving them all, a
|
||||
// subsequent send of an N-byte item must NOT block (no items
|
||||
// in flight, all capacity refunded).
|
||||
let (tx, rx) = channel::<Item>(1024);
|
||||
for _ in 0..4 {
|
||||
tx.send(Item { sz: 256, tag: 0 }).unwrap();
|
||||
}
|
||||
for _ in 0..4 {
|
||||
rx.recv().unwrap();
|
||||
}
|
||||
// Cap is now fully available again. Send a 1024-byte item; the
|
||||
// `used > 0` guard means it goes through alone (no wait).
|
||||
let start = Instant::now();
|
||||
tx.send(Item { sz: 1024, tag: 99 }).unwrap();
|
||||
assert!(start.elapsed() < Duration::from_millis(100));
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got.tag, 99);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_blocks_at_capacity_unblocks_on_recv() {
|
||||
// Cap = 200 bytes, item = 100 bytes. First two sends fit
|
||||
// exactly; the third must block until a recv frees capacity.
|
||||
let (tx, rx) = channel::<Item>(200);
|
||||
tx.send(Item { sz: 100, tag: 0 }).unwrap();
|
||||
tx.send(Item { sz: 100, tag: 1 }).unwrap();
|
||||
|
||||
let tx2 = tx.clone();
|
||||
let sent_at = Arc::new(Mutex::new(None::<Instant>));
|
||||
let sent_at2 = sent_at.clone();
|
||||
let h = thread::spawn(move || {
|
||||
tx2.send(Item { sz: 100, tag: 2 }).unwrap();
|
||||
*sent_at2.lock().unwrap() = Some(Instant::now());
|
||||
});
|
||||
|
||||
// Give the sender thread a head start; it should be parked in
|
||||
// `cv.wait` because used (200) + 100 > capacity (200).
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
assert!(
|
||||
sent_at.lock().unwrap().is_none(),
|
||||
"third send should be blocked at capacity"
|
||||
);
|
||||
|
||||
// Drain one. Sender wakes and completes.
|
||||
let recv_at = Instant::now();
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got.tag, 0);
|
||||
h.join().unwrap();
|
||||
|
||||
let sent_when = sent_at.lock().unwrap().unwrap();
|
||||
assert!(
|
||||
sent_when >= recv_at,
|
||||
"sender must complete AFTER receiver freed capacity"
|
||||
);
|
||||
|
||||
// Drain the remaining two.
|
||||
assert_eq!(rx.recv().unwrap().tag, 1);
|
||||
assert_eq!(rx.recv().unwrap().tag, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_larger_than_capacity_still_goes_through() {
|
||||
// Pathological case: a single item bigger than the capacity.
|
||||
// The guard `*used > 0` lets it through when the channel is
|
||||
// empty (otherwise the sender deadlocks forever). Matches
|
||||
// `mpsc::SyncSender` semantics for oversize messages.
|
||||
let (tx, rx) = channel::<Item>(100);
|
||||
tx.send(Item { sz: 1000, tag: 7 }).unwrap();
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got, Item { sz: 1000, tag: 7 });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_send_recv_stress() {
|
||||
// 4 sender threads × 1k items each, 1 receiver. Verify byte
|
||||
// accounting stays sane (channel never deadlocks, every item
|
||||
// arrives exactly once) under contention.
|
||||
const SENDERS: u32 = 4;
|
||||
const PER_SENDER: u32 = 1000;
|
||||
const TOTAL: u32 = SENDERS * PER_SENDER;
|
||||
|
||||
let (tx, rx) = channel::<Item>(8 * 1024);
|
||||
let sent = Arc::new(AtomicUsize::new(0));
|
||||
let mut handles = Vec::new();
|
||||
for s in 0..SENDERS {
|
||||
let tx = tx.clone();
|
||||
let sent = sent.clone();
|
||||
handles.push(thread::spawn(move || {
|
||||
for i in 0..PER_SENDER {
|
||||
// Vary item size so accounting actually has to
|
||||
// multiplex differently-sized blockers. 1B → 256B.
|
||||
let sz = 1 + ((i as usize) % 256);
|
||||
tx.send(Item {
|
||||
sz,
|
||||
tag: s * PER_SENDER + i,
|
||||
})
|
||||
.unwrap();
|
||||
sent.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}));
|
||||
}
|
||||
// Drop our local sender so the receiver can eventually see
|
||||
// RecvError once all sender clones are done. Cloning the
|
||||
// sender into each producer means each clone Drop'd separately.
|
||||
drop(tx);
|
||||
|
||||
let mut received = 0u32;
|
||||
while let Ok(_item) = rx.recv() {
|
||||
received += 1;
|
||||
}
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
assert_eq!(received, TOTAL);
|
||||
assert_eq!(sent.load(Ordering::SeqCst) as u32, TOTAL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_after_recv_dropped_returns_err() {
|
||||
let (tx, rx) = channel::<Item>(1024);
|
||||
drop(rx);
|
||||
let r = tx.send(Item { sz: 10, tag: 0 });
|
||||
assert!(r.is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
//! Linux read-side platform hooks: sequential-access hint at open +
|
||||
//! periodic page-cache eviction during streaming reads.
|
||||
//!
|
||||
//! ## Why both
|
||||
//!
|
||||
//! `POSIX_FADV_SEQUENTIAL` at open widens the kernel's readahead window
|
||||
//! so each pread aggregates into fewer NFS round-trips. `DONTNEED` on
|
||||
//! the consumed window (called periodically by the caller) drops the
|
||||
//! already-read pages from the page cache so an 85 GB streaming ISO
|
||||
//! read doesn't fill memory and starve concurrent writes (the MKV
|
||||
//! output during mux). Together they mirror the write-side
|
||||
//! WritebackPipeline's policy.
|
||||
//!
|
||||
//! ## History
|
||||
//!
|
||||
//! Pre-Phase-1 (0.20.7 baseline) had both. Phase 1's introduction of
|
||||
//! `FileSectorSource` silently dropped the read-side DONTNEED, and
|
||||
//! 0.21.2's revert of `SEQUENTIAL` (mistakenly attributing a regression
|
||||
//! to it) removed the hint. Net effect: 85 GB of ISO reads pinned in
|
||||
//! the page cache + no readahead widening → mux throughput collapse
|
||||
//! from 18 MB/s historical to 2.7-8 MB/s on 0.21.x. Restored in 0.21.6.
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
pub(super) fn hint_sequential(file: &File, _len_bytes: u64) {
|
||||
// Best-effort: return value ignored. A fadvise failure has no
|
||||
// user-observable consequence.
|
||||
unsafe {
|
||||
libc::posix_fadvise(file.as_raw_fd(), 0, 0, libc::POSIX_FADV_SEQUENTIAL);
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop pages in the half-open byte range `[start, start+len)` from
|
||||
/// the page cache. Called periodically by `read_sectors` to bound the
|
||||
/// read-side page cache pressure.
|
||||
pub(super) fn drop_window(file: &File, start: u64, len: u64) {
|
||||
unsafe {
|
||||
libc::posix_fadvise(
|
||||
file.as_raw_fd(),
|
||||
start as i64,
|
||||
len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
//! macOS: hint the kernel to prefetch a generous chunk. macOS has no
|
||||
//! direct `POSIX_FADV_SEQUENTIAL` equivalent; the idiomatic hint is
|
||||
//! `fcntl(F_RDADVISE, &radvisory)` describing the byte range you
|
||||
//! intend to read soon. We point it at the whole file (clamped to a
|
||||
//! ceiling so a multi-TB ISO doesn't ask the kernel to prefetch
|
||||
//! everything at once).
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
/// `F_RDADVISE` opcode — not in libc's named constants on all SDKs.
|
||||
const F_RDADVISE: libc::c_int = 44;
|
||||
|
||||
/// Cap on the byte length we pass to `F_RDADVISE`. Asking for a
|
||||
/// multi-GB readahead window is counterproductive — the OS doesn't
|
||||
/// have that much cache to throw at one fd. 64 MiB is generous for
|
||||
/// our use case (sweep, mux) and matches the byte-channel cap so the
|
||||
/// kernel's prefetch ≥ our app-level pipeline depth.
|
||||
const RDADVISE_MAX_BYTES: i64 = 64 * 1024 * 1024;
|
||||
|
||||
/// `radvisory` per `<sys/fcntl.h>`. repr(C) layout is stable.
|
||||
#[repr(C)]
|
||||
struct RadAdvisory {
|
||||
ra_offset: libc::off_t,
|
||||
ra_count: libc::c_int,
|
||||
}
|
||||
|
||||
pub(super) fn hint_sequential(file: &File, len_bytes: u64) {
|
||||
let bytes = (len_bytes as i64).min(RDADVISE_MAX_BYTES);
|
||||
let mut ra = RadAdvisory {
|
||||
ra_offset: 0,
|
||||
ra_count: bytes as libc::c_int,
|
||||
};
|
||||
// Best-effort.
|
||||
unsafe {
|
||||
libc::fcntl(file.as_raw_fd(), F_RDADVISE, &mut ra);
|
||||
}
|
||||
}
|
||||
|
||||
/// macOS has no direct `POSIX_FADV_DONTNEED` equivalent for a byte
|
||||
/// range. `fcntl(F_NOCACHE)` would disable caching globally on the fd
|
||||
/// (too coarse — we want the unread region to still benefit). Best
|
||||
/// approximation: no-op. macOS's unified buffer cache is generally
|
||||
/// less prone to the pin-everything pathology that triggers the
|
||||
/// regression on Linux NFS clients.
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
@@ -0,0 +1,428 @@
|
||||
//! [`FileSectorSource`] — read 2048-byte sectors from an ISO file on
|
||||
//! disk, with an internal 32 MiB read-ahead buffer.
|
||||
//!
|
||||
//! ## Why the buffer
|
||||
//!
|
||||
//! On NFS-mounted ISOs, an unbuffered `pread(2048)` per sector pays an
|
||||
//! NFS round-trip for every sector. With `rsize=1 MiB` and a 100-150 ms
|
||||
//! NFS RTT, that's three orders of magnitude more round trips than
|
||||
//! necessary — the muxer goes read-bound on every read, even though
|
||||
//! the local NFS client could deliver MB/s on bigger requests.
|
||||
//!
|
||||
//! Internally this source keeps a [`READAHEAD_BUF_BYTES`] (32 MiB)
|
||||
//! window pre-read from the file. `read_sectors(lba, count)` slices
|
||||
//! into the window if `[lba, lba+count)` is contained in it; otherwise
|
||||
//! the window is refilled (full-size aligned to the requested LBA's
|
||||
//! buffer position).
|
||||
//!
|
||||
//! ## Access pattern assumption
|
||||
//!
|
||||
//! The buffer is sized for **forward-sequential** reads (sweep, mux).
|
||||
//! Reverse-mode patch is range-local, so a refill per range works out
|
||||
//! fine (the buffer covers the whole range for typical bad-range
|
||||
//! sizes). Random-access reads thrash the buffer — at which point the
|
||||
//! 32 MiB pre-read is wasted work. We accept that: the use case is
|
||||
//! mux + sweep, both forward-sequential.
|
||||
//!
|
||||
//! Backward seeks rebuffer from the new LBA; partial reads at EOF
|
||||
//! return only the bytes that exist (the underlying file is shorter
|
||||
//! than a full buffer slot).
|
||||
//!
|
||||
//! ## Platform open hints
|
||||
//!
|
||||
//! On `open()` each platform issues its "sequential access expected"
|
||||
//! hint to the kernel so OS-level readahead widens. The hint lives in
|
||||
//! a per-OS sibling module ([`linux::hint_sequential`] et al.) — no
|
||||
//! inline `#[cfg]` in this file.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
mod other;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux as platform;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos as platform;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
use other as platform;
|
||||
#[cfg(target_os = "windows")]
|
||||
use windows as platform;
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::Path;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorSource;
|
||||
|
||||
/// Internal read-ahead buffer size. 32 MiB amortises one NFS round
|
||||
/// trip across ~16 k sectors — three orders of magnitude fewer trips
|
||||
/// than per-sector pread, and large enough to coast through a typical
|
||||
/// NFS server commit blip.
|
||||
///
|
||||
/// 0.21.2: shrunk from 32 MiB → 4 MiB. On NFS-backed ISOs with
|
||||
/// concurrent NFS writes (the mux phase), a 32 MiB refill bursts the
|
||||
/// TCP connection hard enough to starve the writer thread, observed
|
||||
/// empirically as a ~3× drop in sustained mux throughput on the
|
||||
/// rip1/unraid-1 setup. 4 MiB matches `rsize=1 MiB` × 4 round-trips
|
||||
/// and interleaves cleanly with writes.
|
||||
///
|
||||
/// Tweakable. Named const, not a magic number.
|
||||
pub const READAHEAD_BUF_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
const SECTOR_SIZE: usize = 2048;
|
||||
/// Sectors per refill: [`READAHEAD_BUF_BYTES`] / [`SECTOR_SIZE`]. The
|
||||
/// buffer always tries to hold this many, except at the tail of the
|
||||
/// file where less data exists.
|
||||
const BUF_SECTORS: u32 = (READAHEAD_BUF_BYTES / SECTOR_SIZE) as u32;
|
||||
|
||||
/// SectorSource backed by a file (ISO image) with an internal
|
||||
/// `READAHEAD_BUF_BYTES`-sized read-ahead window.
|
||||
///
|
||||
/// `read_sectors` is satisfied from the buffer when possible; otherwise
|
||||
/// a full-buffer refill is issued at the requested LBA's position and
|
||||
/// the call is re-tried against the freshly populated window.
|
||||
/// Bytes-read threshold per `posix_fadvise(DONTNEED)` drop on the
|
||||
/// read side. Mirrors `WRITEBACK_CHUNK_BYTES` so the read-side page
|
||||
/// cache stays bounded the same way the write side does.
|
||||
///
|
||||
/// 0.21.6: re-added after empirical discovery that Phase 1 had silently
|
||||
/// dropped this from the pre-Phase-1 (0.20.7) hot path. Without it,
|
||||
/// 85 GB of streaming ISO reads pin the entire file in the kernel page
|
||||
/// cache, starving the MKV writeback and collapsing mux throughput
|
||||
/// (observed: 2.7 MB/s mux on 0.21.5 vs. 70 MB/s isolated NFS reads).
|
||||
const READ_DROP_CHUNK_BYTES: u64 = 32 * 1024 * 1024;
|
||||
|
||||
pub struct FileSectorSource {
|
||||
file: File,
|
||||
/// Total file size in sectors. Constant after construction;
|
||||
/// surfaced via [`SectorSource::capacity_sectors`].
|
||||
capacity: u32,
|
||||
/// 0.21.3+: the app-level buffer is no longer touched on the hot
|
||||
/// path (every `read_sectors` is a direct pread). The fields are
|
||||
/// retained so a future per-source-type policy (e.g. a local-disk
|
||||
/// source where batched reads ARE beneficial) can re-enable
|
||||
/// buffering cleanly without re-plumbing the struct.
|
||||
#[allow(dead_code)]
|
||||
buf: Box<[u8]>,
|
||||
#[allow(dead_code)]
|
||||
buf_start_lba: u32,
|
||||
buf_len_sectors: u32,
|
||||
/// 0.21.6: bytes read since the last DONTNEED drop. Drives the
|
||||
/// per-`READ_DROP_CHUNK_BYTES` page-cache eviction in read_sectors.
|
||||
bytes_read_since_drop: u64,
|
||||
/// 0.21.6: file offset at which the current drop window starts.
|
||||
/// The next DONTNEED drops from `drop_window_start` for
|
||||
/// `bytes_read_since_drop` bytes.
|
||||
drop_window_start: u64,
|
||||
}
|
||||
|
||||
impl FileSectorSource {
|
||||
/// Open an existing ISO file for reading. Capacity is derived
|
||||
/// from `metadata().len() / 2048`. Returns
|
||||
/// [`Error::IsoTooLarge`] if the file would exceed the 32-bit
|
||||
/// LBA address space (~8 TB).
|
||||
///
|
||||
/// Issues the platform's "sequential access expected" hint on the
|
||||
/// fd (Linux `posix_fadvise(SEQUENTIAL)`, macOS `fcntl(F_RDADVISE)`,
|
||||
/// Windows TODO stub) so the kernel's readahead widens.
|
||||
pub fn open(path: &Path) -> std::io::Result<Self> {
|
||||
let file = File::open(path)?;
|
||||
let len = file.metadata()?.len();
|
||||
let sectors = len / SECTOR_SIZE as u64;
|
||||
if sectors > u32::MAX as u64 {
|
||||
return Err(Error::IsoTooLarge {
|
||||
path: path.to_string_lossy().into_owned(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let capacity = sectors as u32;
|
||||
|
||||
// Best-effort sequential hint. Ignored on platforms without
|
||||
// an equivalent primitive (or where the API exists but the
|
||||
// FS doesn't honour it).
|
||||
platform::hint_sequential(&file, len);
|
||||
|
||||
// Pre-allocate the buffer once. `vec![0u8; N].into_boxed_slice()`
|
||||
// is the canonical way to fix the allocation size up-front;
|
||||
// `Vec::with_capacity` would leave `len == 0` and force callers
|
||||
// to do unsafe length manipulation to write into it.
|
||||
let buf = vec![0u8; READAHEAD_BUF_BYTES].into_boxed_slice();
|
||||
|
||||
Ok(Self {
|
||||
file,
|
||||
capacity,
|
||||
buf,
|
||||
buf_start_lba: 0,
|
||||
buf_len_sectors: 0,
|
||||
bytes_read_since_drop: 0,
|
||||
drop_window_start: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// True if `[lba, lba + count)` is wholly inside the current
|
||||
/// buffer window. `count == 0` is vacuously true.
|
||||
#[allow(dead_code)]
|
||||
fn buffer_covers(&self, lba: u32, count: u32) -> bool {
|
||||
if self.buf_len_sectors == 0 {
|
||||
return false;
|
||||
}
|
||||
let end = match lba.checked_add(count) {
|
||||
Some(e) => e,
|
||||
None => return false,
|
||||
};
|
||||
let buf_end = self.buf_start_lba.saturating_add(self.buf_len_sectors);
|
||||
lba >= self.buf_start_lba && end <= buf_end
|
||||
}
|
||||
|
||||
/// Refill the buffer so it starts at `lba`. Read as many sectors
|
||||
/// as we have buffer space AND file capacity for. Caller has
|
||||
/// already checked `lba < capacity`.
|
||||
#[allow(dead_code)]
|
||||
fn refill(&mut self, lba: u32) -> Result<()> {
|
||||
debug_assert!(lba < self.capacity, "refill past capacity");
|
||||
// Don't read past EOF — clamp the request to remaining
|
||||
// sectors. partial-buffer-at-EOF behaviour is intentional.
|
||||
let want = BUF_SECTORS.min(self.capacity - lba);
|
||||
let want_bytes = want as usize * SECTOR_SIZE;
|
||||
let offset = lba as u64 * SECTOR_SIZE as u64;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.read_exact(&mut self.buf[..want_bytes])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.buf_start_lba = lba;
|
||||
self.buf_len_sectors = want;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for FileSectorSource {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
out: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
let count = count as u32;
|
||||
let bytes = count as usize * SECTOR_SIZE;
|
||||
debug_assert!(
|
||||
out.len() >= bytes,
|
||||
"FileSectorSource::read_sectors: out len {} < requested {}",
|
||||
out.len(),
|
||||
bytes
|
||||
);
|
||||
if count == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
// 0.21.3: bypass the application-level buffer entirely.
|
||||
//
|
||||
// Empirically the 32 MiB readahead window (0.21.0–0.21.1) and the
|
||||
// 4 MiB shrink (0.21.2) both regressed mux throughput vs the
|
||||
// pre-Phase-1 0.20.7 baseline on NFS bidirectional workloads
|
||||
// (sweep ~25 MB/s OK; mux dropped from 18 → 7-8 → 5-6 MB/s).
|
||||
// Direct pread per call lets the kernel's own readahead policy
|
||||
// run, which interleaves naturally with concurrent NFS writes on
|
||||
// the same TCP connection.
|
||||
//
|
||||
// Buffer fields are retained (currently unused on this path) so
|
||||
// any future per-source policy can be reintroduced without
|
||||
// re-plumbing structure. `refill` / `buffer_covers` are kept too
|
||||
// (still exercised by the tests so the API contract is locked).
|
||||
let offset = lba as u64 * SECTOR_SIZE as u64;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.read_exact(&mut out[..bytes])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.buf_len_sectors = 0;
|
||||
|
||||
// 0.21.6: periodic page-cache eviction on the read side. Without
|
||||
// this, an 85 GB streaming ISO read pins the entire file in
|
||||
// kernel page cache, which starves concurrent NFS writes (the
|
||||
// MKV output) and collapses mux throughput. Mirrors the
|
||||
// write-side WritebackPipeline's DONTNEED policy.
|
||||
self.bytes_read_since_drop += bytes as u64;
|
||||
if self.bytes_read_since_drop >= READ_DROP_CHUNK_BYTES {
|
||||
let drop_start = self.drop_window_start;
|
||||
let drop_len = self.bytes_read_since_drop;
|
||||
platform::drop_window(&self.file, drop_start, drop_len);
|
||||
self.drop_window_start = drop_start + drop_len;
|
||||
self.bytes_read_since_drop = 0;
|
||||
}
|
||||
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
use tempfile::tempdir;
|
||||
|
||||
/// Build a deterministic ISO of `sectors` sectors where sector `n`
|
||||
/// is filled with the byte pattern `((n & 0xff) as u8)`. Lets us
|
||||
/// verify any sector by content alone.
|
||||
fn make_iso(path: &std::path::Path, sectors: u32) {
|
||||
let mut f = std::fs::File::create(path).unwrap();
|
||||
let mut chunk = vec![0u8; SECTOR_SIZE];
|
||||
for n in 0..sectors {
|
||||
let b = (n & 0xff) as u8;
|
||||
chunk.iter_mut().for_each(|c| *c = b);
|
||||
f.write_all(&chunk).unwrap();
|
||||
}
|
||||
f.flush().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sequential_reads_match_file() {
|
||||
// Two full buffer windows + a tail = exercise refill across
|
||||
// boundaries.
|
||||
let total = BUF_SECTORS * 2 + 17;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("seq.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), total);
|
||||
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
for lba in 0..total {
|
||||
src.read_sectors(lba, 1, &mut got, false).unwrap();
|
||||
let expected = (lba & 0xff) as u8;
|
||||
assert!(
|
||||
got.iter().all(|b| *b == expected),
|
||||
"sector {lba} content mismatch: expected 0x{expected:02x}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_sector_read_spanning_buffer_boundary() {
|
||||
// A read that lands exactly on the last sector of the buffer
|
||||
// plus the first sector of the next refill must rebuffer
|
||||
// mid-read. Bypass path triggers when count > BUF_SECTORS; we
|
||||
// want the in-window path, so count stays small but
|
||||
// straddles the boundary.
|
||||
let total = BUF_SECTORS * 2;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("span.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
|
||||
// Prime: read sector 0. (0.21.3+: app-level buffer is bypassed,
|
||||
// so we don't assert internal buf state here — just exercise
|
||||
// the read path.)
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
|
||||
// Now read 4 sectors crossing what used to be the buffer
|
||||
// boundary. Still a valid SectorSource-contract test.
|
||||
let span_lba = BUF_SECTORS - 2;
|
||||
let mut buf4 = vec![0u8; SECTOR_SIZE * 4];
|
||||
src.read_sectors(span_lba, 4, &mut buf4, false).unwrap();
|
||||
for i in 0..4 {
|
||||
let lba = span_lba + i as u32;
|
||||
let expected = (lba & 0xff) as u8;
|
||||
for b in &buf4[i * SECTOR_SIZE..(i + 1) * SECTOR_SIZE] {
|
||||
assert_eq!(*b, expected, "byte mismatch at sub-sector {i}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backward_seek_rebuffers() {
|
||||
// Read forward across two windows, then jump back to sector
|
||||
// 0. Buffer must refill from the start.
|
||||
let total = BUF_SECTORS * 2 + 5;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("back.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
|
||||
// Forward to the second window.
|
||||
src.read_sectors(BUF_SECTORS + 1, 1, &mut got, false)
|
||||
.unwrap();
|
||||
|
||||
// Backward to sector 0. (0.21.3+: app-level buffer is bypassed
|
||||
// so we only assert the byte-level contract, not internal
|
||||
// buffer state.)
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
assert!(got.iter().all(|b| *b == 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_buffer_at_eof() {
|
||||
// File is smaller than one buffer window. The buffer must
|
||||
// populate with only the available sectors and reads must
|
||||
// still succeed.
|
||||
let total: u32 = 100;
|
||||
assert!(total < BUF_SECTORS);
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("small.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), total);
|
||||
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
// First read at sector 0.
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
|
||||
// Read the very last sector. (0.21.3+: app-level buffer is
|
||||
// bypassed; the test still verifies that EOF-region reads
|
||||
// return correct bytes.)
|
||||
src.read_sectors(total - 1, 1, &mut got, false).unwrap();
|
||||
let expected = ((total - 1) & 0xff) as u8;
|
||||
assert!(got.iter().all(|b| *b == expected));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_read_bypasses_buffer() {
|
||||
// A request larger than the buffer must not deadlock the
|
||||
// refill (which only loads BUF_SECTORS at a time). Bypass
|
||||
// path handles it via direct pread.
|
||||
let total = BUF_SECTORS + 100;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("over.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
// Read more than BUF_SECTORS in one call. count is u16, so we
|
||||
// can't actually exceed BUF_SECTORS (16k) — but the path also
|
||||
// triggers via `out.len() / SECTOR_SIZE > BUF_SECTORS` check
|
||||
// implicitly because count > BUF_SECTORS. BUF_SECTORS for
|
||||
// 32 MiB is 16384, which does fit in u16 (max 65535). Cap
|
||||
// at BUF_SECTORS + 1 to exercise the bypass.
|
||||
let req = (BUF_SECTORS + 1) as u16;
|
||||
let req_bytes = req as usize * SECTOR_SIZE;
|
||||
let mut big = vec![0u8; req_bytes];
|
||||
src.read_sectors(0, req, &mut big, false).unwrap();
|
||||
// Spot-check sector 0 and the last requested sector.
|
||||
assert!(big[..SECTOR_SIZE].iter().all(|b| *b == 0));
|
||||
let last_lba = req as u32 - 1;
|
||||
let exp = (last_lba & 0xff) as u8;
|
||||
let last_off = (req as usize - 1) * SECTOR_SIZE;
|
||||
assert!(
|
||||
big[last_off..last_off + SECTOR_SIZE]
|
||||
.iter()
|
||||
.all(|b| *b == exp)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
//! Fallback for targets without a known sequential-readahead hint
|
||||
//! (BSDs, illumos, etc.). No-op — reads still work, they just don't
|
||||
//! get the OS-level prefetch widening.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn hint_sequential(_file: &File, _len_bytes: u64) {}
|
||||
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
@@ -0,0 +1,24 @@
|
||||
//! Windows: the canonical sequential-access hint is
|
||||
//! `FILE_FLAG_SEQUENTIAL_SCAN` passed to `CreateFile` at open time —
|
||||
//! it cannot be set after the fact via `SetFileInformationByHandle`.
|
||||
//! Routing the open call through this module would mean a custom
|
||||
//! `File::from_raw_handle` plumb for every `FileSectorSource::open`
|
||||
//! caller, which is more invasive than the Phase 1 scope.
|
||||
//!
|
||||
//! TODO: replumb `FileSectorSource::open` to take an
|
||||
//! `OpenOptions`-style builder so the Windows path can flip the flag
|
||||
//! at open time. For now this is a no-op stub.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn hint_sequential(_file: &File, _len_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"FileSectorSource hint_sequential: windows stub (TODO: FILE_FLAG_SEQUENTIAL_SCAN at open)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Windows page-cache eviction is not exposed via a posix_fadvise
|
||||
/// equivalent. The kernel does its own working-set management. No-op
|
||||
/// for now.
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
@@ -0,0 +1,46 @@
|
||||
//! File I/O helpers that bound kernel cache pressure on big writes.
|
||||
//!
|
||||
//! `WritebackFile` is a drop-in wrapper around `std::fs::File` for any
|
||||
//! call site that performs large sequential writes (sweep, patch, mux,
|
||||
//! etc.). It implements `Write` and `Seek` so existing code paths can
|
||||
//! swap `File` for `WritebackFile` with no body changes. Internally it
|
||||
//! drives a `WritebackPipeline` that, on Linux, drains dirty pages
|
||||
//! continuously at 32 MB granularity to avoid the kernel's
|
||||
//! accumulate-then-burst flush behaviour. macOS and Windows use a
|
||||
//! no-op pipeline — their default cache policies have not been shown
|
||||
//! to exhibit the same pathology for this access pattern.
|
||||
//!
|
||||
//! `FileSectorSource` is the read-side dual — it implements
|
||||
//! [`crate::sector::SectorSource`] for an ISO file with an internal
|
||||
//! 32 MiB read-ahead buffer that amortises NFS round-trip latency
|
||||
//! across thousands of sector reads.
|
||||
//!
|
||||
//! `Pipeline` + `Sink` (0.18) is the generic producer/consumer primitive
|
||||
//! used by sweep, patch, and mux to overlap reads with writes via a
|
||||
//! bounded channel + dedicated consumer thread.
|
||||
//!
|
||||
//! `byte_channel` is a byte-sized producer/consumer channel for the
|
||||
//! mux pipeline, sized to absorb worst-case input read stalls (see
|
||||
//! `(internal)/memory/project_buffering_architecture.md`).
|
||||
|
||||
pub(crate) mod bounded;
|
||||
pub mod byte_channel;
|
||||
pub mod file_sector_source;
|
||||
pub mod sink;
|
||||
mod writeback;
|
||||
mod writeback_file;
|
||||
|
||||
pub mod pipeline;
|
||||
|
||||
pub(crate) use writeback_file::WritebackFile;
|
||||
|
||||
// Re-exports for the 0.18 redesign. Sweep + patch are both wired up
|
||||
// (disc/sweep.rs, disc/patch.rs); mux migrates separately in autorip.
|
||||
// `WRITE_THROUGH_DEPTH` is patch-specific and has no other in-tree
|
||||
// caller — the targeted `#[allow]` keeps the re-export visible without
|
||||
// dragging the rest of the module under `dead_code`.
|
||||
#[allow(unused_imports)]
|
||||
pub use pipeline::{
|
||||
DEFAULT_PIPELINE_DEPTH, Flow, Pipeline, READ_PIPELINE_DEPTH, Sink, WRITE_PIPELINE_DEPTH,
|
||||
WRITE_THROUGH_DEPTH,
|
||||
};
|
||||
@@ -0,0 +1,918 @@
|
||||
//! Generic bounded producer/consumer pipeline.
|
||||
//!
|
||||
//! `Pipeline<I, R>` spawns a single consumer thread, hands it items
|
||||
//! through a bounded `mpsc::sync_channel`, and joins it on `finish()`.
|
||||
//! The consumer's behaviour is supplied by a [`Sink`] implementation:
|
||||
//! `apply` is called once per item, `close` is called once at the end.
|
||||
//!
|
||||
//! Three call sites in libfreemkv want a producer/consumer split —
|
||||
//! sweep (migrated to `disc/sweep.rs::SweepSink`), patch, and mux.
|
||||
//! 0.18 collapses all three onto this primitive; sweep is in,
|
||||
//! patch and mux migrate in later 0.18 slices.
|
||||
//! See `(internal)/memory/0_18_redesign.md` for the full picture.
|
||||
//!
|
||||
//! ## Cancellation and error semantics
|
||||
//!
|
||||
//! - Producer dropping the channel (via `Pipeline::finish` dropping
|
||||
//! `tx`) signals end-of-stream; consumer flushes via `close()` and
|
||||
//! returns its `Output`.
|
||||
//! - Consumer returning [`Flow::Stop`] also calls `close()` and
|
||||
//! returns its `Output`. `send()` from the producer will then either
|
||||
//! succeed (if the item already fit in the channel buffer) or fail
|
||||
//! with `Err(item)` once the consumer has dropped its receiver.
|
||||
//! - Consumer returning `Err` from `apply` skips `close()` entirely;
|
||||
//! the consumer keeps draining the channel so the producer never
|
||||
//! blocks on a dead receiver, and the first error is propagated as
|
||||
//! the `JoinHandle` result.
|
||||
//! - Consumer panic is converted into
|
||||
//! `Error::IoError { source: io::Error::other(...) }`.
|
||||
//!
|
||||
//! ## Debug logging
|
||||
//!
|
||||
//! Set `FREEMKV_DEBUG=1` environment variable to enable verbose debug
|
||||
//! logging throughout the pipeline (channel sends/receives, backpressure,
|
||||
//! consumer lag detection). This is critical for diagnosing stalls.
|
||||
|
||||
use std::io;
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crossbeam_channel::{Sender, TrySendError, bounded};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::halt::Halt;
|
||||
|
||||
/// Deadline for [`Pipeline::finish_with_halt`]'s polling join. Chosen
|
||||
/// to be comfortably longer than the autorip hard watchdog
|
||||
/// (`HARD_WATCHDOG_STALL_SECS = 300s`) so the watchdog's `exit(1)`
|
||||
/// fires first when both are racing on the same wedged consumer.
|
||||
///
|
||||
/// 10 minutes is a backstop, not a normal timeout — the consumer is
|
||||
/// expected to drain in seconds. If we hit this, something is wedged
|
||||
/// inside a kernel call the consumer thread can't unwind from, and the
|
||||
/// caller has already lost the rip.
|
||||
pub const JOIN_TIMEOUT_SECS: u64 = 600;
|
||||
|
||||
/// Polling slice for the halt-aware send/finish loops. Mirrors the
|
||||
/// `bounded_syscall` cadence (250 ms) so halt observation feels equally
|
||||
/// responsive across both primitives.
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(250);
|
||||
|
||||
/// Halt-check cadence for the send loop. Producer blocks on
|
||||
/// [`crossbeam_channel::Sender::send_timeout`] for this slice — the
|
||||
/// kernel wakes it the instant the consumer drains a slot, so on the
|
||||
/// happy path there's no throughput cap from this primitive at all
|
||||
/// (the cap is whatever the underlying medium can sustain). When the
|
||||
/// consumer is genuinely wedged, the timeout fires every 250 ms and
|
||||
/// the producer checks the halt token; that's the latency a stop
|
||||
/// request will observe.
|
||||
///
|
||||
/// 0.21.7 replaced an old `std::sync::mpsc::sync_channel` + 50 ms
|
||||
/// `thread::sleep` polling loop that capped mux throughput at
|
||||
/// ~20 frames/sec ≈ 1 MB/s on saturated channels. See
|
||||
/// (internal)/memory/feedback_send_with_halt_poll_throttle.md
|
||||
/// for the multi-day diagnostic that surfaced it.
|
||||
const SEND_HALT_CHECK_INTERVAL: Duration = Duration::from_millis(250);
|
||||
|
||||
/// Check if verbose debug logging is enabled via FREEMKV_DEBUG env var.
|
||||
pub fn debug_enabled() -> bool {
|
||||
std::env::var("FREEMKV_DEBUG")
|
||||
.ok()
|
||||
.map(|v| v == "1" || v == "true" || v == "yes")
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Default channel depth for callers without a specific reason to
|
||||
/// pick another value. Kept conservative (4) — most callers should
|
||||
/// use READ_PIPELINE_DEPTH or WRITE_PIPELINE_DEPTH instead.
|
||||
pub const DEFAULT_PIPELINE_DEPTH: usize = 4;
|
||||
|
||||
/// Read pipeline depth. Larger buffer compensates for drive variability
|
||||
/// and NFS sync_file_range stalls; keeps ISO reader thread fed even when
|
||||
/// consumer blocks on write.
|
||||
pub const READ_PIPELINE_DEPTH: usize = 32;
|
||||
|
||||
/// Write pipeline depth. Smaller buffer reduces backpressure risk when
|
||||
/// sync_file_range blocks; prevents producer from accumulating too much
|
||||
/// work while consumer waits for NFS to drain.
|
||||
pub const WRITE_PIPELINE_DEPTH: usize = 16;
|
||||
|
||||
/// Channel depth for write-through pipelines. Each `send` fully
|
||||
/// drains before the next can enqueue. Use this when the producer
|
||||
/// must observe consumer side-effects (e.g. mapfile state) before
|
||||
/// emitting the next item.
|
||||
#[allow(dead_code)]
|
||||
pub const WRITE_THROUGH_DEPTH: usize = 1;
|
||||
|
||||
/// Outcome of [`Sink::apply`]: either keep feeding items
|
||||
/// ([`Flow::Continue`]), or stop the pipeline early and run `close()`
|
||||
/// ([`Flow::Stop`]).
|
||||
///
|
||||
/// `Stop` has no in-tree caller in this slice — sweep never returns
|
||||
/// it (it always processes the producer's full work-list before the
|
||||
/// channel is dropped). Patch and mux are the intended consumers and
|
||||
/// migrate in later 0.18 slices. The variant ships now so the contract
|
||||
/// is fixed; the targeted `#[allow]` is removed when patch lands.
|
||||
pub enum Flow {
|
||||
Continue,
|
||||
#[allow(dead_code)]
|
||||
Stop,
|
||||
}
|
||||
|
||||
/// Consumer-side of a [`Pipeline`]. The pipeline owns one of these on
|
||||
/// its consumer thread and calls `apply` once per received item, then
|
||||
/// `close` once at end-of-stream.
|
||||
pub trait Sink<I>: Send + 'static {
|
||||
/// Type returned from `close()` and surfaced via
|
||||
/// [`Pipeline::finish`].
|
||||
type Output: Send + 'static;
|
||||
|
||||
/// Apply one item. Returning [`Flow::Continue`] keeps the
|
||||
/// pipeline running; [`Flow::Stop`] ends it cleanly (still calls
|
||||
/// `close()`). An error short-circuits: `close()` is *not* called
|
||||
/// and the error is what `finish()` will return, but the consumer
|
||||
/// keeps draining the channel so the producer never blocks on a
|
||||
/// dead receiver.
|
||||
fn apply(&mut self, item: I) -> Result<Flow, Error>;
|
||||
|
||||
/// Called once at end-of-stream — either because the producer
|
||||
/// dropped `tx` or because `apply` returned [`Flow::Stop`]. Use
|
||||
/// this to flush, fsync, finalise. Skipped if any prior `apply`
|
||||
/// returned `Err`.
|
||||
fn close(self) -> Result<Self::Output, Error>;
|
||||
}
|
||||
|
||||
/// Bounded producer/consumer pipeline. Holds the producer-side
|
||||
/// channel and the consumer thread's join handle.
|
||||
pub struct Pipeline<I: Send + 'static, R: Send + 'static> {
|
||||
tx: Sender<I>,
|
||||
handle: JoinHandle<Result<R, Error>>,
|
||||
}
|
||||
|
||||
impl<I: Send + 'static, R: Send + 'static> Pipeline<I, R> {
|
||||
/// Spawn the consumer thread with the given channel depth and
|
||||
/// [`Sink`].
|
||||
///
|
||||
/// The thread is named `freemkv-pipeline-consumer` so it shows up
|
||||
/// distinctly in stack traces and `top -H`. Callers that want a
|
||||
/// more specific name (e.g. `freemkv-sweep-consumer`) should use
|
||||
/// [`Pipeline::spawn_named`] instead. Returns an `Error::IoError`
|
||||
/// if the OS refuses the thread spawn (resource exhaustion);
|
||||
/// callers already operate in fallible context, so this is
|
||||
/// propagated rather than panicked.
|
||||
///
|
||||
/// Sweep uses [`Pipeline::spawn_named`] directly so the consumer
|
||||
/// thread shows up as `freemkv-sweep-consumer`; this function has
|
||||
/// no in-tree caller yet. Patch and mux migrate in later 0.18
|
||||
/// slices. The targeted `#[allow]` is removed when one of them
|
||||
/// lands on the default name.
|
||||
#[allow(dead_code)]
|
||||
pub fn spawn<S: Sink<I, Output = R>>(depth: usize, sink: S) -> Result<Self, Error> {
|
||||
Self::spawn_named("freemkv-pipeline-consumer", depth, sink)
|
||||
}
|
||||
|
||||
/// Like [`Pipeline::spawn`] but lets the caller supply the
|
||||
/// consumer thread's name. Useful when several pipelines run in
|
||||
/// the same process and stack traces / `top -H` need to tell them
|
||||
/// apart (e.g. `freemkv-sweep-consumer`, `freemkv-mux-consumer`).
|
||||
pub fn spawn_named<S: Sink<I, Output = R>>(
|
||||
name: &str,
|
||||
depth: usize,
|
||||
sink: S,
|
||||
) -> Result<Self, Error> {
|
||||
let (tx, rx) = bounded::<I>(depth);
|
||||
let handle = thread::Builder::new()
|
||||
.name(name.into())
|
||||
.spawn(move || -> Result<R, Error> {
|
||||
let mut sink = sink;
|
||||
let mut first_err: Option<Error> = None;
|
||||
let mut stopped = false;
|
||||
|
||||
while let Ok(item) = rx.recv() {
|
||||
if debug_enabled() {
|
||||
tracing::debug!("Pipeline receive: item={}", std::any::type_name::<I>());
|
||||
}
|
||||
|
||||
let apply_start = std::time::Instant::now();
|
||||
|
||||
if first_err.is_some() || stopped {
|
||||
// Drain remaining items so the producer never
|
||||
// blocks on a dead receiver. `apply` is not
|
||||
// called once we've decided to stop.
|
||||
continue;
|
||||
}
|
||||
match sink.apply(item) {
|
||||
Ok(Flow::Continue) => {}
|
||||
Ok(Flow::Stop) => {
|
||||
stopped = true;
|
||||
if debug_enabled() {
|
||||
tracing::debug!("Pipeline: consumer returned Flow::Stop");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
if debug_enabled() {
|
||||
tracing::debug!("Pipeline: apply error, stopping, err={:?}", e);
|
||||
}
|
||||
first_err = Some(e);
|
||||
}
|
||||
}
|
||||
|
||||
let apply_elapsed = apply_start.elapsed();
|
||||
if debug_enabled() && apply_elapsed > std::time::Duration::from_millis(100) {
|
||||
tracing::debug!(
|
||||
"Pipeline apply: took {:.2}s, item={}",
|
||||
apply_elapsed.as_secs_f64(),
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
} else if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline apply: OK in {:.3}ms, item={}",
|
||||
apply_elapsed.as_micros(),
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
match first_err {
|
||||
Some(e) => Err(e),
|
||||
None => sink.close(),
|
||||
}
|
||||
})
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
|
||||
Ok(Pipeline { tx, handle })
|
||||
}
|
||||
|
||||
/// Push one item. Blocks if the channel is full — that's the
|
||||
/// back-pressure the whole primitive exists to provide. Returns
|
||||
/// the item back if the consumer thread is gone (panicked or
|
||||
/// already returned).
|
||||
///
|
||||
/// After the consumer returns [`Flow::Stop`], `send` will silently
|
||||
/// buffer items into the channel until the channel fills, then
|
||||
/// return `Err(item)` once the consumer has dropped its receiver.
|
||||
/// Producers that need to stop pushing on `Stop` should track an
|
||||
/// independent signal (e.g. `Halt`) — `send` alone is not the
|
||||
/// notification edge.
|
||||
pub fn send(&self, item: I) -> Result<(), I> {
|
||||
let start = std::time::Instant::now();
|
||||
match self.tx.send(item) {
|
||||
Ok(()) => {
|
||||
let elapsed = start.elapsed();
|
||||
if debug_enabled() && elapsed > std::time::Duration::from_millis(10) {
|
||||
tracing::debug!(
|
||||
"Pipeline send: blocked {:.2}s, item={}",
|
||||
elapsed.as_secs_f64(),
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
} else if debug_enabled() {
|
||||
tracing::debug!("Pipeline send: OK in {:.3}ms", elapsed.as_micros());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
let elapsed = start.elapsed();
|
||||
if debug_enabled() && elapsed > std::time::Duration::from_millis(10) {
|
||||
tracing::debug!(
|
||||
"Pipeline send: blocked {:.2}s before channel closed, item={}",
|
||||
elapsed.as_secs_f64(),
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
} else if debug_enabled() {
|
||||
tracing::debug!("Pipeline send: failed after {:.3}ms", elapsed.as_micros());
|
||||
}
|
||||
Err(e.0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Non-blocking variant of [`Pipeline::send`]. If the channel is
|
||||
/// full or the consumer has hung up, the item is returned in
|
||||
/// `Err`. Useful for best-effort signalling (e.g. sweep's
|
||||
/// throttled `StatsRequest`) where dropping the message is
|
||||
/// preferable to blocking the producer.
|
||||
pub fn try_send(&self, item: I) -> Result<(), TrySendError<I>> {
|
||||
self.tx.try_send(item)
|
||||
}
|
||||
|
||||
/// Halt-aware bounded variant of [`Pipeline::send`].
|
||||
///
|
||||
/// Uses [`crossbeam_channel::Sender::send_timeout`] so the producer
|
||||
/// thread BLOCKS on consumer drain (kernel-wakeup) rather than
|
||||
/// polling. The timeout slice is just the halt-observation cadence
|
||||
/// ([`SEND_HALT_CHECK_INTERVAL`]) — on the happy path the producer
|
||||
/// wakes the instant the consumer drains a slot, so there is no
|
||||
/// throughput cap from this primitive at any medium speed.
|
||||
///
|
||||
/// Returns:
|
||||
///
|
||||
/// - `Ok(())` once the item lands in the channel.
|
||||
/// - `Err(item)` if the consumer disconnected, the halt fired, or
|
||||
/// the deadline elapsed — the caller gets the item back so it
|
||||
/// can decide whether to drop it, route it elsewhere, or unwind.
|
||||
///
|
||||
/// Use this in producer threads that have a `Halt` token threaded
|
||||
/// through (mux, sweep, patch). Plain [`Pipeline::send`] is
|
||||
/// preserved for callers that don't (yet) plumb halt through.
|
||||
///
|
||||
/// Unlike [`Pipeline::send`], this never blocks the producer
|
||||
/// thread inside an unkillable `mpsc::send` — if the consumer is
|
||||
/// wedged inside an unkillable syscall, the producer can still
|
||||
/// observe `/api/stop` and unwind within
|
||||
/// [`SEND_HALT_CHECK_INTERVAL`].
|
||||
pub fn send_with_halt(&self, item: I, halt: &Halt, deadline: Duration) -> Result<(), I> {
|
||||
use crossbeam_channel::SendTimeoutError;
|
||||
let end = Instant::now() + deadline;
|
||||
let mut pending = item;
|
||||
loop {
|
||||
// Pre-check the cheap exit conditions before parking.
|
||||
if halt.is_cancelled() {
|
||||
if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline send_with_halt: halt observed, returning item={}",
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
return Err(pending);
|
||||
}
|
||||
let now = Instant::now();
|
||||
if now >= end {
|
||||
if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline send_with_halt: deadline elapsed, returning item={}",
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
return Err(pending);
|
||||
}
|
||||
// Wait for space-available or halt-check tick, whichever
|
||||
// is sooner. Crossbeam's send_timeout is kernel-wakeup
|
||||
// based: the consumer's recv on a saturated channel
|
||||
// signals this thread the moment a slot opens up.
|
||||
let slice = SEND_HALT_CHECK_INTERVAL.min(end.saturating_duration_since(now));
|
||||
match self.tx.send_timeout(pending, slice) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(SendTimeoutError::Timeout(returned)) => {
|
||||
pending = returned;
|
||||
// loop: re-check halt + deadline, then park again
|
||||
}
|
||||
Err(SendTimeoutError::Disconnected(returned)) => {
|
||||
if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline send_with_halt: consumer disconnected, item={}",
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
return Err(returned);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the producer-side channel and wait for the consumer
|
||||
/// thread to finish. Returns whatever the consumer's `close()`
|
||||
/// produced, or the first `apply` error, or — on consumer panic —
|
||||
/// an `Error::IoError` whose source is `io::Error::other(...)`
|
||||
/// with a "pipeline consumer panicked: <payload>" message
|
||||
/// (callers can match on the constant prefix).
|
||||
pub fn finish(self) -> Result<R, Error> {
|
||||
let Pipeline { tx, handle } = self;
|
||||
// Explicit drop, although the destructure already drops `tx`
|
||||
// at end-of-scope. Being explicit keeps the intent obvious.
|
||||
drop(tx);
|
||||
match handle.join() {
|
||||
Ok(result) => result,
|
||||
Err(payload) => {
|
||||
// Preserve the original panic message when the
|
||||
// consumer's panic payload was a `&str` or `String`
|
||||
// (the two stdlib formats that `panic!` produces).
|
||||
// Anything else falls back to "(no message)".
|
||||
let msg = payload
|
||||
.downcast_ref::<&'static str>()
|
||||
.copied()
|
||||
.or_else(|| payload.downcast_ref::<String>().map(|s| s.as_str()))
|
||||
.unwrap_or("(no message)");
|
||||
Err(Error::IoError {
|
||||
source: io::Error::other(format!("pipeline consumer panicked: {msg}")),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Halt-aware, deadline-bounded variant of [`Pipeline::finish`].
|
||||
///
|
||||
/// Drops the producer-side channel (same as `finish`) and then
|
||||
/// polls `JoinHandle::is_finished()` on a 250 ms cadence. Between
|
||||
/// slices, checks (1) the optional [`Halt`] token and (2) the
|
||||
/// [`JOIN_TIMEOUT_SECS`] deadline. Returns:
|
||||
///
|
||||
/// - `Ok(R)` on a clean consumer exit.
|
||||
/// - `Err(Error::IoError)` with one of three message prefixes for
|
||||
/// wedge cases:
|
||||
/// - `"pipeline join halted"` — halt fired while waiting.
|
||||
/// - `"pipeline join timed out"` — `JOIN_TIMEOUT_SECS` elapsed.
|
||||
/// - `"pipeline consumer panicked"` — same as `finish()`.
|
||||
///
|
||||
/// In the `halted` and `timed out` branches the consumer thread is
|
||||
/// intentionally leaked — exactly the same trade-off the
|
||||
/// `bounded_syscall` primitive makes. The wedged kernel call
|
||||
/// inside the consumer will unwind whenever it does, or at
|
||||
/// process exit. The caller is free to fall back to a degraded
|
||||
/// path (in autorip's case: `exit(1)` after the hard watchdog
|
||||
/// escalation, letting Docker restart the container).
|
||||
///
|
||||
/// Plain [`Pipeline::finish`] is preserved for callers without a
|
||||
/// halt-token plumbed through; that path still blocks indefinitely
|
||||
/// on `join()`, matching pre-0.20.8 behaviour.
|
||||
pub fn finish_with_halt(self, halt: Option<&Halt>) -> Result<R, Error> {
|
||||
let Pipeline { tx, handle } = self;
|
||||
drop(tx);
|
||||
let deadline = Instant::now() + Duration::from_secs(JOIN_TIMEOUT_SECS);
|
||||
loop {
|
||||
if handle.is_finished() {
|
||||
return match handle.join() {
|
||||
Ok(result) => result,
|
||||
Err(payload) => {
|
||||
let msg = payload
|
||||
.downcast_ref::<&'static str>()
|
||||
.copied()
|
||||
.or_else(|| payload.downcast_ref::<String>().map(|s| s.as_str()))
|
||||
.unwrap_or("(no message)");
|
||||
Err(Error::IoError {
|
||||
source: io::Error::other(format!("pipeline consumer panicked: {msg}")),
|
||||
})
|
||||
}
|
||||
};
|
||||
}
|
||||
if let Some(h) = halt {
|
||||
if h.is_cancelled() {
|
||||
// Consumer thread is intentionally leaked.
|
||||
return Err(Error::IoError {
|
||||
source: io::Error::other("pipeline join halted"),
|
||||
});
|
||||
}
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
// Consumer thread is intentionally leaked.
|
||||
return Err(Error::IoError {
|
||||
source: io::Error::other("pipeline join timed out"),
|
||||
});
|
||||
}
|
||||
thread::sleep(POLL_INTERVAL);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Sums u64s; returns the total from `close`.
|
||||
struct SumSink {
|
||||
total: u64,
|
||||
}
|
||||
|
||||
impl Sink<u64> for SumSink {
|
||||
type Output = u64;
|
||||
|
||||
fn apply(&mut self, item: u64) -> Result<Flow, Error> {
|
||||
self.total += item;
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(self) -> Result<u64, Error> {
|
||||
Ok(self.total)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn happy_path_sums_items() {
|
||||
let pipe = Pipeline::spawn(DEFAULT_PIPELINE_DEPTH, SumSink { total: 0 })
|
||||
.expect("spawn should succeed");
|
||||
let mut expected = 0u64;
|
||||
for i in 0..100u64 {
|
||||
expected += i;
|
||||
pipe.send(i).expect("send should succeed");
|
||||
}
|
||||
let total = pipe.finish().expect("finish should succeed");
|
||||
assert_eq!(total, expected);
|
||||
assert_eq!(total, (0..100u64).sum::<u64>());
|
||||
}
|
||||
|
||||
/// Sleeps `delay` per apply; counts how many it received.
|
||||
struct SlowSink {
|
||||
delay: Duration,
|
||||
count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Sink<()> for SlowSink {
|
||||
type Output = usize;
|
||||
|
||||
fn apply(&mut self, _item: ()) -> Result<Flow, Error> {
|
||||
std::thread::sleep(self.delay);
|
||||
self.count.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(self) -> Result<usize, Error> {
|
||||
Ok(self.count.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn back_pressure_blocks_sender() {
|
||||
// depth=2 + 5 sends + 50ms/apply: with the consumer pinned at
|
||||
// 50 ms per item, the producer can buffer 2 (channel cap) +
|
||||
// 1 (consumer in flight) = 3 items before sends 4 and 5 must
|
||||
// block on consumer progress. Wall-clock floor across all 5
|
||||
// sends is therefore ~2 * 50ms = 100ms (sends 4 and 5 each
|
||||
// wait roughly one apply-cycle). Use 80 ms as the assertion
|
||||
// floor to stay above the 50ms-per-item progress floor while
|
||||
// tolerating CI jitter — it still proves blocking is real.
|
||||
let count = Arc::new(AtomicUsize::new(0));
|
||||
let sink = SlowSink {
|
||||
delay: Duration::from_millis(50),
|
||||
count: count.clone(),
|
||||
};
|
||||
let pipe = Pipeline::spawn(2, sink).expect("spawn should succeed");
|
||||
|
||||
let start = Instant::now();
|
||||
for _ in 0..5 {
|
||||
pipe.send(()).expect("send should succeed");
|
||||
}
|
||||
let elapsed_send = start.elapsed();
|
||||
|
||||
let total = pipe.finish().expect("finish should succeed");
|
||||
assert_eq!(total, 5);
|
||||
assert!(
|
||||
elapsed_send >= Duration::from_millis(80),
|
||||
"back-pressure not observed: 5 sends with depth=2 and 50ms/apply \
|
||||
took {elapsed_send:?}, expected ≥ ~100ms (one or more sends \
|
||||
should have blocked behind the consumer)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns `Err` on the Nth apply (1-indexed). Tracks all calls.
|
||||
struct FailOnNthSink {
|
||||
n: usize,
|
||||
seen: Arc<AtomicUsize>,
|
||||
close_called: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Sink<u64> for FailOnNthSink {
|
||||
type Output = ();
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
let i = self.seen.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if i == self.n {
|
||||
Err(Error::DecryptFailed)
|
||||
} else {
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
fn close(self) -> Result<(), Error> {
|
||||
self.close_called.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_error_drains_then_propagates() {
|
||||
let seen = Arc::new(AtomicUsize::new(0));
|
||||
let close_called = Arc::new(AtomicUsize::new(0));
|
||||
let pipe = Pipeline::spawn(
|
||||
DEFAULT_PIPELINE_DEPTH,
|
||||
FailOnNthSink {
|
||||
n: 3,
|
||||
seen: seen.clone(),
|
||||
close_called: close_called.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
|
||||
// Send 10 items. Subsequent sends after the 3rd error must
|
||||
// still succeed (the consumer is draining).
|
||||
for i in 0..10u64 {
|
||||
pipe.send(i).expect("send should succeed even after error");
|
||||
}
|
||||
|
||||
let res = pipe.finish();
|
||||
assert!(matches!(res, Err(Error::DecryptFailed)));
|
||||
assert_eq!(
|
||||
close_called.load(Ordering::SeqCst),
|
||||
0,
|
||||
"close() must not be called when apply returned Err"
|
||||
);
|
||||
// The consumer kept calling `recv` to drain after the error;
|
||||
// it just stopped invoking `apply`. So `seen` is exactly 3
|
||||
// (apply was called for items 1, 2, 3).
|
||||
assert_eq!(seen.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
/// Returns `Flow::Stop` on the Nth apply.
|
||||
struct StopOnNthSink {
|
||||
n: usize,
|
||||
seen: Arc<AtomicUsize>,
|
||||
close_called: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Sink<u64> for StopOnNthSink {
|
||||
type Output = usize;
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
let i = self.seen.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if i >= self.n {
|
||||
Ok(Flow::Stop)
|
||||
} else {
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
fn close(self) -> Result<usize, Error> {
|
||||
self.close_called.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(self.seen.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_stop_calls_close_and_returns_output() {
|
||||
let seen = Arc::new(AtomicUsize::new(0));
|
||||
let close_called = Arc::new(AtomicUsize::new(0));
|
||||
let pipe = Pipeline::spawn(
|
||||
DEFAULT_PIPELINE_DEPTH,
|
||||
StopOnNthSink {
|
||||
n: 3,
|
||||
seen: seen.clone(),
|
||||
close_called: close_called.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
|
||||
// Send 10 items. After Stop, subsequent sends may either
|
||||
// succeed (already buffered) or fail with Err(I) (channel
|
||||
// closed). Both are valid — we don't assert on the send
|
||||
// results.
|
||||
for i in 0..10u64 {
|
||||
let _ = pipe.send(i);
|
||||
}
|
||||
|
||||
let out = pipe.finish().expect("finish should succeed after Stop");
|
||||
assert_eq!(close_called.load(Ordering::SeqCst), 1);
|
||||
// At least 3 items processed (the one that returned Stop).
|
||||
assert!(
|
||||
out >= 3,
|
||||
"expected ≥ 3 applies before Stop took effect, got {out}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Panics on the first apply call.
|
||||
struct PanickingSink;
|
||||
|
||||
impl Sink<u64> for PanickingSink {
|
||||
type Output = ();
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
panic!("synthetic test panic");
|
||||
}
|
||||
|
||||
fn close(self) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consumer_panic_becomes_io_error() {
|
||||
// Silence the panic message that would otherwise pollute the
|
||||
// test output — we expect this panic.
|
||||
let prev = std::panic::take_hook();
|
||||
std::panic::set_hook(Box::new(|_| {}));
|
||||
|
||||
let pipe =
|
||||
Pipeline::spawn(DEFAULT_PIPELINE_DEPTH, PanickingSink).expect("spawn should succeed");
|
||||
// First send may succeed (item buffered before panic) or fail
|
||||
// (channel closed after panic) — either is fine.
|
||||
let _ = pipe.send(1);
|
||||
// Drain a few more sends; once the channel is closed they'll
|
||||
// return Err(I), which we just discard.
|
||||
for i in 0..5u64 {
|
||||
let _ = pipe.send(i);
|
||||
}
|
||||
let res = pipe.finish();
|
||||
|
||||
std::panic::set_hook(prev);
|
||||
|
||||
match res {
|
||||
Err(Error::IoError { source }) => {
|
||||
let msg = source.to_string();
|
||||
// Constant prefix lets callers match without parsing
|
||||
// the variable payload tail.
|
||||
assert!(
|
||||
msg.contains("pipeline consumer panicked"),
|
||||
"expected constant panic prefix, got: {msg}"
|
||||
);
|
||||
// The original `panic!` payload (a `&'static str`) must
|
||||
// be preserved — without the downcast the message
|
||||
// would just be the prefix.
|
||||
assert!(
|
||||
msg.contains("synthetic test panic"),
|
||||
"expected original panic payload, got: {msg}"
|
||||
);
|
||||
}
|
||||
other => panic!("expected Err(IoError), got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Never-completing sink — `apply` blocks until cancelled. Signals
|
||||
/// `started` once it has consumed its first item so the test
|
||||
/// driver knows the consumer thread is wedged in `apply` (and
|
||||
/// will no longer drain the channel). Used to drive the
|
||||
/// halt/timeout paths of `send_with_halt` and `finish_with_halt`
|
||||
/// without depending on real I/O.
|
||||
struct NeverDrainsSink {
|
||||
cancel: Arc<std::sync::atomic::AtomicBool>,
|
||||
started: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
impl Sink<u64> for NeverDrainsSink {
|
||||
type Output = ();
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
self.started.store(true, Ordering::SeqCst);
|
||||
while !self.cancel.load(Ordering::SeqCst) {
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(self) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Spin until `started` flips or `bail` elapses. Used by the
|
||||
/// send_with_halt tests to synchronise with the consumer thread
|
||||
/// before exercising the bounded-send timeout path.
|
||||
fn wait_for_started(started: &Arc<std::sync::atomic::AtomicBool>, bail: Duration) {
|
||||
let end = Instant::now() + bail;
|
||||
while !started.load(Ordering::SeqCst) {
|
||||
assert!(Instant::now() < end, "consumer never started apply()");
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_with_halt_returns_item_on_deadline() {
|
||||
// depth=1 + consumer wedged in apply on the first item, AND
|
||||
// the channel buffer already loaded with a second item, means
|
||||
// any further `try_send` sees Full; with a 200 ms deadline and
|
||||
// no halt fired, send_with_halt must return `Err(item)` within
|
||||
// roughly the deadline. Synchronising on `started` ensures the
|
||||
// consumer has actually started its wedged apply BEFORE we
|
||||
// load the channel-buffer slot — without that, the consumer
|
||||
// could still drain in a race window.
|
||||
let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipe = Pipeline::spawn(
|
||||
1,
|
||||
NeverDrainsSink {
|
||||
cancel: cancel.clone(),
|
||||
started: started.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
// First send: consumer recv()s it and wedges in apply.
|
||||
pipe.send(0u64).expect("first send hands off to consumer");
|
||||
wait_for_started(&started, Duration::from_secs(2));
|
||||
// Second send: lands in the depth=1 buffer slot, consumer
|
||||
// can't pick it up because it's wedged in apply. Channel now
|
||||
// full from the producer's perspective.
|
||||
pipe.send(1u64).expect("second send fills the buffer");
|
||||
|
||||
let halt = crate::halt::Halt::new();
|
||||
let start = Instant::now();
|
||||
let res = pipe.send_with_halt(99u64, &halt, Duration::from_millis(200));
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Release the leaked consumer so the test process winds down.
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
let _ = pipe.finish();
|
||||
|
||||
assert!(matches!(res, Err(99)), "expected item returned on deadline");
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(150),
|
||||
"deadline returned too early: {elapsed:?}"
|
||||
);
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"deadline blew past tolerance: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_with_halt_returns_item_on_halt() {
|
||||
// Same setup, but the halt fires before the deadline elapses.
|
||||
// The send loop must observe the halt within ~50 ms (the
|
||||
// SEND_POLL_INTERVAL) and return the item.
|
||||
let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipe = Pipeline::spawn(
|
||||
1,
|
||||
NeverDrainsSink {
|
||||
cancel: cancel.clone(),
|
||||
started: started.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
pipe.send(0u64).expect("first send hands off to consumer");
|
||||
wait_for_started(&started, Duration::from_secs(2));
|
||||
pipe.send(1u64).expect("second send fills the buffer");
|
||||
|
||||
let halt = crate::halt::Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
halt2.cancel();
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
let res = pipe.send_with_halt(7u64, &halt, Duration::from_secs(10));
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
let _ = pipe.finish();
|
||||
|
||||
assert!(matches!(res, Err(7)), "expected item returned on halt");
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"halt observation took too long: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_with_halt_returns_halted_when_consumer_wedged() {
|
||||
// Consumer wedges on the first apply; halt fires; finish
|
||||
// returns the documented "pipeline join halted" error rather
|
||||
// than blocking forever.
|
||||
let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipe = Pipeline::spawn(
|
||||
DEFAULT_PIPELINE_DEPTH,
|
||||
NeverDrainsSink {
|
||||
cancel: cancel.clone(),
|
||||
started: started.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
pipe.send(0u64).expect("seed item the consumer wedges on");
|
||||
wait_for_started(&started, Duration::from_secs(2));
|
||||
|
||||
let halt = crate::halt::Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(400));
|
||||
halt2.cancel();
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
let res = pipe.finish_with_halt(Some(&halt));
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Release the leaked consumer so the test process exits cleanly.
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
|
||||
match res {
|
||||
Err(Error::IoError { source }) => {
|
||||
assert!(
|
||||
source.to_string().contains("pipeline join halted"),
|
||||
"expected halt-prefix error, got: {source}"
|
||||
);
|
||||
}
|
||||
other => panic!("expected Err(IoError) halted, got {other:?}"),
|
||||
}
|
||||
// Bailed out within ~1 second of the halt firing (worst case
|
||||
// one POLL_INTERVAL = 250 ms of slack).
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"halt observation took too long: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_with_halt_happy_path_returns_output() {
|
||||
// No halt token, sink completes normally — finish_with_halt
|
||||
// must return the same Output that `finish` would.
|
||||
let pipe = Pipeline::spawn(DEFAULT_PIPELINE_DEPTH, SumSink { total: 0 })
|
||||
.expect("spawn should succeed");
|
||||
for i in 0..10u64 {
|
||||
pipe.send(i).expect("send should succeed");
|
||||
}
|
||||
let total = pipe
|
||||
.finish_with_halt(None)
|
||||
.expect("happy-path finish_with_halt should succeed");
|
||||
assert_eq!(total, (0..10u64).sum::<u64>());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
//! `LocalFileSink` — `BufWriter<File>` for the common local-disk case.
|
||||
//!
|
||||
//! Buffering: 4 MiB internal `BufWriter`. Sized to coalesce the small
|
||||
//! per-PES writes that come out of the muxer into kernel-page-aligned
|
||||
//! flushes without making the buffer big enough to matter for memory
|
||||
//! pressure on a single concurrent rip.
|
||||
//!
|
||||
//! `Seek` flushes the underlying `BufWriter` first; otherwise a seek
|
||||
//! could leapfrog buffered data and silently corrupt the file. This is
|
||||
//! the same shape `BufWriter` itself uses when it impls `Seek` in
|
||||
//! stdlib, and is necessary for MKV's seek-back operations (cluster
|
||||
//! size patch, Cues index, segment header backpatch) to land on the
|
||||
//! right offset.
|
||||
//!
|
||||
//! `RandomAccessSink` is satisfied via the blanket impl in
|
||||
//! [`super::mod`]; no explicit impl needed here.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, BufWriter, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use super::preallocate;
|
||||
|
||||
const BUFFER_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Random-access write sink for local disks.
|
||||
///
|
||||
/// Wraps a `BufWriter<File>` with a 4 MiB internal buffer and forwards
|
||||
/// `Write`/`Seek` so any call site that previously held a `File` or
|
||||
/// `WritebackFile` can drop this in. `finish()` flushes the buffer and
|
||||
/// runs `sync_all` on the underlying file so the caller can drop it
|
||||
/// without losing data.
|
||||
///
|
||||
/// Construction always opens the file `create + truncate + read +
|
||||
/// write`. `read` is enabled so the same handle can be reused for a
|
||||
/// verification re-read after the mux (the existing
|
||||
/// `FileSectorSink::create` pattern). On Linux, [`with_size_hint`]
|
||||
/// additionally calls `fallocate(FALLOC_FL_KEEP_SIZE)` to pre-reserve
|
||||
/// extents.
|
||||
///
|
||||
/// [`with_size_hint`]: Self::with_size_hint
|
||||
pub struct LocalFileSink {
|
||||
inner: BufWriter<File>,
|
||||
}
|
||||
|
||||
impl LocalFileSink {
|
||||
/// Open `path` for writing, truncating any existing contents.
|
||||
pub fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
Ok(Self {
|
||||
inner: BufWriter::with_capacity(BUFFER_BYTES, file),
|
||||
})
|
||||
}
|
||||
|
||||
/// Like [`Self::create`] but additionally calls the per-OS
|
||||
/// preallocate path with `size_bytes`. On Linux this is
|
||||
/// `fallocate(FALLOC_FL_KEEP_SIZE)` so the on-disk extents are
|
||||
/// reserved up front (reducing fragmentation for big sequential
|
||||
/// muxer output); on other OSes it is a no-op today. Failures
|
||||
/// from the preallocate call are non-fatal — the file is still
|
||||
/// returned, just without the size reservation.
|
||||
pub fn with_size_hint(path: &Path, size_bytes: u64) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
preallocate::preallocate(&file, size_bytes);
|
||||
Ok(Self {
|
||||
inner: BufWriter::with_capacity(BUFFER_BYTES, file),
|
||||
})
|
||||
}
|
||||
|
||||
/// Drain the internal buffer and `fsync` the underlying file.
|
||||
/// Idempotent with `Drop` (the `BufWriter` also flushes on drop;
|
||||
/// this call additionally surfaces fsync errors to the caller).
|
||||
#[allow(dead_code)] // exposed for parity with WritebackFile::sync_all
|
||||
pub fn sync_all(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()?;
|
||||
self.inner.get_ref().sync_all()
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for LocalFileSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.inner.write(buf)
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.inner.write_all(buf)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for LocalFileSink {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
// Flush before seeking so buffered bytes land at the offset
|
||||
// they were written for, not the new one.
|
||||
self.inner.flush()?;
|
||||
self.inner.get_mut().seek(from)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
|
||||
#[test]
|
||||
fn write_seek_roundtrip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("rt.bin");
|
||||
let mut s = LocalFileSink::create(&p).unwrap();
|
||||
s.write_all(b"AAAA").unwrap();
|
||||
s.write_all(b"BBBB").unwrap();
|
||||
// Seek back over the second word and overwrite.
|
||||
s.seek(SeekFrom::Start(4)).unwrap();
|
||||
s.write_all(b"CCCC").unwrap();
|
||||
s.sync_all().unwrap();
|
||||
drop(s);
|
||||
|
||||
let mut f = File::open(&p).unwrap();
|
||||
let mut got = Vec::new();
|
||||
f.read_to_end(&mut got).unwrap();
|
||||
assert_eq!(&got[..], b"AAAACCCC");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_flushes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("drop.bin");
|
||||
{
|
||||
let mut s = LocalFileSink::create(&p).unwrap();
|
||||
s.write_all(b"buffered").unwrap();
|
||||
// No explicit flush / sync_all — BufWriter drop runs the
|
||||
// flush and the file should land on disk.
|
||||
}
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..], b"buffered");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_size_hint_creates_writable_file() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("sz.bin");
|
||||
let mut s = LocalFileSink::with_size_hint(&p, 64 * 1024).unwrap();
|
||||
s.write_all(b"hint-ok").unwrap();
|
||||
s.sync_all().unwrap();
|
||||
drop(s);
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..], b"hint-ok");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//! Output-sink trait split for the buffering architecture.
|
||||
//!
|
||||
//! Two traits, one for each capability axis of an output destination:
|
||||
//!
|
||||
//! - [`SequentialSink`] — anything you can `Write` to in order. Sockets,
|
||||
//! pipes, append-only stores, plain files. Containers that don't need
|
||||
//! seek (M2TS, fMP4, HEVC elementary) target this.
|
||||
//! - [`RandomAccessSink`] — everything `SequentialSink` plus a working
|
||||
//! `Seek`. Local files, NFS files, anything with random-write
|
||||
//! semantics. Containers that need backpatch (MKV cluster sizes, Cues
|
||||
//! index, MP4 moov-at-end) target this.
|
||||
//!
|
||||
//! `RandomAccessSink: SequentialSink` — every random-access sink is
|
||||
//! also a valid sequential sink. The muxer is generic over which it
|
||||
//! requires (`MkvMux<S: RandomAccessSink>`, `M2tsMux<S: SequentialSink>`)
|
||||
//! so an attempt to mux MKV to a network socket is a compile error.
|
||||
//!
|
||||
//! Buffering policy belongs to the concrete sink, not to a wrapper at
|
||||
//! the call site. `LocalFileSink` wraps a `BufWriter<File>` with a
|
||||
//! 4 MiB buffer for the common local-disk case; `WritebackFile`
|
||||
//! (separate module) wraps a `File` with the adaptive-chunk
|
||||
//! `sync_file_range` machinery for the Linux+NFS case.
|
||||
//!
|
||||
//! See `(internal)/memory/project_buffering_architecture.md` for
|
||||
//! the full design and the source/sink matrix.
|
||||
|
||||
use std::io::{Seek, Write};
|
||||
|
||||
mod local_file;
|
||||
mod preallocate;
|
||||
mod socket;
|
||||
|
||||
pub use local_file::LocalFileSink;
|
||||
pub use socket::{SocketSink, UdpSocketSink};
|
||||
|
||||
/// Sequential-only write destination. Sockets, pipes, append-only
|
||||
/// stores. No seek. Implementations own their write buffering — the
|
||||
/// trait does not impose or hide any buffering of its own.
|
||||
///
|
||||
/// `finish` drains any internal buffering and signals end-of-stream to
|
||||
/// the underlying transport (close-write on a socket, flush on a
|
||||
/// buffered writer, etc.). The default impl is a no-op; concrete
|
||||
/// implementations that need explicit shutdown can override it but the
|
||||
/// blanket impl below keeps it optional for adapter types like
|
||||
/// `&mut File`.
|
||||
pub trait SequentialSink: Write + Send {
|
||||
fn finish(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Random-access write destination. Local files, NFS files, anything
|
||||
/// with a working `Seek`. Inherits the `SequentialSink` contract — a
|
||||
/// random-access sink is always usable as a sequential sink.
|
||||
pub trait RandomAccessSink: SequentialSink + Seek {}
|
||||
|
||||
// Blanket impls so any `Write + Send` type acts as a `SequentialSink`
|
||||
// (with default `finish`), and any sink that also impls `Seek` is
|
||||
// automatically a `RandomAccessSink`. Keeps call-site ergonomics simple
|
||||
// — `&mut File`, `LocalFileSink`, `WritebackFile`, `BufWriter<File>`,
|
||||
// and `Cursor<Vec<u8>>` all satisfy the right trait without per-type
|
||||
// boilerplate.
|
||||
impl<T: Write + Send + ?Sized> SequentialSink for T {}
|
||||
impl<T: SequentialSink + Seek + ?Sized> RandomAccessSink for T {}
|
||||
|
||||
/// Pick the right `RandomAccessSink` impl for `dest` based on its
|
||||
/// filesystem type.
|
||||
///
|
||||
/// - Linux + NFS path → `WritebackFile` with its adaptive-chunk
|
||||
/// sync_file_range machinery and (when supported) `fallocate` size
|
||||
/// hint.
|
||||
/// - everything else → [`LocalFileSink`] over `BufWriter<File>`. On
|
||||
/// non-Linux there is no `WritebackFile` machinery to opt into, and
|
||||
/// on local Linux the kernel's default writeback policy is already
|
||||
/// fine.
|
||||
///
|
||||
/// `size_hint`, when present, is forwarded to the per-OS preallocate
|
||||
/// path (`fallocate(KEEP_SIZE)` on Linux, `F_PREALLOCATE` on macOS when
|
||||
/// implemented, no-op elsewhere).
|
||||
///
|
||||
/// Returns a boxed trait object so the call site (mux construction)
|
||||
/// stays agnostic of which concrete sink got picked.
|
||||
#[allow(dead_code)] // wiring to mux::resolve is a follow-up commit
|
||||
pub fn open_for_mkv(
|
||||
dest: &std::path::Path,
|
||||
size_hint: Option<u64>,
|
||||
) -> std::io::Result<Box<dyn RandomAccessSink>> {
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
use crate::platform::fs_type::detect;
|
||||
#[cfg(target_os = "linux")]
|
||||
use crate::platform::fs_type::{FsType, detect};
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
if detect(dest) == FsType::Nfs {
|
||||
let wf = match size_hint {
|
||||
Some(n) => crate::io::WritebackFile::create_with_size_hint(dest, n)?,
|
||||
None => crate::io::WritebackFile::create(dest)?,
|
||||
};
|
||||
return Ok(Box::new(wf));
|
||||
}
|
||||
}
|
||||
// Silence the unused-binding warning on non-Linux where the only
|
||||
// branch above is cfg-gated out.
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
let _ = detect(dest);
|
||||
}
|
||||
|
||||
let sink = match size_hint {
|
||||
Some(n) => LocalFileSink::with_size_hint(dest, n)?,
|
||||
None => LocalFileSink::create(dest)?,
|
||||
};
|
||||
Ok(Box::new(sink))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs::File;
|
||||
|
||||
// Type-level assertion: the blanket impls cover the shapes we care
|
||||
// about. These functions never run; they just have to type-check.
|
||||
fn _assert_file_is_sequential(_: &mut dyn SequentialSink) {}
|
||||
fn _assert_file_is_random_access(_: &mut dyn RandomAccessSink) {}
|
||||
|
||||
#[test]
|
||||
fn blanket_impls_cover_file_and_localfilesink() {
|
||||
// `File` directly via blanket impls.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut f = File::create(dir.path().join("a.bin")).unwrap();
|
||||
_assert_file_is_sequential(&mut f);
|
||||
_assert_file_is_random_access(&mut f);
|
||||
|
||||
// `LocalFileSink` ditto.
|
||||
let mut s = LocalFileSink::create(&dir.path().join("b.bin")).unwrap();
|
||||
_assert_file_is_sequential(&mut s);
|
||||
_assert_file_is_random_access(&mut s);
|
||||
|
||||
// `WritebackFile` — confirms the Phase 1 type still satisfies
|
||||
// the trait via the blanket impl without needing an explicit
|
||||
// `impl RandomAccessSink for WritebackFile {}`.
|
||||
let mut wf = crate::io::WritebackFile::create(&dir.path().join("c.bin")).unwrap();
|
||||
_assert_file_is_sequential(&mut wf);
|
||||
_assert_file_is_random_access(&mut wf);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_for_mkv_returns_a_random_access_sink() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("c.bin");
|
||||
let mut sink = open_for_mkv(&p, Some(64 * 1024)).unwrap();
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
sink.write_all(b"hello").unwrap();
|
||||
sink.seek(SeekFrom::Start(0)).unwrap();
|
||||
sink.finish().unwrap();
|
||||
drop(sink);
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..5], b"hello");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
//! Linux `fallocate(FALLOC_FL_KEEP_SIZE)` preallocation.
|
||||
//!
|
||||
//! `KEEP_SIZE` reserves extents without changing the apparent file
|
||||
//! length, which matches the muxer's expectation that writes still grow
|
||||
//! the file naturally.
|
||||
|
||||
use std::fs::File;
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
pub(super) fn preallocate_impl(file: &File, size_bytes: u64) {
|
||||
let fd = file.as_raw_fd();
|
||||
// FALLOC_FL_KEEP_SIZE = 0x01.
|
||||
let rc = unsafe { libc::fallocate(fd, libc::FALLOC_FL_KEEP_SIZE, 0, size_bytes as i64) };
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink fallocate size_hint={size_bytes} rc={rc} ok={}",
|
||||
rc == 0
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
//! macOS `F_PREALLOCATE` extent reservation.
|
||||
//!
|
||||
//! `fcntl(F_PREALLOCATE)` with `F_ALLOCATECONTIG` first (try for a
|
||||
//! contiguous run) and fall back to `F_ALLOCATEALL` (non-contig OK).
|
||||
//! Reported file size is unchanged — the muxer's writes still grow it.
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
// Mirror the Darwin `fstore_t` struct from `<sys/fcntl.h>`. libc on
|
||||
// some Rust toolchains/versions doesn't ship this binding, so define
|
||||
// it locally with the layout the kernel ABI requires.
|
||||
#[repr(C)]
|
||||
struct Fstore {
|
||||
fst_flags: libc::c_uint,
|
||||
fst_posmode: libc::c_int,
|
||||
fst_offset: libc::off_t,
|
||||
fst_length: libc::off_t,
|
||||
fst_bytesalloc: libc::off_t,
|
||||
}
|
||||
|
||||
// Constants from <sys/fcntl.h>.
|
||||
const F_PREALLOCATE: libc::c_int = 42;
|
||||
const F_ALLOCATECONTIG: libc::c_uint = 0x0000_0002;
|
||||
const F_ALLOCATEALL: libc::c_uint = 0x0000_0004;
|
||||
const F_PEOFPOSMODE: libc::c_int = 3;
|
||||
|
||||
pub(super) fn preallocate_impl(file: &File, size_bytes: u64) {
|
||||
let fd = file.as_raw_fd();
|
||||
let mut store = Fstore {
|
||||
fst_flags: F_ALLOCATECONTIG,
|
||||
fst_posmode: F_PEOFPOSMODE,
|
||||
fst_offset: 0,
|
||||
fst_length: size_bytes as libc::off_t,
|
||||
fst_bytesalloc: 0,
|
||||
};
|
||||
let mut rc = unsafe { libc::fcntl(fd, F_PREALLOCATE, &mut store as *mut Fstore) };
|
||||
if rc == -1 {
|
||||
// Fall back to non-contiguous.
|
||||
store.fst_flags = F_ALLOCATEALL;
|
||||
rc = unsafe { libc::fcntl(fd, F_PREALLOCATE, &mut store as *mut Fstore) };
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink F_PREALLOCATE size_hint={size_bytes} rc={rc} bytesalloc={}",
|
||||
store.fst_bytesalloc
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! Per-OS extent preallocation. Best-effort; failures are logged at
|
||||
//! debug and otherwise swallowed because the file is still usable
|
||||
//! without the size reservation — only large-file fragmentation gets
|
||||
//! marginally worse.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
mod other;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux::preallocate_impl;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos::preallocate_impl;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
use other::preallocate_impl;
|
||||
|
||||
/// Reserve `size_bytes` of disk space for `file`'s on-disk extents.
|
||||
/// Reported file size is unchanged — writes still grow the file
|
||||
/// naturally; only the allocator's extent map is primed.
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
preallocate_impl(file, size_bytes);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
//! Fallback preallocate impl. No-op.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn preallocate_impl(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink preallocate size_hint={size_bytes} skipped (no platform impl)"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
//! TCP / UDP socket sinks (sequential-only).
|
||||
//!
|
||||
//! [`SocketSink`] wraps a `TcpStream` in a 1 MiB `BufWriter`. Constructor
|
||||
//! tunes `SO_SNDBUF` to a caller hint when provided. `finish()` flushes
|
||||
//! the buffer then `shutdown(Write)`s the socket so the peer sees clean
|
||||
//! end-of-stream.
|
||||
//!
|
||||
//! [`UdpSocketSink`] wraps a connected `UdpSocket`. Each `write` call
|
||||
//! emits exactly one datagram — the caller is responsible for packetizing
|
||||
//! to a reasonable MTU (188 × 7 = 1316 bytes for MPEG-TS-over-UDP is the
|
||||
//! conventional choice). `finish()` is a no-op; UDP has no end-of-stream
|
||||
//! marker.
|
||||
//!
|
||||
//! Both types satisfy [`SequentialSink`] via the blanket impl in
|
||||
//! `super::mod`. Neither implements `Seek`, so neither satisfies
|
||||
//! [`RandomAccessSink`] — using one with `MkvMux` is a compile error,
|
||||
//! which is the design intent.
|
||||
//!
|
||||
//! [`SequentialSink`]: super::SequentialSink
|
||||
//! [`RandomAccessSink`]: super::RandomAccessSink
|
||||
|
||||
use std::io::{self, BufWriter, Write};
|
||||
use std::net::{Shutdown, TcpStream, ToSocketAddrs, UdpSocket};
|
||||
|
||||
/// `BufWriter` capacity for [`SocketSink`]. 1 MiB matches the typical
|
||||
/// kernel send-buffer ceiling and keeps small-write amplification from
|
||||
/// containers (TS = 188-byte packets, fMP4 fragment headers = ~100 bytes)
|
||||
/// from translating into syscall storms.
|
||||
const TCP_BUF_CAPACITY: usize = 1024 * 1024;
|
||||
|
||||
/// Sequential-only sink over a TCP connection.
|
||||
///
|
||||
/// Wraps a `BufWriter<TcpStream>`; the inner `TcpStream` is kept as a
|
||||
/// clone so [`finish`](Self::finish) can call `shutdown(Write)` after
|
||||
/// flushing the buffer (the buffered writer doesn't expose the socket
|
||||
/// directly).
|
||||
pub struct SocketSink {
|
||||
/// Buffered write half. All payload bytes go through this.
|
||||
buf: BufWriter<TcpStream>,
|
||||
/// Shutdown handle — clone of the socket inside `buf`. Used only by
|
||||
/// `finish()` for `shutdown(Write)`; never read or written through.
|
||||
shutdown_handle: TcpStream,
|
||||
}
|
||||
|
||||
impl SocketSink {
|
||||
/// Open a TCP connection to `addr` and wrap it for sequential
|
||||
/// writing. `sndbuf_bytes`, when present, is forwarded to
|
||||
/// `setsockopt(SO_SNDBUF)` as a kernel hint — the OS may clamp it.
|
||||
///
|
||||
/// `addr` accepts anything `ToSocketAddrs` does: `"10.0.0.1:1234"`,
|
||||
/// `("host", 1234)`, a `SocketAddr`, etc.
|
||||
pub fn connect<A: ToSocketAddrs>(addr: A, sndbuf_bytes: Option<usize>) -> io::Result<Self> {
|
||||
let stream = TcpStream::connect(addr)?;
|
||||
// `set_nodelay(true)` keeps small writes (TS packet trains, fMP4
|
||||
// moof headers) from sitting in Nagle's algorithm until the buffer
|
||||
// fills. The BufWriter already absorbs syscall overhead; Nagle
|
||||
// would just add latency without coalescing more.
|
||||
stream.set_nodelay(true)?;
|
||||
if let Some(n) = sndbuf_bytes {
|
||||
set_send_buffer(&stream, n)?;
|
||||
}
|
||||
let shutdown_handle = stream.try_clone()?;
|
||||
Ok(Self {
|
||||
buf: BufWriter::with_capacity(TCP_BUF_CAPACITY, stream),
|
||||
shutdown_handle,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for SocketSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.buf.write(buf)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.buf.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl SocketSink {
|
||||
/// Drain the BufWriter and `shutdown(Write)` the underlying socket
|
||||
/// so the peer sees a clean EOF.
|
||||
///
|
||||
/// Note: [`SequentialSink::finish`](super::SequentialSink::finish)'s
|
||||
/// blanket-impl default is a no-op. Trait-object call sites that
|
||||
/// need socket shutdown should call this inherent method directly
|
||||
/// before dropping the sink, or hold the concrete `SocketSink` type
|
||||
/// (typical pattern: each muxer's `finish()` calls the appropriate
|
||||
/// inherent close method on its captured concrete sink).
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
self.buf.flush()?;
|
||||
// `shutdown(Write)` signals clean EOF to the peer. Errors here
|
||||
// are non-fatal — the connection may have already been torn down
|
||||
// by the peer — but we surface them so callers can log.
|
||||
self.shutdown_handle.shutdown(Shutdown::Write)
|
||||
}
|
||||
}
|
||||
|
||||
/// Sequential-only sink over a connected UDP socket.
|
||||
///
|
||||
/// Each [`write`](Write::write) call sends exactly one datagram. The
|
||||
/// caller is responsible for splitting payload at packet boundaries —
|
||||
/// for MPEG-TS this means 7 × 188 = 1316 bytes per datagram, the
|
||||
/// industry standard for MPEG-TS-over-UDP. No buffering happens here;
|
||||
/// adding it would silently merge datagrams.
|
||||
///
|
||||
/// `finish()` is a no-op: UDP has no end-of-stream marker. Closing the
|
||||
/// socket happens on drop.
|
||||
pub struct UdpSocketSink {
|
||||
socket: UdpSocket,
|
||||
}
|
||||
|
||||
impl UdpSocketSink {
|
||||
/// Bind a local UDP socket to an ephemeral port and `connect` it to
|
||||
/// `peer`. `connect` doesn't open a connection — it just fixes the
|
||||
/// peer address so subsequent `send` calls don't need to repeat it,
|
||||
/// and so receive-side filtering rejects packets from other sources.
|
||||
///
|
||||
/// `sndbuf_bytes`, when present, is a hint to `SO_SNDBUF`.
|
||||
pub fn connect<A: ToSocketAddrs>(peer: A, sndbuf_bytes: Option<usize>) -> io::Result<Self> {
|
||||
// Bind to all-zeros / any port. The kernel picks an ephemeral
|
||||
// source port and the source IP at first send.
|
||||
let socket = UdpSocket::bind("0.0.0.0:0")?;
|
||||
socket.connect(peer)?;
|
||||
if let Some(n) = sndbuf_bytes {
|
||||
set_udp_send_buffer(&socket, n)?;
|
||||
}
|
||||
Ok(Self { socket })
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for UdpSocketSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
// `send` writes the entire datagram or fails — no partial sends
|
||||
// for UDP. Match `Write::write`'s contract by reporting bytes
|
||||
// accepted.
|
||||
self.socket.send(buf)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl UdpSocketSink {
|
||||
/// No-op — UDP has no end-of-stream marker. Provided for parity
|
||||
/// with [`SocketSink::finish`] so call sites can treat them uniformly.
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ── Platform `SO_SNDBUF` tuning ────────────────────────────────────────────
|
||||
//
|
||||
// std's `TcpStream` / `UdpSocket` don't expose `SO_SNDBUF`. We drop to
|
||||
// libc on Linux + macOS (the libc-dep targets in Cargo.toml). On other
|
||||
// targets the hint is silently ignored — the socket still works, the
|
||||
// kernel just picks its own send-buffer size.
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn set_send_buffer(stream: &TcpStream, bytes: usize) -> io::Result<()> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
setsockopt_sndbuf(stream.as_raw_fd(), bytes)
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn set_udp_send_buffer(socket: &UdpSocket, bytes: usize) -> io::Result<()> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
setsockopt_sndbuf(socket.as_raw_fd(), bytes)
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
fn set_send_buffer(_stream: &TcpStream, _bytes: usize) -> io::Result<()> {
|
||||
// Non-Linux-non-macOS targets aren't in Cargo.toml's libc dep list;
|
||||
// silently ignore the hint rather than failing the connect. Callers
|
||||
// can detect via the lack of an explicit "sndbuf applied" signal
|
||||
// (not provided, intentionally — this is a hint, not a guarantee).
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
fn set_udp_send_buffer(_socket: &UdpSocket, _bytes: usize) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn setsockopt_sndbuf(fd: std::os::unix::io::RawFd, bytes: usize) -> io::Result<()> {
|
||||
// Clamp into c_int range; SO_SNDBUF takes an `int` argument.
|
||||
let want: libc::c_int = bytes.try_into().unwrap_or(libc::c_int::MAX);
|
||||
let ret = unsafe {
|
||||
libc::setsockopt(
|
||||
fd,
|
||||
libc::SOL_SOCKET,
|
||||
libc::SO_SNDBUF,
|
||||
&want as *const _ as *const libc::c_void,
|
||||
std::mem::size_of::<libc::c_int>() as libc::socklen_t,
|
||||
)
|
||||
};
|
||||
if ret != 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
use std::net::{TcpListener, UdpSocket};
|
||||
use std::thread;
|
||||
|
||||
/// Bind a listener, accept on a thread, return (listener_addr,
|
||||
/// accepted-bytes future via JoinHandle).
|
||||
#[test]
|
||||
fn socket_sink_round_trips_bytes() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let accept = thread::spawn(move || {
|
||||
let (mut sock, _) = listener.accept().unwrap();
|
||||
let mut buf = Vec::new();
|
||||
sock.read_to_end(&mut buf).unwrap();
|
||||
buf
|
||||
});
|
||||
|
||||
let mut sink = SocketSink::connect(addr, Some(256 * 1024)).unwrap();
|
||||
// Write enough to overflow the BufWriter at least once, then a
|
||||
// tail that lives in the buffer until `finish` flushes.
|
||||
let big: Vec<u8> = (0..(2 * TCP_BUF_CAPACITY))
|
||||
.map(|i| (i & 0xff) as u8)
|
||||
.collect();
|
||||
sink.write_all(&big).unwrap();
|
||||
sink.write_all(b"tail\n").unwrap();
|
||||
sink.finish().unwrap();
|
||||
drop(sink);
|
||||
|
||||
let received = accept.join().unwrap();
|
||||
assert_eq!(received.len(), big.len() + 5);
|
||||
assert_eq!(&received[..big.len()], &big[..]);
|
||||
assert_eq!(&received[big.len()..], b"tail\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn socket_sink_is_sequential_only() {
|
||||
// Compile-time assertion via dyn — if this ever started
|
||||
// satisfying `RandomAccessSink`, the trait split would be broken.
|
||||
fn _assert_seq(_: &mut dyn super::super::SequentialSink) {}
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let _accept = thread::spawn(move || {
|
||||
let _ = listener.accept();
|
||||
});
|
||||
let mut sink = SocketSink::connect(addr, None).unwrap();
|
||||
_assert_seq(&mut sink);
|
||||
// The negative is harder to assert directly (no `is_not<T>`),
|
||||
// but `SocketSink` does not impl `Seek`, so it can't unify with
|
||||
// `RandomAccessSink`'s super-bound. The Phase 2 blanket impl
|
||||
// `impl<T: SequentialSink + Seek> RandomAccessSink for T {}` thus
|
||||
// excludes it by construction.
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn udp_socket_sink_delivers_datagrams() {
|
||||
let receiver = UdpSocket::bind("127.0.0.1:0").unwrap();
|
||||
receiver
|
||||
.set_read_timeout(Some(std::time::Duration::from_secs(2)))
|
||||
.unwrap();
|
||||
let addr = receiver.local_addr().unwrap();
|
||||
let mut sink = UdpSocketSink::connect(addr, Some(128 * 1024)).unwrap();
|
||||
|
||||
sink.write_all(&[1, 2, 3, 4, 5]).unwrap();
|
||||
sink.write_all(&[9, 9, 9]).unwrap();
|
||||
sink.finish().unwrap();
|
||||
|
||||
let mut buf = [0u8; 64];
|
||||
let n1 = receiver.recv(&mut buf).unwrap();
|
||||
assert_eq!(&buf[..n1], &[1, 2, 3, 4, 5]);
|
||||
let n2 = receiver.recv(&mut buf).unwrap();
|
||||
assert_eq!(&buf[..n2], &[9, 9, 9]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
//! Per-platform writeback pipeline. On Linux, drains dirty pages
|
||||
//! continuously at chunk granularity to keep the kernel's writeback
|
||||
//! queue bounded. On macOS and Windows, a no-op stub.
|
||||
//!
|
||||
//! The platform decision lives entirely in this file (the cfg-gated
|
||||
//! `pub use` below). Callers — and `DiskWriter` itself — are
|
||||
//! platform-independent.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
mod noop;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) use linux::WritebackPipeline;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub(super) use noop::WritebackPipeline;
|
||||
@@ -0,0 +1,389 @@
|
||||
//! Linux writeback pipeline using `sync_file_range` + `posix_fadvise`.
|
||||
//!
|
||||
//! Pathology this fixes: the kernel's default `vm.dirty_ratio` (~20 %
|
||||
//! of RAM) lets dirty pages accumulate to hundreds of MB during a
|
||||
//! big sequential write, then bursts a flush at 99 % disk utilisation.
|
||||
//! While the burst runs, app writes block on the writeback queue —
|
||||
//! observed empirically as instantaneous speed dropping from ~15 MB/s
|
||||
//! to ~1 MB/s every ~30 s during a Pass 1 sweep.
|
||||
//!
|
||||
//! Strategy: every `chunk_bytes` of new sequential output, kick async
|
||||
//! writeback (`SYNC_FILE_RANGE_WRITE`) on the just-completed chunk and
|
||||
//! finalise the *previous* chunk via `WAIT_AFTER` + `posix_fadvise
|
||||
//! (DONTNEED)`. By the time we finalise, that previous chunk has had
|
||||
//! a full chunk's worth of work to flush — the wait is near-instant.
|
||||
//! Dirty cache stays bounded at ~2 × `chunk_bytes` and writes drain
|
||||
//! continuously instead of in bursts.
|
||||
//!
|
||||
//! The chunk size is adaptive: we measure the elapsed time of the
|
||||
//! `WAIT_AFTER` call over a rolling window of the last 16 chunks and
|
||||
//! resize the chunk based on the p95. Slow storage (NFS, network
|
||||
//! shares, HDD) sees larger chunks to amortise per-chunk overhead;
|
||||
//! fast storage (NVMe) sees smaller chunks to keep cache pressure
|
||||
//! tight. Bounds: [4 MiB, 256 MiB].
|
||||
//!
|
||||
//! ## Medium-agnostic by construction
|
||||
//!
|
||||
//! `sync_file_range(WAIT_AFTER)` runs on every medium (local FS, NFS,
|
||||
//! etc.). It bounds the dirty-page set at ~2 × `chunk_bytes` by
|
||||
//! waiting for each chunk's kernel writeback to complete before
|
||||
//! advancing. There is no `if is_nfs` branch in the hot path. The
|
||||
//! safety net described below ([`WAIT_AFTER_TIMEOUT`]) handles wedged
|
||||
//! filesystems generically: if a particular FS proves unable to ack
|
||||
//! `WAIT_AFTER` within the deadline, the pipeline flips to `degraded`
|
||||
//! and skips for the rest of its life. That covers the original
|
||||
//! NFS-hang concern (network partition, server stuck on commit)
|
||||
//! without baking the failure mode into a per-FS branch.
|
||||
//!
|
||||
//! **No `posix_fadvise(DONTNEED)` after the wait.** Earlier revisions
|
||||
//! evicted the just-WAIT_AFTER'd range from the page cache on the
|
||||
//! theory that bounding *dirty* pages required also bounding *clean*
|
||||
//! pages. That was unnecessary and counterproductive: clean cached
|
||||
//! pages cost nothing (the kernel reclaims them under LRU when memory
|
||||
//! is needed) and DONTNEED forces a read-modify-write on any
|
||||
//! subsequent in-window write — exactly the pattern matroska
|
||||
//! cluster-size backpatches produce, costing ~40-50% of measured NFS
|
||||
//! write bandwidth in empirical tests 2026-05-15/16.
|
||||
//!
|
||||
//! Medium detection happens *exactly once*, at construction, and
|
||||
//! produces *exactly one* value: the initial `chunk_bytes` seed for
|
||||
//! the autotuner. The detector's job is to give the autotuner a
|
||||
//! decent starting point on cold start; from there p95 of WAIT_AFTER
|
||||
//! latency drives all subsequent decisions, identically on every
|
||||
//! medium.
|
||||
//!
|
||||
//! ## Defence in depth: WAIT_AFTER timeout
|
||||
//!
|
||||
//! A degraded disk, wedged NFS server, or odd FS driver could in
|
||||
//! principle hang inside WAIT_AFTER. Each call runs on a worker
|
||||
//! thread with a 30s recv_timeout on its result channel
|
||||
//! ([`crate::io::bounded::bounded_syscall`]). On timeout the pipeline
|
||||
//! flips to `degraded`, logs a loud error, and skips WAIT_AFTER for
|
||||
//! the rest of its life. The worker thread is intentionally leaked —
|
||||
//! it unwinds whenever the syscall eventually returns or the process
|
||||
//! exits. The mux continues; the original dirty-burst pathology
|
||||
//! re-emerges only on the specific (medium, server, FS) combination
|
||||
//! that actually wedged.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::{AsRawFd, RawFd};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const ADAPTIVE_WINDOW: usize = 16;
|
||||
const CHUNK_BYTES_MIN: u64 = 4 * 1024 * 1024;
|
||||
const CHUNK_BYTES_MAX: u64 = 256 * 1024 * 1024;
|
||||
const ADAPTIVE_GROW_MS: u64 = 200;
|
||||
const ADAPTIVE_SHRINK_MS: u64 = 20;
|
||||
/// Every N chunks, emit a `debug!` snapshot of the current chunk
|
||||
/// size so operators tailing the log can see where the autoscaler
|
||||
/// settled.
|
||||
const SIZE_LOG_INTERVAL: u64 = 32;
|
||||
/// Hard upper bound on a single `sync_file_range(WAIT_AFTER)` call.
|
||||
/// Beyond this we declare the pipeline degraded and stop calling
|
||||
/// WAIT_AFTER for the rest of its life.
|
||||
const WAIT_AFTER_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
pub(crate) struct WritebackPipeline {
|
||||
/// Aliases the wrapping `WritebackFile::file`. Only valid for the
|
||||
/// lifetime of that struct — moving the `File` independently
|
||||
/// would silently UAF this fd. The pipeline is a private field of
|
||||
/// `WritebackFile` and never exposed outside that wrapper, which
|
||||
/// is what keeps the alias sound.
|
||||
fd: RawFd,
|
||||
chunk_bytes: u64,
|
||||
last_flush_pos: u64,
|
||||
pending: Option<(u64, u64)>,
|
||||
/// Rolling window of recent `WAIT_AFTER` elapsed_ms measurements.
|
||||
wait_after_window: VecDeque<u64>,
|
||||
/// Count of chunks emitted (used to space out periodic
|
||||
/// `debug!` size snapshots).
|
||||
chunk_count: u64,
|
||||
/// What `detect_storage_class` saw at construction. Carried for
|
||||
/// diagnostic logging only — *not* keyed off in the hot path.
|
||||
storage_class: StorageClass,
|
||||
/// Set the first time WAIT_AFTER exceeds [`WAIT_AFTER_TIMEOUT`].
|
||||
/// Once set, behaviour matches the NFS path for the rest of the
|
||||
/// pipeline's life. Wrapped in `Arc` only because both this
|
||||
/// struct and the spawned worker thread (which itself doesn't
|
||||
/// touch the flag) share-via-fd patterns might one day need it;
|
||||
/// today it's effectively a single-owner cell — the `Arc` shape
|
||||
/// keeps the door open for moving the read side into a worker
|
||||
/// without re-plumbing types.
|
||||
degraded: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl WritebackPipeline {
|
||||
/// Construct a pipeline aliasing `file`'s file descriptor. The
|
||||
/// returned `WritebackPipeline` MUST be dropped before `file`
|
||||
/// itself, or kept inside the same struct that owns `file` — the
|
||||
/// alias is unchecked.
|
||||
///
|
||||
/// `chunk_bytes_hint` is the caller's preferred starting chunk
|
||||
/// size; the constructor overrides it with a medium-specific
|
||||
/// initial seed if `detect_storage_class` recognises the fd's
|
||||
/// filesystem. The autotuner then drives the value from there
|
||||
/// based on measured `WAIT_AFTER` latency, identically on every
|
||||
/// medium.
|
||||
pub(crate) fn new(file: &File, start_pos: u64, chunk_bytes_hint: u64) -> Self {
|
||||
let fd = file.as_raw_fd();
|
||||
let (class, seed) = detect_storage_class(fd);
|
||||
// Medium-aware initial seed: bias toward bigger chunks on
|
||||
// slow-commit media so the autotuner doesn't waste a minute
|
||||
// climbing from a too-small starting value. A returned seed
|
||||
// of 0 means "the detector has no opinion; use the caller's
|
||||
// hint". All subsequent tuning is medium-agnostic.
|
||||
let chunk_bytes = if seed > 0 { seed } else { chunk_bytes_hint };
|
||||
tracing::info!(
|
||||
target: "mux",
|
||||
"WritebackPipeline fd={fd} storage_class={class:?} chunk_bytes={chunk_bytes}"
|
||||
);
|
||||
Self {
|
||||
fd,
|
||||
chunk_bytes,
|
||||
last_flush_pos: start_pos,
|
||||
pending: None,
|
||||
wait_after_window: VecDeque::with_capacity(ADAPTIVE_WINDOW),
|
||||
chunk_count: 0,
|
||||
storage_class: class,
|
||||
degraded: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
/// True if we should bypass the WAIT_AFTER finalisation step. The
|
||||
/// pipeline only skips after `WAIT_AFTER` has *empirically* hung
|
||||
/// past [`WAIT_AFTER_TIMEOUT`] on this particular fd. No per-FS
|
||||
/// branch — the failure is detected, not predicted.
|
||||
#[inline]
|
||||
fn skip_wait(&self) -> bool {
|
||||
self.degraded.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Caller advanced the file position to `pos`. If a chunk boundary
|
||||
/// was crossed, kick async writeback for the just-completed chunk
|
||||
/// and finalise the previous one.
|
||||
pub(crate) fn note_progress(&mut self, pos: u64) {
|
||||
if pos < self.last_flush_pos.saturating_add(self.chunk_bytes) {
|
||||
return;
|
||||
}
|
||||
let chunk_off = self.last_flush_pos as i64;
|
||||
let chunk_len = (pos - self.last_flush_pos) as i64;
|
||||
let mut wait_ms: u64 = 0;
|
||||
let mut fadvise_ms: u64 = 0;
|
||||
// Async kickoff for the just-completed chunk runs on every
|
||||
// path (NFS, degraded, normal) — it's nominally non-blocking
|
||||
// by spec and gives the kernel an early hint that this range
|
||||
// is ready to flush.
|
||||
unsafe {
|
||||
libc::sync_file_range(self.fd, chunk_off, chunk_len, libc::SYNC_FILE_RANGE_WRITE);
|
||||
}
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
if self.skip_wait() {
|
||||
// Degraded path (set only after an empirical
|
||||
// WAIT_AFTER timeout on this fd). We still advance
|
||||
// `pending` so the next call has a stable cycle.
|
||||
} else {
|
||||
match wait_after_with_timeout(self.fd, prev_off, prev_len) {
|
||||
Some(ms) => {
|
||||
wait_ms = ms;
|
||||
// Deliberately NO `posix_fadvise(DONTNEED)`
|
||||
// here. See module-level docs: evicting clean
|
||||
// pages costs us read-modify-write on
|
||||
// matroska cluster backpatches and gains
|
||||
// nothing — the kernel will reclaim clean
|
||||
// pages under LRU when memory is needed.
|
||||
self.record_wait(wait_ms);
|
||||
}
|
||||
None => {
|
||||
// Timeout branch: this fd's FS or server is
|
||||
// unable to ack WAIT_AFTER. Flip to skip mode
|
||||
// for the rest of the pipeline's life.
|
||||
self.degraded.store(true, Ordering::Relaxed);
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackPipeline WAIT_AFTER timed out after {}s on chunk off={} len={}, marking writeback degraded (subsequent chunks will skip WAIT_AFTER)",
|
||||
WAIT_AFTER_TIMEOUT.as_secs(),
|
||||
prev_off,
|
||||
prev_len
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.pending = Some((chunk_off as u64, chunk_len as u64));
|
||||
self.last_flush_pos = pos;
|
||||
self.chunk_count += 1;
|
||||
let _ = fadvise_ms;
|
||||
tracing::trace!(
|
||||
target: "mux",
|
||||
"WritebackPipeline chunk off={} len={} sync_file_range_ms={wait_ms} chunk_bytes={} skip_wait={}",
|
||||
chunk_off,
|
||||
chunk_len,
|
||||
self.chunk_bytes,
|
||||
self.skip_wait(),
|
||||
);
|
||||
if self.chunk_count % SIZE_LOG_INTERVAL == 0 {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackPipeline chunk_bytes={} after {} chunks storage_class={:?} degraded={}",
|
||||
self.chunk_bytes,
|
||||
self.chunk_count,
|
||||
self.storage_class,
|
||||
self.degraded.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Push a new `WAIT_AFTER` measurement into the rolling window
|
||||
/// and, if the window is full, adapt `chunk_bytes` based on p95.
|
||||
fn record_wait(&mut self, wait_ms: u64) {
|
||||
if self.wait_after_window.len() == ADAPTIVE_WINDOW {
|
||||
self.wait_after_window.pop_front();
|
||||
}
|
||||
self.wait_after_window.push_back(wait_ms);
|
||||
if self.wait_after_window.len() < ADAPTIVE_WINDOW {
|
||||
return;
|
||||
}
|
||||
// p95 of 16 samples ≈ sorted[14] (5 % of 16 = 0.8 ≈ 1 above).
|
||||
let mut sorted: Vec<u64> = self.wait_after_window.iter().copied().collect();
|
||||
sorted.sort_unstable();
|
||||
let p95 = sorted[14];
|
||||
let old = self.chunk_bytes;
|
||||
let new = if p95 > ADAPTIVE_GROW_MS && self.chunk_bytes < CHUNK_BYTES_MAX {
|
||||
(self.chunk_bytes * 2).min(CHUNK_BYTES_MAX)
|
||||
} else if p95 < ADAPTIVE_SHRINK_MS && self.chunk_bytes > CHUNK_BYTES_MIN {
|
||||
(self.chunk_bytes / 2).max(CHUNK_BYTES_MIN)
|
||||
} else {
|
||||
self.chunk_bytes
|
||||
};
|
||||
if new != old {
|
||||
self.chunk_bytes = new;
|
||||
tracing::info!(
|
||||
target: "mux",
|
||||
"WritebackPipeline adaptive chunk_bytes {} -> {} p95_ms={p95}",
|
||||
old,
|
||||
new
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Caller is about to seek away from the current write region.
|
||||
/// Drain any in-flight chunk and reset tracking.
|
||||
pub(crate) fn handle_seek(&mut self, new_pos: u64) {
|
||||
self.finalize();
|
||||
self.last_flush_pos = new_pos;
|
||||
}
|
||||
|
||||
/// Drain any in-flight chunk. Idempotent. Call before `sync_all()`
|
||||
/// or when discarding the pipeline.
|
||||
pub(crate) fn finalize(&mut self) {
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackPipeline finalize chunk off={prev_off} len={prev_len} skip_wait={} storage_class={:?} degraded={}",
|
||||
self.skip_wait(),
|
||||
self.storage_class,
|
||||
self.degraded.load(Ordering::Relaxed),
|
||||
);
|
||||
if self.skip_wait() {
|
||||
return;
|
||||
}
|
||||
if wait_after_with_timeout(self.fd, prev_off, prev_len).is_none() {
|
||||
self.degraded.store(true, Ordering::Relaxed);
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackPipeline finalize WAIT_AFTER timed out after {}s on chunk off={prev_off} len={prev_len}, marking writeback degraded",
|
||||
WAIT_AFTER_TIMEOUT.as_secs(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Coarse classification of the medium an open file is on. Used only
|
||||
/// at construction to seed the autotuner with a sensible initial
|
||||
/// `chunk_bytes`. Never branched on in the hot path.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum StorageClass {
|
||||
/// `fstatfs.f_type == NFS_SUPER_MAGIC`. Slow-commit; seed a
|
||||
/// generous initial chunk so the autotuner doesn't waste cycles
|
||||
/// climbing from a too-small value.
|
||||
Nfs,
|
||||
/// Anything else we successfully stat'd. Local FS, tmpfs, ZFS,
|
||||
/// network FS we don't have a constant for, etc. Use the caller's
|
||||
/// hint as the seed.
|
||||
Other,
|
||||
/// `fstatfs` failed. Caller's hint wins.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Probe the FS class via `fstatfs` and return both the class label
|
||||
/// and a recommended `chunk_bytes` seed. `seed == 0` means "no
|
||||
/// medium-specific recommendation; use the caller's hint." Failure
|
||||
/// modes default to `Unknown` + seed 0 so the caller's hint applies.
|
||||
///
|
||||
/// This is the single medium-detection point in the whole writeback
|
||||
/// pipeline. Everything else is medium-agnostic.
|
||||
fn detect_storage_class(fd: RawFd) -> (StorageClass, u64) {
|
||||
// `libc::statfs` is repr(C) with a fixed layout; zeroing is the
|
||||
// documented init pattern for the kernel uapi struct.
|
||||
let mut buf: libc::statfs = unsafe { std::mem::zeroed() };
|
||||
let rc = unsafe { libc::fstatfs(fd, &mut buf) };
|
||||
if rc != 0 {
|
||||
let errno = std::io::Error::last_os_error();
|
||||
tracing::warn!(
|
||||
target: "mux",
|
||||
"WritebackPipeline fstatfs(fd={fd}) failed: {errno} — defaulting to Unknown",
|
||||
);
|
||||
return (StorageClass::Unknown, 0);
|
||||
}
|
||||
// `f_type` is signed (`__fsword_t`) on glibc and unsigned
|
||||
// (`c_ulong`) on musl. Cast both sides to i64 for a portable
|
||||
// comparison. On glibc x86_64 both already are i64 — clippy flags
|
||||
// the cast as unnecessary on that target only, but we need it for
|
||||
// musl, so silence the lint.
|
||||
#[allow(clippy::unnecessary_cast)]
|
||||
let f_type = buf.f_type as i64;
|
||||
#[allow(clippy::unnecessary_cast)]
|
||||
let nfs_magic = libc::NFS_SUPER_MAGIC as i64;
|
||||
if f_type == nfs_magic {
|
||||
// NFS commit ack typically ~10-30 ms RTT. The autotuner grows
|
||||
// by doubling when p95 > 200 ms — so starting at 32 MiB it
|
||||
// would never grow on a healthy NFS. Seed at 64 MiB so the
|
||||
// commit cadence amortizes properly from the first chunk.
|
||||
(StorageClass::Nfs, 64 * 1024 * 1024)
|
||||
} else {
|
||||
// Local FS / unknown remote FS. Caller's hint (typically 32
|
||||
// MiB) is a fine starting point; autotuner adjusts from
|
||||
// measured latency.
|
||||
(StorageClass::Other, 0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Run `sync_file_range(WAIT_AFTER)` on a worker thread and wait up
|
||||
/// to [`WAIT_AFTER_TIMEOUT`] for it to return. `Some(elapsed_ms)` on
|
||||
/// success; `None` on timeout. On timeout the worker thread is
|
||||
/// 0.20.6 generalizes the worker-thread + recv_timeout pattern into
|
||||
/// [`crate::io::bounded::bounded_syscall`]; this helper now just adapts
|
||||
/// the generic primitive to the WAIT_AFTER call shape (returns elapsed_ms
|
||||
/// instead of the syscall's `()` return, treats `WorkerLost` as a benign
|
||||
/// no-op to match the original semantics).
|
||||
fn wait_after_with_timeout(fd: RawFd, off: u64, len: u64) -> Option<u64> {
|
||||
let started = Instant::now();
|
||||
match crate::io::bounded::bounded_syscall(None, WAIT_AFTER_TIMEOUT, move || unsafe {
|
||||
libc::sync_file_range(fd, off as i64, len as i64, libc::SYNC_FILE_RANGE_WAIT_AFTER);
|
||||
}) {
|
||||
Ok(()) => Some(started.elapsed().as_millis() as u64),
|
||||
Err(crate::io::bounded::BoundedError::Timeout)
|
||||
| Err(crate::io::bounded::BoundedError::Halted) => None,
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => {
|
||||
// Worker thread spawn failed or panicked before sending.
|
||||
// Treat as a benign success (no syscall ran) rather than
|
||||
// a degrade trigger — falling through with elapsed_ms=0
|
||||
// matches the no-op behaviour.
|
||||
Some(0)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//! No-op writeback pipeline for non-Linux targets. macOS and Windows
|
||||
//! page cache policies have not been shown to exhibit the Linux
|
||||
//! accumulate-then-burst flush pathology for our access pattern.
|
||||
//! If that changes, replace this stub with a real implementation
|
||||
//! (e.g. `F_NOCACHE` on macOS, `FILE_FLAG_WRITE_THROUGH` on Windows).
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(crate) struct WritebackPipeline;
|
||||
|
||||
impl WritebackPipeline {
|
||||
pub(crate) fn new(_file: &File, _start_pos: u64, _chunk_bytes: u64) -> Self {
|
||||
Self
|
||||
}
|
||||
pub(crate) fn note_progress(&mut self, _pos: u64) {}
|
||||
pub(crate) fn handle_seek(&mut self, _new_pos: u64) {}
|
||||
pub(crate) fn finalize(&mut self) {}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//! Linux platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! - `preallocate`: `fallocate(FALLOC_FL_KEEP_SIZE)` — reserve extents
|
||||
//! without growing the reported file size. Reduces extent
|
||||
//! fragmentation on large sequential writes (mux output on NFS in
|
||||
//! particular).
|
||||
//! - `durable_sync`: `fsync` wrapped in
|
||||
//! [`crate::io::bounded::bounded_syscall`] with a 60 s deadline so a
|
||||
//! wedged NFS server can't trap the calling thread indefinitely.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Pre-reserve extents for `size_bytes` of upcoming sequential writes.
|
||||
/// Best-effort: a non-zero rc is logged but not propagated, since the
|
||||
/// caller would just continue with the unreserved file anyway.
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
// FALLOC_FL_KEEP_SIZE = 0x01 — keep the reported file size at 0
|
||||
// (writes grow it normally) while still pre-reserving the extents.
|
||||
let rc = unsafe {
|
||||
libc::fallocate(
|
||||
file.as_raw_fd(),
|
||||
libc::FALLOC_FL_KEEP_SIZE,
|
||||
0,
|
||||
size_bytes as i64,
|
||||
)
|
||||
};
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile fallocate size_hint={size_bytes} rc={rc} ok={}",
|
||||
rc == 0
|
||||
);
|
||||
}
|
||||
|
||||
/// Run `fsync` on `file` with a 60 s deadline. On timeout we log loudly
|
||||
/// and return `Ok(())` — the kernel will still flush on close, so the
|
||||
/// data is best-effort durable; the alternative (trap the thread for
|
||||
/// the rest of the rip) defeats `/api/stop`.
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
let fd = file.as_raw_fd();
|
||||
match crate::io::bounded::bounded_syscall(
|
||||
None,
|
||||
Duration::from_secs(60),
|
||||
move || -> io::Result<()> {
|
||||
let rc = unsafe { libc::fsync(fd) };
|
||||
if rc == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::last_os_error())
|
||||
}
|
||||
},
|
||||
) {
|
||||
Ok(inner) => inner,
|
||||
Err(crate::io::bounded::BoundedError::Timeout) => {
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all fsync timed out after 60s; kernel will flush on close (best-effort)"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(crate::io::bounded::BoundedError::Halted) => Ok(()),
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => Ok(()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
//! macOS platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! - `preallocate`: `fcntl(F_PREALLOCATE)` — macOS's fallocate-equiv.
|
||||
//! Reserves a contiguous extent when possible, falling back to a
|
||||
//! non-contiguous reservation if the FS can't satisfy it. Reported
|
||||
//! file size is unchanged (`F_ALLOCATEALL` is not set, so allocation
|
||||
//! is "best effort up to length"; growth happens via writes).
|
||||
//! - `durable_sync`: `fcntl(F_FULLFSYNC)` wrapped in
|
||||
//! [`crate::io::bounded::bounded_syscall`] with a 60 s deadline.
|
||||
//! F_FULLFSYNC is HFS+/APFS's true-fsync (flushes the disk's own
|
||||
//! write cache) — what `fsync` should have been on macOS. Falls back
|
||||
//! to plain `fsync` if F_FULLFSYNC returns ENOTSUP.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::time::Duration;
|
||||
|
||||
/// libc `F_PREALLOCATE` — not exposed by the `libc` crate on all macOS
|
||||
/// SDK versions, so define it here.
|
||||
const F_PREALLOCATE: libc::c_int = 42;
|
||||
/// Allocate from current EOF.
|
||||
const F_PEOFPOSMODE: libc::c_int = 3;
|
||||
/// Hint: contiguous extent preferred.
|
||||
const F_ALLOCATECONTIG: libc::c_uint = 0x00000002;
|
||||
/// Allocate all the requested bytes (fall back to non-contig if needed).
|
||||
const F_ALLOCATEALL: libc::c_uint = 0x00000004;
|
||||
|
||||
/// `fcntl(F_FULLFSYNC)` opcode. Documented in `man 2 fcntl` on macOS;
|
||||
/// not in the `libc` crate as a named constant.
|
||||
const F_FULLFSYNC: libc::c_int = 51;
|
||||
|
||||
/// `fstore_t` layout matches `sys/fcntl.h`. Repr is C-stable so we can
|
||||
/// build it manually.
|
||||
#[repr(C)]
|
||||
struct Fstore {
|
||||
fst_flags: libc::c_uint,
|
||||
fst_posmode: libc::c_int,
|
||||
fst_offset: libc::off_t,
|
||||
fst_length: libc::off_t,
|
||||
fst_bytesalloc: libc::off_t,
|
||||
}
|
||||
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
let mut fst = Fstore {
|
||||
fst_flags: F_ALLOCATECONTIG | F_ALLOCATEALL,
|
||||
fst_posmode: F_PEOFPOSMODE,
|
||||
fst_offset: 0,
|
||||
fst_length: size_bytes as libc::off_t,
|
||||
fst_bytesalloc: 0,
|
||||
};
|
||||
// First attempt: contiguous.
|
||||
let mut rc = unsafe { libc::fcntl(file.as_raw_fd(), F_PREALLOCATE, &mut fst) };
|
||||
if rc == -1 {
|
||||
// Fall back: drop the contiguous hint, allow scattered extents.
|
||||
fst.fst_flags = F_ALLOCATEALL;
|
||||
rc = unsafe { libc::fcntl(file.as_raw_fd(), F_PREALLOCATE, &mut fst) };
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile F_PREALLOCATE size_hint={size_bytes} rc={rc} bytes_allocated={} ok={}",
|
||||
fst.fst_bytesalloc,
|
||||
rc != -1
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
let fd = file.as_raw_fd();
|
||||
match crate::io::bounded::bounded_syscall(
|
||||
None,
|
||||
Duration::from_secs(60),
|
||||
move || -> io::Result<()> {
|
||||
// Try F_FULLFSYNC first. If it isn't supported on this
|
||||
// filesystem (older HFS, some network mounts) fall back to
|
||||
// plain fsync — better than nothing.
|
||||
let rc = unsafe { libc::fcntl(fd, F_FULLFSYNC, 0) };
|
||||
if rc == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let err = io::Error::last_os_error();
|
||||
if err.raw_os_error() == Some(libc::ENOTSUP) {
|
||||
let rc = unsafe { libc::fsync(fd) };
|
||||
if rc == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::last_os_error())
|
||||
}
|
||||
} else {
|
||||
Err(err)
|
||||
}
|
||||
},
|
||||
) {
|
||||
Ok(inner) => inner,
|
||||
Err(crate::io::bounded::BoundedError::Timeout) => {
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all F_FULLFSYNC timed out after 60s; kernel will flush on close (best-effort)"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(crate::io::bounded::BoundedError::Halted) => Ok(()),
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => Ok(()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
//! `WritebackFile` — a `File` wrapper whose reason for existing is the
|
||||
//! bounded-cache writeback pipeline.
|
||||
//!
|
||||
//! Why: large sequential writes (sweep, patch, mux on UHD-scale output)
|
||||
//! left to the kernel's default writeback policy accumulate hundreds of
|
||||
//! megabytes of dirty pages and then burst-flush, stalling subsequent
|
||||
//! writes for seconds at a time. `WritebackFile` drives a continuous
|
||||
//! [`super::writeback::WritebackPipeline`] that on Linux issues
|
||||
//! incremental `sync_file_range` + `posix_fadvise(DONTNEED)` calls at
|
||||
//! 32 MB granularity so dirty pages drain at the same rate they're
|
||||
//! produced. macOS and Windows fall through to a no-op pipeline — their
|
||||
//! default cache policies have not been shown to exhibit the same
|
||||
//! pathology for this access pattern.
|
||||
//!
|
||||
//! It implements `Write` and `Seek` so any call site that wrote to a
|
||||
//! plain `File` through those traits (sweep, patch, mux) can swap in
|
||||
//! `WritebackFile` without touching the body of the loop. The wrapper
|
||||
//! also tracks the current file position to feed the pipeline with
|
||||
//! progress + seek boundaries.
|
||||
//!
|
||||
//! See `super::writeback::linux` for the underlying pathology and the
|
||||
//! strategy.
|
||||
//!
|
||||
//! ## Platform split
|
||||
//!
|
||||
//! The platform-specific pieces of this wrapper — extent preallocation
|
||||
//! (Linux `fallocate(KEEP_SIZE)`, macOS `F_PREALLOCATE`, Windows
|
||||
//! `SetFileValidData`) and the durable-flush primitive (Linux/macOS
|
||||
//! `fsync`/`F_FULLFSYNC` wrapped in a bounded syscall, Windows
|
||||
//! `FlushFileBuffers`) — live in per-OS sibling modules. The dispatch
|
||||
//! happens once at the bottom of this file via cfg-gated `mod` decls.
|
||||
//! No inline `#[cfg(target_os = "...")]` in the business-logic above.
|
||||
//!
|
||||
//! ## Write path
|
||||
//!
|
||||
//! Writes are direct passthrough to the underlying `File` (no writer
|
||||
//! thread, no ring, no batching). Empirically the Phase-2.5
|
||||
//! writer-thread architecture introduced a ~60% mux throughput
|
||||
//! regression on NFS bidirectional workloads; reverting the write path
|
||||
//! to direct passthrough restores the 0.20.7 baseline. The writeback
|
||||
//! pipeline still runs (it's called inline from `write` / `write_all` /
|
||||
//! `seek`) so the bounded-cache invariant on Linux is preserved.
|
||||
//!
|
||||
//! ## Halt-safety
|
||||
//!
|
||||
//! `sync_all` runs the per-OS durable-flush primitive, which on
|
||||
//! Linux/macOS is wrapped in [`crate::io::bounded::bounded_syscall`]
|
||||
//! with a 60 s deadline. A wedged NFS server cannot trap the muxer
|
||||
//! indefinitely on the final fsync.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
mod other;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux as platform;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos as platform;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
use other as platform;
|
||||
#[cfg(target_os = "windows")]
|
||||
use windows as platform;
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use super::writeback::WritebackPipeline;
|
||||
|
||||
/// Granularity at which the Linux writeback pipeline issues
|
||||
/// `sync_file_range` / `posix_fadvise(DONTNEED)` pairs. 32 MiB is the
|
||||
/// historical default — bounded-cache pressure stays at ~2 × this size.
|
||||
const WRITEBACK_CHUNK_BYTES: u64 = 32 * 1024 * 1024;
|
||||
|
||||
pub(crate) struct WritebackFile {
|
||||
file: File,
|
||||
pipeline: WritebackPipeline,
|
||||
pos: u64,
|
||||
}
|
||||
|
||||
impl WritebackFile {
|
||||
/// Wrap an open `File`. The current OS file position is queried
|
||||
/// once so the pipeline starts tracking from wherever the file
|
||||
/// already is (typically 0 for fresh files; non-zero for resumed
|
||||
/// or appended files).
|
||||
pub(crate) fn new(mut file: File) -> io::Result<Self> {
|
||||
let pos = file.stream_position()?;
|
||||
let pipeline = WritebackPipeline::new(&file, pos, WRITEBACK_CHUNK_BYTES);
|
||||
Ok(Self {
|
||||
file,
|
||||
pipeline,
|
||||
pos,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new file at `path` (truncating any existing contents)
|
||||
/// and wrap it. Convenience for the common
|
||||
/// `File::create(path)` + `WritebackFile::new(file)` pair so callers
|
||||
/// don't have to assemble a `File` first.
|
||||
///
|
||||
/// Callers that know the target output size should prefer
|
||||
/// [`Self::create_with_size_hint`] so the kernel can pre-reserve
|
||||
/// extents.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = File::create(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Like [`Self::create`] but pre-reserves `size_bytes` of disk
|
||||
/// space via the platform's extent-preallocation primitive (Linux
|
||||
/// `fallocate(KEEP_SIZE)`, macOS `F_PREALLOCATE`, Windows
|
||||
/// `SetFileValidData` stub). The reported file size is unchanged
|
||||
/// (writes still grow the file naturally) — only the on-disk extent
|
||||
/// allocation is preallocated, which reduces extent fragmentation
|
||||
/// on large sequential writes (mux output, especially on slow
|
||||
/// storage / NFS).
|
||||
///
|
||||
/// On platforms without an extent-preallocation primitive this is
|
||||
/// equivalent to `create` — the size hint is dropped after a debug
|
||||
/// log.
|
||||
pub(crate) fn create_with_size_hint(path: &Path, size_bytes: u64) -> io::Result<Self> {
|
||||
let file = File::create(path)?;
|
||||
platform::preallocate(&file, size_bytes);
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Open an existing file at `path` for writing (no truncation) and
|
||||
/// wrap it. Mirrors `File::open` semantics for the writable case
|
||||
/// — used by patch / resume paths that mutate an existing ISO in
|
||||
/// place.
|
||||
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new().write(true).open(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Drain in-flight writeback then issue a full fsync. Use this in
|
||||
/// place of `File::sync_all`.
|
||||
///
|
||||
/// The final durable flush is wrapped in
|
||||
/// [`crate::io::bounded::bounded_syscall`] (per the per-OS module)
|
||||
/// with a 60 s deadline on Linux/macOS — a wedged NFS server cannot
|
||||
/// trap the calling thread indefinitely. On timeout the page cache
|
||||
/// is left to the kernel's normal flush-on-close path — best
|
||||
/// effort, but bounded.
|
||||
pub(crate) fn sync_all(&mut self) -> io::Result<()> {
|
||||
self.pipeline.finalize();
|
||||
platform::durable_sync(&self.file)
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for WritebackFile {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let n = self.file.write(buf)?;
|
||||
self.pos += n as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.file.write_all(buf)?;
|
||||
self.pos += buf.len() as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for WritebackFile {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
let p = self.file.seek(from)?;
|
||||
// Only treat seeks that actually move the position as
|
||||
// boundaries — sweep does a redundant `seek(Current(pos))`
|
||||
// before every write, and we don't want that to drain the
|
||||
// pipeline on every iteration.
|
||||
if p != self.pos {
|
||||
// Diagnostic for the NFS mux hang: the MKV format requires
|
||||
// the muxer to seek back occasionally (cluster size
|
||||
// patching, Cues index write, Segment header backpatch).
|
||||
// Each such seek invalidates the writeback chunk tracking
|
||||
// and forces a finalize → WAIT_AFTER on the in-flight
|
||||
// chunk. Logging the seek delta lets us correlate hang
|
||||
// offsets with specific muxer operations.
|
||||
let from_pos = self.pos;
|
||||
let to_pos = p;
|
||||
let delta: i64 = (to_pos as i64).wrapping_sub(from_pos as i64);
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile seek from={from_pos} to={to_pos} delta={delta}"
|
||||
);
|
||||
self.pipeline.handle_seek(p);
|
||||
self.pos = p;
|
||||
}
|
||||
Ok(p)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WritebackFile {
|
||||
fn drop(&mut self) {
|
||||
// Run the pipeline's tail finalize so the last in-flight chunk
|
||||
// gets its `WAIT_AFTER` + `posix_fadvise(DONTNEED)`. Without
|
||||
// this, callers that drop a `WritebackFile` without calling
|
||||
// `sync_all` (panic, early-return, idiomatic `let _ = w;`)
|
||||
// leave the trailing chunk in cache; the kernel still flushes
|
||||
// on close, but the bounded-cache invariant fails at the tail.
|
||||
// We deliberately do *not* call `self.file.sync_all()` here —
|
||||
// close already triggers a flush, and an `fsync` from `Drop`
|
||||
// would silently swallow its `io::Error` anyway. `finalize` is
|
||||
// idempotent so an explicit `sync_all` followed by drop is
|
||||
// still safe.
|
||||
self.pipeline.finalize();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
|
||||
fn read_back(path: &Path) -> Vec<u8> {
|
||||
let mut f = File::open(path).unwrap();
|
||||
let mut v = Vec::new();
|
||||
f.read_to_end(&mut v).unwrap();
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_then_drop_persists_bytes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("a.bin");
|
||||
{
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"hello world").unwrap();
|
||||
// Drop drains the pipeline tail.
|
||||
}
|
||||
assert_eq!(read_back(&p), b"hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_all_drains_and_flushes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("b.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
for _ in 0..32 {
|
||||
w.write_all(&[0x5au8; 1024]).unwrap();
|
||||
}
|
||||
// After sync_all, the bytes MUST be visible to a separate
|
||||
// reader. The pipeline has been finalised and durable-sync has
|
||||
// run.
|
||||
w.sync_all().unwrap();
|
||||
let bytes = read_back(&p);
|
||||
assert_eq!(bytes.len(), 32 * 1024);
|
||||
assert!(bytes.iter().all(|&b| b == 0x5a));
|
||||
drop(w);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seek_then_patch_roundtrip() {
|
||||
// Write A; seek back; patch with B; read back; the patch lands
|
||||
// at the right offset.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("c.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
let big = vec![b'A'; 4096];
|
||||
w.write_all(&big).unwrap();
|
||||
// Seek back to offset 1000 and overwrite 8 bytes.
|
||||
w.seek(SeekFrom::Start(1000)).unwrap();
|
||||
w.write_all(b"PATCHED!").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
let bytes = read_back(&p);
|
||||
assert_eq!(bytes.len(), 4096);
|
||||
assert_eq!(&bytes[1000..1008], b"PATCHED!");
|
||||
// Bytes outside the patch are still 'A'.
|
||||
assert_eq!(bytes[999], b'A');
|
||||
assert_eq!(bytes[1008], b'A');
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flush_is_observed_in_order() {
|
||||
// `Write::flush` should not panic or reorder; verify the bytes
|
||||
// land in order through interleaved flushes.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("f.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"one").unwrap();
|
||||
w.flush().unwrap();
|
||||
w.write_all(b"two").unwrap();
|
||||
w.flush().unwrap();
|
||||
w.write_all(b"three").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
assert_eq!(read_back(&p), b"onetwothree");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//! Fallback platform impl for [`super::WritebackFile`] on targets
|
||||
//! without a dedicated implementation (BSDs, illumos, etc.).
|
||||
//!
|
||||
//! - `preallocate` is a logged no-op.
|
||||
//! - `durable_sync` calls `File::sync_all` directly (no bounded-syscall
|
||||
//! wrapper — the wrapper depends on Linux/macOS unix idioms that
|
||||
//! aren't universally portable). If a future BSD impl needs the
|
||||
//! 60-s deadline, it should land in its own per-OS file rather than
|
||||
//! bloat this fallback.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
|
||||
pub(super) fn preallocate(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile preallocate size_hint={size_bytes} skipped (no impl on this target)"
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
file.sync_all()
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//! Windows platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! TODO: this stub matches the design's "validate without a Windows
|
||||
//! build env, leave a stub" carve-out. The real impl should use:
|
||||
//!
|
||||
//! - `SetEndOfFile` + `SetFileValidData` for extent preallocation
|
||||
//! (caller needs `SE_MANAGE_VOLUME_NAME` privilege; if unavailable
|
||||
//! fall back to a write-zero path or just skip).
|
||||
//! - `FlushFileBuffers` for fsync-equivalent durable flush.
|
||||
//!
|
||||
//! Until then: preallocate is a debug-logged no-op; durable_sync calls
|
||||
//! the std `File::sync_all` (which on Windows maps to
|
||||
//! `FlushFileBuffers` internally).
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
|
||||
pub(super) fn preallocate(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile preallocate size_hint={size_bytes} skipped (windows stub; TODO: SetFileValidData)"
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
// `File::sync_all` on Windows is `FlushFileBuffers`. Acceptable
|
||||
// for now; the bounded-syscall wrapper is not used here because
|
||||
// the stub also skips the worker-thread + leak machinery (the
|
||||
// wrapper would need an `unsafe impl Send` for `RawHandle`, and
|
||||
// designing that without a Windows test env is asking for it).
|
||||
file.sync_all()
|
||||
}
|
||||
+3
-3
@@ -71,6 +71,7 @@ pub fn save(data: &[u8]) -> Result<UpdateResult> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Result of a KEYDB update -- path written, entry count, and byte size.
|
||||
#[derive(Debug)]
|
||||
pub struct UpdateResult {
|
||||
pub path: PathBuf,
|
||||
@@ -82,7 +83,7 @@ fn http_get(url: &str) -> Result<Vec<u8>> {
|
||||
let (mut host, mut port, mut path) = parse_url(url)?;
|
||||
|
||||
for _ in 0..5 {
|
||||
let addr = format!("{}:{}", host, port);
|
||||
let addr = format!("{host}:{port}");
|
||||
let mut stream =
|
||||
TcpStream::connect(&addr).map_err(|_| Error::KeydbConnect { host: host.clone() })?;
|
||||
stream
|
||||
@@ -90,8 +91,7 @@ fn http_get(url: &str) -> Result<Vec<u8>> {
|
||||
.ok();
|
||||
|
||||
let request = format!(
|
||||
"GET {} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\nAccept-Encoding: identity\r\n\r\n",
|
||||
path, host
|
||||
"GET {path} HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\nAccept-Encoding: identity\r\n\r\n"
|
||||
);
|
||||
stream
|
||||
.write_all(request.as_bytes())
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
//! BDMV disc-library metadata (`/BDMV/META/DL/bdmt_<lang>.xml`).
|
||||
//!
|
||||
//! Every commercial Blu-ray carries a disc-library metadata directory
|
||||
//! with one XML file per shipped language. The schema is the Blu-ray
|
||||
//! "disc library metadata" namespace (`urn:BDA:bdmv;disclibmeta`),
|
||||
//! conventionally prefixed `di:`. Fields commonly present:
|
||||
//!
|
||||
//! - `<di:title>` or `<di:name>` — the title string. Vendor practice
|
||||
//! varies (Paramount discs tend to use `<di:name>`).
|
||||
//! - `<di:description>` — optional synopsis (often absent on retail
|
||||
//! discs; common on box sets and special editions).
|
||||
//! - `<di:discNumber>` / `<di:numSets>` (or `<di:numberOfSets>`) —
|
||||
//! set position for multi-disc releases.
|
||||
//!
|
||||
//! This module is intentionally separate from the BD-J `StreamLabel`
|
||||
//! parsers under `labels/*.rs`. The XML here is disc-level (title,
|
||||
//! description, set position), not per-stream — wiring into the main
|
||||
//! parser registry happens elsewhere.
|
||||
//!
|
||||
//! Real-world XML is irregular: missing description elements, multiple
|
||||
//! title elements (first one wins), and occasional malformed content.
|
||||
//! Extraction is best-effort — a malformed file is treated as "no
|
||||
//! metadata" (returns `None` from the helper), and the caller can
|
||||
//! still get metadata from sibling-language XML files.
|
||||
|
||||
// The module wiring (registry hook + public re-export) is added
|
||||
// separately. Until then the parse/detect entry points have no
|
||||
use super::xml;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Disc-level metadata extracted from `/BDMV/META/DL/bdmt_*.xml`.
|
||||
///
|
||||
/// All maps are keyed by 3-char ISO 639-2 language code (e.g.
|
||||
/// `"eng"`, `"fra"`, `"jpn"`) — the same key segment used in the
|
||||
/// `bdmt_<lang>.xml` filename.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)]
|
||||
pub struct DiscMetadata {
|
||||
/// Localized titles, keyed by 3-char ISO 639-2 lang code
|
||||
/// (e.g. "eng" → "Dune Part Two")
|
||||
pub titles: BTreeMap<String, String>,
|
||||
/// First-line / short description, per lang
|
||||
pub descriptions: BTreeMap<String, String>,
|
||||
/// Disc N of M for box sets (None if not a box set)
|
||||
pub disc_number: Option<(u32, u32)>,
|
||||
}
|
||||
|
||||
/// True if `/BDMV/META/DL/` exists and contains at least one
|
||||
/// `bdmt_*.xml` file.
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
let Some(dir) = udf.find_dir("/BDMV/META/DL") else {
|
||||
return false;
|
||||
};
|
||||
dir.entries
|
||||
.iter()
|
||||
.any(|e| !e.is_dir && is_bdmt_filename(&e.name))
|
||||
}
|
||||
|
||||
/// Read every `bdmt_<lang>.xml` under `/BDMV/META/DL/` and return the
|
||||
/// aggregated [`DiscMetadata`]. Returns `None` if no titles could be
|
||||
/// extracted from any file.
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<DiscMetadata> {
|
||||
let dir = udf.find_dir("/BDMV/META/DL")?;
|
||||
let mut out = DiscMetadata::default();
|
||||
|
||||
for entry in &dir.entries {
|
||||
if entry.is_dir {
|
||||
continue;
|
||||
}
|
||||
let Some(lang) = lang_code_from_filename(&entry.name) else {
|
||||
continue;
|
||||
};
|
||||
let path = format!("/BDMV/META/DL/{}", entry.name);
|
||||
let Ok(bytes) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(text) = std::str::from_utf8(&bytes) else {
|
||||
continue;
|
||||
};
|
||||
let Some((title, description, disc_set)) = parse_bdmt_xml(&lang, text) else {
|
||||
continue;
|
||||
};
|
||||
out.titles.insert(lang.clone(), title);
|
||||
if let Some(desc) = description {
|
||||
out.descriptions.insert(lang.clone(), desc);
|
||||
}
|
||||
// Disc-set position is disc-global; first one we successfully
|
||||
// read wins. (All bdmt_*.xml on a given disc carry the same
|
||||
// value in practice.)
|
||||
if out.disc_number.is_none() {
|
||||
if let Some(ds) = disc_set {
|
||||
out.disc_number = Some(ds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if out.titles.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// True if `name` matches the `bdmt_<lang>.xml` convention with a
|
||||
/// 3-character ISO 639-2 lang code segment. Case-insensitive.
|
||||
fn is_bdmt_filename(name: &str) -> bool {
|
||||
lang_code_from_filename(name).is_some()
|
||||
}
|
||||
|
||||
/// Extract the 3-char language code from a `bdmt_<lang>.xml` filename.
|
||||
/// Returns `None` if the filename doesn't match. Lang code is
|
||||
/// lowercased so callers always see e.g. `"eng"` not `"ENG"`.
|
||||
fn lang_code_from_filename(name: &str) -> Option<String> {
|
||||
let lower = name.to_ascii_lowercase();
|
||||
let stem = lower.strip_suffix(".xml")?;
|
||||
let lang = stem.strip_prefix("bdmt_")?;
|
||||
// ISO 639-2 codes are exactly 3 ASCII letters. Be strict — keeps
|
||||
// us from picking up unrelated `bdmt_foo.xml` siblings.
|
||||
if lang.len() != 3 || !lang.chars().all(|c| c.is_ascii_alphabetic()) {
|
||||
return None;
|
||||
}
|
||||
Some(lang.to_string())
|
||||
}
|
||||
|
||||
/// Tuple returned by [`parse_bdmt_xml`]: `(title, description?, disc_set?)`.
|
||||
/// Aliased so the function signature isn't a clippy::type-complexity offender.
|
||||
pub(crate) type BdmtFields = (String, Option<String>, Option<(u32, u32)>);
|
||||
|
||||
/// Parse one `bdmt_<lang>.xml` document and return
|
||||
/// `(title, description?, disc_set?)`. Returns `None` if no title
|
||||
/// could be located — the caller treats this as "skip this file".
|
||||
///
|
||||
/// Title-element preference: `<di:name>` → `<di:title>` →
|
||||
/// `<di:tableOfContents>/<di:titleName>` (first match wins, per the
|
||||
/// authoring-tool conventions documented at the module level).
|
||||
pub(crate) fn parse_bdmt_xml(_lang_code: &str, xml_text: &str) -> Option<BdmtFields> {
|
||||
let title = extract_title(xml_text)?;
|
||||
let description = xml::text(xml_text, "description")
|
||||
.filter(|s| !s.is_empty())
|
||||
.filter(|s| !looks_like_xml(s))
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty());
|
||||
let disc_set = extract_disc_set(xml_text);
|
||||
Some((title, description, disc_set))
|
||||
}
|
||||
|
||||
/// Reject candidate description strings that are themselves XML
|
||||
/// fragments — observed on disc-04 (Top Gun: Maverick), where
|
||||
/// `<di:description>` contained `<di:thumbnail href="…"/>` child
|
||||
/// elements and no actual prose. Surfacing that raw to the JSON
|
||||
/// output is worse than dropping the field entirely.
|
||||
fn looks_like_xml(s: &str) -> bool {
|
||||
let t = s.trim_start();
|
||||
t.starts_with('<')
|
||||
}
|
||||
|
||||
/// Try title-bearing element variants in priority order. The `xml`
|
||||
/// helpers are case- and namespace-insensitive, so callers pass the
|
||||
/// bare local name (no `di:` prefix).
|
||||
fn extract_title(xml_text: &str) -> Option<String> {
|
||||
// Order matches the module-level convention: <di:name> first
|
||||
// (Paramount-style), then <di:title>, then the nested
|
||||
// tableOfContents/titleName form.
|
||||
for tag in ["name", "title"] {
|
||||
if let Some(s) = xml::text(xml_text, tag) {
|
||||
let trimmed = s.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
// tableOfContents/titleName: search inside the toc block so we
|
||||
// don't accidentally pick a stray <titleName> from elsewhere.
|
||||
if let Some((s, e)) = xml::find_element(xml_text, "tableOfContents", 0) {
|
||||
let block = &xml_text[s..e];
|
||||
if let Some(t) = xml::text(block, "titleName") {
|
||||
let trimmed = t.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Extract `(discNumber, numSets)` if both are present and parse as
|
||||
/// `u32`. Accepts either `<di:numSets>` or `<di:numberOfSets>` for
|
||||
/// the denominator (both forms appear in the wild).
|
||||
fn extract_disc_set(xml_text: &str) -> Option<(u32, u32)> {
|
||||
let n = xml::text(xml_text, "discNumber")?
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.ok()?;
|
||||
let total = xml::text(xml_text, "numSets")
|
||||
.or_else(|| xml::text(xml_text, "numberOfSets"))?
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.ok()?;
|
||||
Some((n, total))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn extract_simple_title() {
|
||||
// Minimal Paramount-style document: <di:name> as the title
|
||||
// carrier inside a <discInfo> root.
|
||||
let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Dune Part Two</di:name>
|
||||
</discInfo>"#;
|
||||
let (title, desc, set) = parse_bdmt_xml("eng", xml).expect("title should parse");
|
||||
assert_eq!(title, "Dune Part Two");
|
||||
assert_eq!(desc, None);
|
||||
assert_eq!(set, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_title_element_variant() {
|
||||
// <di:title> is the alternate carrier; should be picked up
|
||||
// when <di:name> is absent.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:title>The Matrix</di:title>
|
||||
<di:description>A film about computers.</di:description>
|
||||
</discInfo>"#;
|
||||
let (title, desc, _) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(title, "The Matrix");
|
||||
assert_eq!(desc.as_deref(), Some("A film about computers."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_title_from_table_of_contents_fallback() {
|
||||
// Some authoring tools nest the title under tableOfContents.
|
||||
// No <di:name> or <di:title> at top level → fall back to
|
||||
// titleName inside tableOfContents.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:tableOfContents>
|
||||
<di:titleName>Inside Out 2</di:titleName>
|
||||
</di:tableOfContents>
|
||||
</discInfo>"#;
|
||||
let (title, _, _) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(title, "Inside Out 2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_box_set_position() {
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>LOTR Disc 2</di:name>
|
||||
<di:discNumber>2</di:discNumber>
|
||||
<di:numSets>5</di:numSets>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(set, Some((2, 5)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_box_set_position_alternate_total_tag() {
|
||||
// <di:numberOfSets> is an alternate spelling we've seen.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>X</di:name>
|
||||
<di:discNumber>3</di:discNumber>
|
||||
<di:numberOfSets>6</di:numberOfSets>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(set, Some((3, 6)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_box_set_requires_both_fields() {
|
||||
// discNumber alone (no total) yields None — we don't fabricate
|
||||
// a denominator.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>X</di:name>
|
||||
<di:discNumber>1</di:discNumber>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(set, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_languages_keyed_correctly() {
|
||||
// Simulate driving parse_bdmt_xml from two synthetic XML
|
||||
// blobs and aggregating into DiscMetadata the same way parse()
|
||||
// would. This exercises the BTreeMap key handling without
|
||||
// needing a UdfFs.
|
||||
let eng_xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Dune Part Two</di:name>
|
||||
</discInfo>"#;
|
||||
let fra_xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Dune Deuxième Partie</di:name>
|
||||
<di:description>Suite du film de 2021.</di:description>
|
||||
</discInfo>"#;
|
||||
|
||||
let mut meta = DiscMetadata::default();
|
||||
for (lang, blob) in [("eng", eng_xml), ("fra", fra_xml)] {
|
||||
let (title, desc, ds) = parse_bdmt_xml(lang, blob).unwrap();
|
||||
meta.titles.insert(lang.to_string(), title);
|
||||
if let Some(d) = desc {
|
||||
meta.descriptions.insert(lang.to_string(), d);
|
||||
}
|
||||
if meta.disc_number.is_none() {
|
||||
if let Some(d) = ds {
|
||||
meta.disc_number = Some(d);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
meta.titles.get("eng").map(String::as_str),
|
||||
Some("Dune Part Two")
|
||||
);
|
||||
assert_eq!(
|
||||
meta.titles.get("fra").map(String::as_str),
|
||||
Some("Dune Deuxième Partie")
|
||||
);
|
||||
assert!(meta.descriptions.get("eng").is_none());
|
||||
assert_eq!(
|
||||
meta.descriptions.get("fra").map(String::as_str),
|
||||
Some("Suite du film de 2021.")
|
||||
);
|
||||
assert_eq!(meta.disc_number, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_xml_returns_none() {
|
||||
// Random gibberish has no recognizable title element. We
|
||||
// document the contract: parse_bdmt_xml returns None, and
|
||||
// parse() (the caller) skips the file. Aggregating across
|
||||
// zero files leaves DiscMetadata::default() — which parse()
|
||||
// surfaces as None to its caller. Either is documented as
|
||||
// acceptable per the module spec.
|
||||
let bad = "this is not xml &&& <<< nope";
|
||||
assert!(parse_bdmt_xml("eng", bad).is_none());
|
||||
|
||||
// Half-open tag, no body, no close: also yields no title.
|
||||
let truncated = "<discInfo><di:name>";
|
||||
assert!(parse_bdmt_xml("eng", truncated).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn description_with_only_child_xml_is_dropped() {
|
||||
// Real-world bug from disc-04 (Top Gun: Maverick, 2026-05-11
|
||||
// capture): <di:description> contained only <di:thumbnail/>
|
||||
// child elements with no actual prose. The previous parser
|
||||
// surfaced the raw XML fragment as the description string.
|
||||
// Now we reject candidates that begin with `<`.
|
||||
let xml = r#"<discInfo>
|
||||
<di:name>Top Gun: Maverick</di:name>
|
||||
<di:description>
|
||||
<di:thumbnail href="tgm_meta_sm.jpg" />
|
||||
<di:thumbnail href="tgm_meta_lg.jpg" />
|
||||
</di:description>
|
||||
</discInfo>"#;
|
||||
let (title, description, _) =
|
||||
parse_bdmt_xml("eng", xml).expect("title is present so parse must succeed");
|
||||
assert_eq!(title, "Top Gun: Maverick");
|
||||
assert!(
|
||||
description.is_none(),
|
||||
"description containing only XML children must be dropped, got {description:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn description_with_plain_text_passes_through() {
|
||||
// The legitimate case still works: a description with actual
|
||||
// prose survives the looks_like_xml filter.
|
||||
let xml = r#"<discInfo>
|
||||
<di:name>Some Movie</di:name>
|
||||
<di:description>An epic tale of one man's quest for tea.</di:description>
|
||||
</discInfo>"#;
|
||||
let (_, description, _) = parse_bdmt_xml("eng", xml).expect("must parse");
|
||||
assert_eq!(
|
||||
description.as_deref(),
|
||||
Some("An epic tale of one man's quest for tea.")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whitespace_in_title_is_trimmed() {
|
||||
let xml = r#"<discInfo><di:name>
|
||||
Dune Part Two
|
||||
</di:name></discInfo>"#;
|
||||
let (title, _, _) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(title, "Dune Part Two");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_code_extraction() {
|
||||
assert_eq!(lang_code_from_filename("bdmt_eng.xml"), Some("eng".into()));
|
||||
assert_eq!(lang_code_from_filename("BDMT_FRA.XML"), Some("fra".into()));
|
||||
assert_eq!(lang_code_from_filename("bdmt_jpn.xml"), Some("jpn".into()));
|
||||
// Non-matching cases:
|
||||
assert_eq!(lang_code_from_filename("bdmt_.xml"), None);
|
||||
assert_eq!(lang_code_from_filename("bdmt_engl.xml"), None);
|
||||
assert_eq!(lang_code_from_filename("bdmt_e1g.xml"), None);
|
||||
assert_eq!(lang_code_from_filename("bdmt_eng.txt"), None);
|
||||
assert_eq!(lang_code_from_filename("foo.xml"), None);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,291 @@
|
||||
//! CLPI vs MPLS cross-validation diagnostic.
|
||||
//!
|
||||
//! Empirical question (raised 2026-05-11): is CLPI's per-stream
|
||||
//! language and codec data truly redundant with MPLS's STN-table data
|
||||
//! on real-world Blu-rays?
|
||||
//!
|
||||
//! Build a quick audit that walks both sources, normalizes their stream
|
||||
//! lists by (PID, language, coding_type), and flags any disagreement.
|
||||
//!
|
||||
//! Three classes of mismatch we want to detect:
|
||||
//!
|
||||
//! 1. **CLPI has streams MPLS doesn't reference.** Orphan streams in
|
||||
//! the .m2ts that no playlist's STN table includes. Means the user
|
||||
//! can't reach them through the menu but they're physically on the
|
||||
//! disc.
|
||||
//! 2. **MPLS has streams CLPI doesn't list.** Should never happen if
|
||||
//! both parsers are correct — playlists reference clips which
|
||||
//! reference streams. If it happens, one of our parsers has a bug.
|
||||
//! 3. **Same PID, different language / coding_type.** The playlist re-
|
||||
//! tagged a stream's metadata. Rare but spec-permitted. Means CLPI
|
||||
//! and MPLS disagree about the same physical stream's properties.
|
||||
//!
|
||||
//! If audits across the corpus show zero mismatches of any class, CLPI
|
||||
//! program_info extraction is **empirically redundant** for labels and
|
||||
//! we can leave it out of the registry. If even one mismatch surfaces,
|
||||
//! we add a CLPI label parser to the registry as belt-and-suspenders.
|
||||
//!
|
||||
//! This module exposes `audit(reader, udf)` returning a structured
|
||||
//! report. Surfaced via the labels-analyze tool — not part of the
|
||||
//! `analyze()` pipeline (no impact on the label output).
|
||||
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// One row in the audit: a stream PID that's known to one source or
|
||||
/// both, with the fields each source reported.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ClpiVsMplsRow {
|
||||
pub pid: u16,
|
||||
pub clpi_coding_type: Option<u8>,
|
||||
pub clpi_language: Option<String>,
|
||||
pub mpls_coding_type: Option<u8>,
|
||||
pub mpls_language: Option<String>,
|
||||
}
|
||||
|
||||
impl ClpiVsMplsRow {
|
||||
/// Three rules for classification:
|
||||
/// - both sources missing (impossible — caller wouldn't insert)
|
||||
/// - one source missing → class A or B (orphan-on-disc / playlist-only)
|
||||
/// - both present but fields differ → class C (metadata divergence)
|
||||
/// - both present and identical → no mismatch
|
||||
pub fn class(&self) -> ClpiVsMplsClass {
|
||||
match (
|
||||
self.clpi_coding_type.is_some(),
|
||||
self.mpls_coding_type.is_some(),
|
||||
) {
|
||||
(true, false) => ClpiVsMplsClass::ClpiOnly,
|
||||
(false, true) => ClpiVsMplsClass::MplsOnly,
|
||||
(true, true) => {
|
||||
let coding_match = self.clpi_coding_type == self.mpls_coding_type;
|
||||
let lang_match = self.clpi_language == self.mpls_language;
|
||||
if coding_match && lang_match {
|
||||
ClpiVsMplsClass::Match
|
||||
} else {
|
||||
ClpiVsMplsClass::Divergent
|
||||
}
|
||||
}
|
||||
(false, false) => ClpiVsMplsClass::Match,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Classification of one (PID) row.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ClpiVsMplsClass {
|
||||
/// PID seen in CLPI ProgramInfo but no MPLS STN table references
|
||||
/// it. Orphan on disc.
|
||||
ClpiOnly,
|
||||
/// PID seen in MPLS STN table but no CLPI ProgramInfo includes it.
|
||||
/// One of our parsers probably has a bug.
|
||||
MplsOnly,
|
||||
/// Both sources see this PID with the same coding_type + language.
|
||||
Match,
|
||||
/// Both sources see this PID but disagree on coding_type or language.
|
||||
/// MPLS wins for label rendering (playlist-authoritative view); CLPI
|
||||
/// is the per-clip ground truth.
|
||||
Divergent,
|
||||
}
|
||||
|
||||
/// Full audit report.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ClpiVsMplsAudit {
|
||||
pub rows: Vec<ClpiVsMplsRow>,
|
||||
}
|
||||
|
||||
impl ClpiVsMplsAudit {
|
||||
pub fn class_counts(&self) -> (usize, usize, usize, usize) {
|
||||
let mut clpi_only = 0;
|
||||
let mut mpls_only = 0;
|
||||
let mut matches = 0;
|
||||
let mut divergent = 0;
|
||||
for r in &self.rows {
|
||||
match r.class() {
|
||||
ClpiVsMplsClass::ClpiOnly => clpi_only += 1,
|
||||
ClpiVsMplsClass::MplsOnly => mpls_only += 1,
|
||||
ClpiVsMplsClass::Match => matches += 1,
|
||||
ClpiVsMplsClass::Divergent => divergent += 1,
|
||||
}
|
||||
}
|
||||
(clpi_only, mpls_only, matches, divergent)
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk `/BDMV/CLIPINF/*.clpi` and `/BDMV/PLAYLIST/*.mpls`, build a
|
||||
/// dedup-by-PID table of (CLPI fields, MPLS fields), return the
|
||||
/// merged view. Missing files (read errors, parse failures) are
|
||||
/// silently skipped — this is diagnostic, not correctness-critical.
|
||||
pub fn audit(reader: &mut dyn SectorSource, udf: &UdfFs) -> ClpiVsMplsAudit {
|
||||
// Aggregate by PID across all CLPI files. If a PID appears in
|
||||
// multiple clips (typical — main movie clip + trailers reference
|
||||
// the same audio stream PIDs), first encountered wins (they should
|
||||
// all agree per BD spec).
|
||||
let mut clpi_by_pid: BTreeMap<u16, (u8, String)> = BTreeMap::new();
|
||||
if let Some(dir) = udf.find_dir("/BDMV/CLIPINF") {
|
||||
let names: Vec<String> = dir
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && e.name.to_ascii_lowercase().ends_with(".clpi"))
|
||||
.map(|e| e.name.clone())
|
||||
.collect();
|
||||
for name in names {
|
||||
let path = format!("/BDMV/CLIPINF/{}", name);
|
||||
let Ok(data) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(clip) = crate::clpi::parse(&data) else {
|
||||
continue;
|
||||
};
|
||||
for s in clip.streams {
|
||||
clpi_by_pid
|
||||
.entry(s.pid)
|
||||
.or_insert((s.coding_type, s.language));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Same for MPLS streams.
|
||||
let mut mpls_by_pid: BTreeMap<u16, (u8, String)> = BTreeMap::new();
|
||||
if let Some(dir) = udf.find_dir("/BDMV/PLAYLIST") {
|
||||
let names: Vec<String> = dir
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && e.name.to_ascii_lowercase().ends_with(".mpls"))
|
||||
.map(|e| e.name.clone())
|
||||
.collect();
|
||||
for name in names {
|
||||
let path = format!("/BDMV/PLAYLIST/{}", name);
|
||||
let Ok(data) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(pl) = crate::mpls::parse(&data) else {
|
||||
continue;
|
||||
};
|
||||
for s in pl.streams {
|
||||
if s.pid == 0 {
|
||||
// PID 0 means "no PID in stream entry" — skip rather
|
||||
// than collide with other entries.
|
||||
continue;
|
||||
}
|
||||
mpls_by_pid
|
||||
.entry(s.pid)
|
||||
.or_insert((s.coding_type, s.language));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Merge views: every PID seen anywhere gets a row.
|
||||
let mut all_pids: std::collections::BTreeSet<u16> = std::collections::BTreeSet::new();
|
||||
all_pids.extend(clpi_by_pid.keys().copied());
|
||||
all_pids.extend(mpls_by_pid.keys().copied());
|
||||
|
||||
let mut rows = Vec::with_capacity(all_pids.len());
|
||||
for pid in all_pids {
|
||||
let clpi = clpi_by_pid.get(&pid);
|
||||
let mpls = mpls_by_pid.get(&pid);
|
||||
rows.push(ClpiVsMplsRow {
|
||||
pid,
|
||||
clpi_coding_type: clpi.map(|(c, _)| *c),
|
||||
clpi_language: clpi.map(|(_, l)| l.clone()),
|
||||
mpls_coding_type: mpls.map(|(c, _)| *c),
|
||||
mpls_language: mpls.map(|(_, l)| l.clone()),
|
||||
});
|
||||
}
|
||||
|
||||
ClpiVsMplsAudit { rows }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn class_match_when_identical() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("eng".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::Match);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_clpi_only_when_mpls_missing() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: None,
|
||||
mpls_language: None,
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::ClpiOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_mpls_only_when_clpi_missing() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: None,
|
||||
clpi_language: None,
|
||||
mpls_coding_type: Some(0x90),
|
||||
mpls_language: Some("fra".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::MplsOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_divergent_on_lang_disagreement() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("und".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::Divergent);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_counts_sum_rows() {
|
||||
let audit = ClpiVsMplsAudit {
|
||||
rows: vec![
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("eng".into()),
|
||||
}, // Match
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1101,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("fra".into()),
|
||||
mpls_coding_type: None,
|
||||
mpls_language: None,
|
||||
}, // ClpiOnly
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1102,
|
||||
clpi_coding_type: None,
|
||||
clpi_language: None,
|
||||
mpls_coding_type: Some(0x90),
|
||||
mpls_language: Some("eng".into()),
|
||||
}, // MplsOnly
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1103,
|
||||
clpi_coding_type: Some(0x86),
|
||||
clpi_language: Some("spa".into()),
|
||||
mpls_coding_type: Some(0x86),
|
||||
mpls_language: Some("ita".into()),
|
||||
}, // Divergent
|
||||
],
|
||||
};
|
||||
let (co, mo, m, d) = audit.class_counts();
|
||||
assert_eq!(co, 1);
|
||||
assert_eq!(mo, 1);
|
||||
assert_eq!(m, 1);
|
||||
assert_eq!(d, 1);
|
||||
}
|
||||
}
|
||||
+56
-92
@@ -3,8 +3,8 @@
|
||||
//! Clean structured XML with Content/Qualifier per stream and
|
||||
//! stream number mapping via playbackconfig.
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, xml};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -12,7 +12,7 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "streamproperties.xml")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let sp_data = super::read_jar_file(reader, udf, "streamproperties.xml")?;
|
||||
let sp_text = std::str::from_utf8(&sp_data).ok()?;
|
||||
|
||||
@@ -66,7 +66,8 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(labels)
|
||||
// High confidence: streamproperties.xml is fully structured.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
|
||||
struct StreamInfo {
|
||||
@@ -78,105 +79,68 @@ struct StreamInfo {
|
||||
qualifier: LabelQualifier,
|
||||
}
|
||||
|
||||
fn parse_stream_infos(xml: &str) -> Vec<StreamInfo> {
|
||||
fn parse_stream_infos(text: &str) -> Vec<StreamInfo> {
|
||||
let mut infos = Vec::new();
|
||||
let mut pos = 0;
|
||||
|
||||
while pos < xml.len() {
|
||||
let (tag, stream_type) = if let Some(p) = xml[pos..].find("<AudioStreamInfos>") {
|
||||
(p + pos, StreamLabelType::Audio)
|
||||
} else if let Some(p) = xml[pos..].find("<SubtitleStreamInfos>") {
|
||||
(p + pos, StreamLabelType::Subtitle)
|
||||
} else {
|
||||
break;
|
||||
};
|
||||
for (tag_name, stream_type) in [
|
||||
("AudioStreamInfos", StreamLabelType::Audio),
|
||||
("SubtitleStreamInfos", StreamLabelType::Subtitle),
|
||||
] {
|
||||
let mut from = 0;
|
||||
while let Some((start, end)) = xml::find_element(text, tag_name, from) {
|
||||
let block = &text[start..end];
|
||||
let id = xml::text(block, "ID").unwrap_or_default();
|
||||
let lang_id = xml::text(block, "LangInfoID").unwrap_or_default();
|
||||
let content = xml::text(block, "Content").unwrap_or_default();
|
||||
let qualifier_str = xml::text(block, "Qualifier").unwrap_or_default();
|
||||
|
||||
let end_tag = match stream_type {
|
||||
StreamLabelType::Audio => "</AudioStreamInfos>",
|
||||
StreamLabelType::Subtitle => "</SubtitleStreamInfos>",
|
||||
};
|
||||
let (language, variant) = if lang_id.contains('_') {
|
||||
let parts: Vec<&str> = lang_id.splitn(2, '_').collect();
|
||||
(parts[0].to_lowercase(), parts[1].to_string())
|
||||
} else {
|
||||
(lang_id.to_lowercase(), String::new())
|
||||
};
|
||||
|
||||
let block_end = match xml[tag..].find(end_tag) {
|
||||
Some(p) => tag + p + end_tag.len(),
|
||||
None => break,
|
||||
};
|
||||
let purpose = if content.eq_ignore_ascii_case("COMMENTARY") {
|
||||
LabelPurpose::Commentary
|
||||
} else {
|
||||
LabelPurpose::Normal
|
||||
};
|
||||
|
||||
let block = &xml[tag..block_end];
|
||||
let id = extract_tag(block, "ID").unwrap_or_default();
|
||||
let lang_id = extract_tag(block, "LangInfoID").unwrap_or_default();
|
||||
let content = extract_tag(block, "Content").unwrap_or_default();
|
||||
let qualifier_str = extract_tag(block, "Qualifier").unwrap_or_default();
|
||||
let qualifier = match qualifier_str.to_ascii_uppercase().as_str() {
|
||||
"SDH" => LabelQualifier::Sdh,
|
||||
"DS" => LabelQualifier::DescriptiveService,
|
||||
_ => LabelQualifier::None,
|
||||
};
|
||||
|
||||
let (language, variant) = if lang_id.contains('_') {
|
||||
let parts: Vec<&str> = lang_id.splitn(2, '_').collect();
|
||||
(parts[0].to_lowercase(), parts[1].to_string())
|
||||
} else {
|
||||
(lang_id.to_lowercase(), String::new())
|
||||
};
|
||||
|
||||
let purpose = match content.as_str() {
|
||||
"COMMENTARY" => LabelPurpose::Commentary,
|
||||
_ => LabelPurpose::Normal,
|
||||
};
|
||||
|
||||
let qualifier = match qualifier_str.as_str() {
|
||||
"SDH" => LabelQualifier::Sdh,
|
||||
"DS" => LabelQualifier::DescriptiveService,
|
||||
_ => LabelQualifier::None,
|
||||
};
|
||||
|
||||
infos.push(StreamInfo {
|
||||
id,
|
||||
stream_type,
|
||||
language,
|
||||
variant,
|
||||
purpose,
|
||||
qualifier,
|
||||
});
|
||||
pos = block_end;
|
||||
infos.push(StreamInfo {
|
||||
id,
|
||||
stream_type,
|
||||
language,
|
||||
variant,
|
||||
purpose,
|
||||
qualifier,
|
||||
});
|
||||
from = end;
|
||||
}
|
||||
}
|
||||
infos
|
||||
}
|
||||
|
||||
fn parse_playback_config(xml: &str, map: &mut HashMap<String, u16>) {
|
||||
let mut pos = 0;
|
||||
while pos < xml.len() {
|
||||
let tag_start = if let Some(p) = xml[pos..].find("<AudioStreams>") {
|
||||
Some(p + pos)
|
||||
} else {
|
||||
xml[pos..].find("<SubtitlesStreams>").map(|p| p + pos)
|
||||
};
|
||||
|
||||
let tag_start = match tag_start {
|
||||
Some(p) => p,
|
||||
None => break,
|
||||
};
|
||||
|
||||
let block_end = xml[tag_start..]
|
||||
.find("</AudioStreams>")
|
||||
.or_else(|| xml[tag_start..].find("</SubtitlesStreams>"))
|
||||
.map(|p| tag_start + p + 20)
|
||||
.unwrap_or(xml.len());
|
||||
|
||||
let block = &xml[tag_start..block_end];
|
||||
|
||||
if let (Some(stream_id_str), Some(info_id)) = (
|
||||
extract_tag(block, "StreamID"),
|
||||
extract_tag(block, "StreamInfo_ID"),
|
||||
) {
|
||||
if let Ok(stream_num) = stream_id_str.parse::<u16>() {
|
||||
map.insert(info_id, stream_num);
|
||||
fn parse_playback_config(text: &str, map: &mut HashMap<String, u16>) {
|
||||
for tag_name in ["AudioStreams", "SubtitlesStreams"] {
|
||||
let mut from = 0;
|
||||
while let Some((start, end)) = xml::find_element(text, tag_name, from) {
|
||||
let block = &text[start..end];
|
||||
if let (Some(stream_id_str), Some(info_id)) = (
|
||||
xml::text(block, "StreamID"),
|
||||
xml::text(block, "StreamInfo_ID"),
|
||||
) {
|
||||
if let Ok(stream_num) = stream_id_str.parse::<u16>() {
|
||||
map.insert(info_id, stream_num);
|
||||
}
|
||||
}
|
||||
from = end;
|
||||
}
|
||||
|
||||
pos = block_end;
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_tag(xml: &str, tag: &str) -> Option<String> {
|
||||
let open = format!("<{}>", tag);
|
||||
let close = format!("</{}>", tag);
|
||||
let start = xml.find(&open)? + open.len();
|
||||
let end = xml[start..].find(&close)? + start;
|
||||
Some(xml[start..end].trim().to_string())
|
||||
}
|
||||
|
||||
+185
-17
@@ -4,8 +4,8 @@
|
||||
//! When both exist, language_streams.txt provides structured types while
|
||||
//! menu_base.prop provides stream number → button name mapping.
|
||||
|
||||
use super::{vocab, LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, vocab};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -14,7 +14,7 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
|| super::jar_file_exists(udf, "language_streams.txt")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
// Try language_streams.txt first (richer structured data)
|
||||
let ls_labels = parse_language_streams(reader, udf);
|
||||
|
||||
@@ -22,12 +22,18 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
let mb_labels = parse_menu_base(reader, udf);
|
||||
|
||||
// If we have both, merge: language_streams for structure, menu_base for names
|
||||
match (ls_labels, mb_labels) {
|
||||
(Some(ls), Some(mb)) => Some(merge(ls, mb)),
|
||||
(Some(ls), None) => Some(ls),
|
||||
(None, Some(mb)) => Some(mb),
|
||||
(None, None) => None,
|
||||
let labels = match (ls_labels, mb_labels) {
|
||||
(Some(ls), Some(mb)) => merge(ls, mb),
|
||||
(Some(ls), None) => ls,
|
||||
(None, Some(mb)) => mb,
|
||||
(None, None) => return None,
|
||||
};
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// High confidence: both language_streams.txt and menu_base.prop
|
||||
// are structured key-value formats with documented types.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
|
||||
fn merge(ls: Vec<StreamLabel>, mb: Vec<StreamLabel>) -> Vec<StreamLabel> {
|
||||
@@ -49,7 +55,7 @@ fn merge(ls: Vec<StreamLabel>, mb: Vec<StreamLabel>) -> Vec<StreamLabel> {
|
||||
|
||||
// ── language_streams.txt parser ────────────────────────────────────────────
|
||||
|
||||
fn parse_language_streams(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
fn parse_language_streams(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
let data = super::read_jar_file(reader, udf, "language_streams.txt")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
@@ -172,9 +178,167 @@ fn parse_language_streams(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<
|
||||
Some(labels)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Build a minimal menu_base.prop text and run `parse_menu_base`'s
|
||||
/// inner logic via a temporary closure. This isolates the prop
|
||||
/// parsing without needing a SectorSource.
|
||||
fn parse_props(text: &str) -> Vec<StreamLabel> {
|
||||
// Mirror the inner loop of parse_menu_base exactly. Kept
|
||||
// separate so the test doesn't need disc fixtures.
|
||||
use std::collections::HashMap;
|
||||
let mut entries: HashMap<String, HashMap<String, String>> = HashMap::new();
|
||||
for line in text.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let Some(eq_pos) = line.find('=') else {
|
||||
continue;
|
||||
};
|
||||
let full_key = &line[..eq_pos];
|
||||
let value = &line[eq_pos + 1..];
|
||||
if let Some(dot_pos) = full_key.rfind('.') {
|
||||
entries
|
||||
.entry(full_key[..dot_pos].to_string())
|
||||
.or_default()
|
||||
.insert(full_key[dot_pos + 1..].to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
let mut labels = Vec::new();
|
||||
for (prefix, props) in &entries {
|
||||
let is_audio = props
|
||||
.get("class")
|
||||
.is_some_and(|c| c.contains("AudioButton"))
|
||||
|| prefix.starts_with("audio_");
|
||||
let is_subtitle = props
|
||||
.get("class")
|
||||
.is_some_and(|c| c.contains("SubtitleButton"))
|
||||
|| prefix.starts_with("subtitle_");
|
||||
let stream_num_str = props
|
||||
.get("streamNumber")
|
||||
.or_else(|| props.get("audioStream"))
|
||||
.or_else(|| props.get("subtitleStream"));
|
||||
let stream_num: u16 = match stream_num_str.and_then(|s| s.parse().ok()) {
|
||||
Some(n) if n > 0 => n,
|
||||
_ => continue,
|
||||
};
|
||||
if !is_audio && !is_subtitle {
|
||||
continue;
|
||||
}
|
||||
let name = props.get("name").cloned().unwrap_or_default();
|
||||
let purpose = match vocab::purpose(&name) {
|
||||
LabelPurpose::Normal if prefix.contains("comm") => LabelPurpose::Commentary,
|
||||
p => p,
|
||||
};
|
||||
let qualifier = if is_subtitle {
|
||||
vocab::qualifier(&name)
|
||||
} else {
|
||||
LabelQualifier::None
|
||||
};
|
||||
let stream_type = if is_audio {
|
||||
StreamLabelType::Audio
|
||||
} else {
|
||||
StreamLabelType::Subtitle
|
||||
};
|
||||
let language = props
|
||||
.get("audioLanguage")
|
||||
.or_else(|| props.get("subtitleLanguage"))
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
labels.push(StreamLabel {
|
||||
stream_number: stream_num,
|
||||
stream_type,
|
||||
language,
|
||||
name,
|
||||
purpose,
|
||||
qualifier,
|
||||
codec_hint: String::new(),
|
||||
variant: String::new(),
|
||||
});
|
||||
}
|
||||
labels.sort_by_key(|l| (l.stream_type as u8, l.stream_number));
|
||||
labels
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commentary_via_name() {
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=2\n\
|
||||
audio_1.name=Director's Commentary\n\
|
||||
audio_1.audioLanguage=eng\n",
|
||||
);
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Commentary);
|
||||
assert_eq!(labels[0].language, "eng");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commentary_via_prefix_when_name_silent() {
|
||||
let labels = parse_props(
|
||||
"audio_commentary_1.class=AudioButton\n\
|
||||
audio_commentary_1.streamNumber=2\n\
|
||||
audio_commentary_1.name=Track 2\n\
|
||||
audio_commentary_1.audioLanguage=eng\n",
|
||||
);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commenter_does_not_false_match_commentary() {
|
||||
// Regression for the pre-refactor `name.contains("comment")`
|
||||
// bug: this would wrongly classify a "Commenter Pro" track as
|
||||
// Commentary. vocab::purpose enforces a word boundary.
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=2\n\
|
||||
audio_1.name=Commenter Pro Track\n\
|
||||
audio_1.audioLanguage=eng\n",
|
||||
);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptive_via_name() {
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=3\n\
|
||||
audio_1.name=English Descriptive Audio\n",
|
||||
);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sdh_only_on_subtitles() {
|
||||
// SDH applied to a subtitle stream.
|
||||
let labels = parse_props(
|
||||
"subtitle_1.class=SubtitleButton\n\
|
||||
subtitle_1.streamNumber=4\n\
|
||||
subtitle_1.name=English SDH\n",
|
||||
);
|
||||
assert_eq!(labels[0].qualifier, LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sdh_not_applied_to_audio_stream_even_if_name_contains_sdh() {
|
||||
// Audio streams should not pick up SDH (it's a subtitle
|
||||
// concept). Edge case: badly-authored name happens to include
|
||||
// "SDH" — we don't propagate it to audio metadata.
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=5\n\
|
||||
audio_1.name=English SDH (track?)\n",
|
||||
);
|
||||
assert_eq!(labels[0].qualifier, LabelQualifier::None);
|
||||
}
|
||||
}
|
||||
|
||||
// ── menu_base.prop parser ──────────────────────────────────────────────────
|
||||
|
||||
fn parse_menu_base(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
fn parse_menu_base(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
let data = super::read_jar_file(reader, udf, "menu_base.prop")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
@@ -231,16 +395,20 @@ fn parse_menu_base(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<Str
|
||||
}
|
||||
|
||||
let name = props.get("name").cloned().unwrap_or_default();
|
||||
let name_lower = name.to_lowercase();
|
||||
|
||||
let purpose = if name_lower.contains("comment") || prefix.contains("comm") {
|
||||
LabelPurpose::Commentary
|
||||
} else {
|
||||
LabelPurpose::Normal
|
||||
// Purpose: ask vocab first (word-boundary matched — avoids the
|
||||
// "Commenter" false positive the prior `name.contains("comment")`
|
||||
// had). Then fall back to the structural prefix check
|
||||
// (`audio_commentary.foo`-style keys group commentary streams
|
||||
// regardless of display name).
|
||||
let purpose = match vocab::purpose(&name) {
|
||||
LabelPurpose::Normal if prefix.contains("comm") => LabelPurpose::Commentary,
|
||||
p => p,
|
||||
};
|
||||
|
||||
let qualifier = if is_subtitle && name_lower.contains("sdh") {
|
||||
LabelQualifier::Sdh
|
||||
// Qualifier: only apply to subtitles (SDH is a subtitle concept).
|
||||
let qualifier = if is_subtitle {
|
||||
vocab::qualifier(&name)
|
||||
} else {
|
||||
LabelQualifier::None
|
||||
};
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
//! "dbp" framework — Magnolia Pictures BD-J authoring shop (per
|
||||
//! `bd-live.magpictures.com` referenced in the disc's
|
||||
//! `com/dbp/bluray.MenuXlet.perm`). Detected on UHD discs whose
|
||||
//! `/BDMV/JAR/<x>.jar` (top-level, not in a subdir) contains
|
||||
//! `com/dbp/` package paths.
|
||||
//!
|
||||
//! Stream labels live as plain ASCII strings inside compiled `.class`
|
||||
//! files in the jar — a quirk of the menu-rendering layer encoding
|
||||
//! its TextField positions and content as constant strings the
|
||||
//! Java compiler retained in the class string pool. Format observed
|
||||
//! in the corpus (Civil War UHD, 2024):
|
||||
//!
|
||||
//! ```text
|
||||
//! LTextField,Audio1,English Dolby Atmos,Fontstrip_Composite,...
|
||||
//! RTextField,Audio2,English Descriptive Audio,Fontstrip_Composite,...
|
||||
//! HTextField,Subtitle1,English SDH,Fontstrip_Composite,...
|
||||
//! ATextField,Subtitle0,None,Fontstrip_Composite,...
|
||||
//! ```
|
||||
//!
|
||||
//! The single uppercase letter before `TextField` is string-pool
|
||||
//! prefix noise — the parser anchors on `TextField,` regardless of
|
||||
//! what precedes it. `Subtitle0` is the disable-subtitles menu
|
||||
//! button and is skipped (not a real subtitle stream).
|
||||
//!
|
||||
//! ## Implementation
|
||||
//!
|
||||
//! v2 (2026-05-10): rewritten on top of [`super::class_reader`] —
|
||||
//! iterates `CpInfo::Utf8` constant-pool entries instead of raw byte
|
||||
//! scanning each class file. Equivalent label coverage (the literal
|
||||
//! `TextField,...` strings live in the CP as Utf8 entries), but
|
||||
//! structurally cleaner: no false-positive risk from method bytecode
|
||||
//! or attribute names happening to contain `TextField,`. Language /
|
||||
//! purpose / qualifier classification moved to [`super::vocab`] so all
|
||||
//! Java-parser families share one source of truth.
|
||||
|
||||
use super::class_reader::CpInfo;
|
||||
use super::{ParseResult, StreamLabel, StreamLabelType, jar, vocab};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// dbp detect can't peek inside a jar without a SectorSource (the
|
||||
/// trait function only takes `&UdfFs`), so we trigger on the cheap
|
||||
/// signal "any top-level .jar in /BDMV/JAR/." That fires on every
|
||||
/// BD-J disc, but parse() does the real `com/dbp/` check and
|
||||
/// returns None on a mismatch — so this parser only ever consumes
|
||||
/// time on discs that fell through every earlier parser.
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
jar::has_any_top_level_jar(udf)
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
jar::for_each_jar(reader, udf, |_entry_name, archive| {
|
||||
if !jar::has_path_prefix(archive, "com/dbp/") {
|
||||
return None;
|
||||
}
|
||||
let labels = scan_jar(archive);
|
||||
if labels.is_empty() {
|
||||
None
|
||||
} else {
|
||||
// High confidence: TextField,Audio1,... is a stable anchor
|
||||
// pattern + vocab routes language/purpose/qualifier.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn scan_jar(archive: &mut jar::Jar) -> Vec<StreamLabel> {
|
||||
// BTreeMap so we keep the highest-numbered (last-written) label
|
||||
// for each stream slot deterministic across runs. The same
|
||||
// TextField,Audio1,... string can appear in multiple classes
|
||||
// (button-state variants, localization fallbacks). Last write
|
||||
// wins — they should all agree, but the structure is defensive.
|
||||
let mut audios: BTreeMap<u16, String> = BTreeMap::new();
|
||||
let mut subs: BTreeMap<u16, String> = BTreeMap::new();
|
||||
|
||||
jar::for_each_class(archive, |_class_name, class| {
|
||||
for (_idx, cp) in class.constant_pool.iter() {
|
||||
if let CpInfo::Utf8(s) = cp {
|
||||
collect_textfield(s, &mut audios, &mut subs);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut out = Vec::new();
|
||||
for (num, label) in audios {
|
||||
out.push(make_label(num, label, StreamLabelType::Audio));
|
||||
}
|
||||
for (num, label) in subs {
|
||||
out.push(make_label(num, label, StreamLabelType::Subtitle));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn collect_textfield(
|
||||
s: &str,
|
||||
audios: &mut BTreeMap<u16, String>,
|
||||
subs: &mut BTreeMap<u16, String>,
|
||||
) {
|
||||
// Anchor on "TextField," — the prefix character before it varies
|
||||
// (string-pool ordering inside compiled Java) and is irrelevant.
|
||||
let Some(idx) = s.find("TextField,") else {
|
||||
return;
|
||||
};
|
||||
let after = &s[idx + "TextField,".len()..];
|
||||
let mut parts = after.splitn(3, ',');
|
||||
let kind_n = parts.next().unwrap_or("").trim();
|
||||
let label = parts.next().unwrap_or("").trim();
|
||||
if label.is_empty() {
|
||||
return;
|
||||
}
|
||||
if let Some(rest) = kind_n.strip_prefix("Audio") {
|
||||
if let Ok(n) = rest.parse::<u16>() {
|
||||
audios.insert(n, label.to_string());
|
||||
}
|
||||
} else if let Some(rest) = kind_n.strip_prefix("Subtitle") {
|
||||
if let Ok(n) = rest.parse::<u16>() {
|
||||
// Subtitle0 is conventionally the "None / Off" disable
|
||||
// button, not an actual subtitle stream.
|
||||
if n > 0 {
|
||||
subs.insert(n, label.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn make_label(num: u16, label: String, stream_type: StreamLabelType) -> StreamLabel {
|
||||
let lang_info = vocab::lang(&label);
|
||||
let language = lang_info.map(|l| l.code).unwrap_or("").to_string();
|
||||
let variant = lang_info.map(|l| l.variant).unwrap_or("").to_string();
|
||||
let qualifier = vocab::qualifier(&label);
|
||||
let purpose = vocab::purpose(&label);
|
||||
StreamLabel {
|
||||
stream_number: num,
|
||||
stream_type,
|
||||
language,
|
||||
name: label,
|
||||
purpose,
|
||||
qualifier,
|
||||
codec_hint: String::new(),
|
||||
variant,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::{LabelPurpose, LabelQualifier};
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn collect_extracts_audio_and_subtitle_indices() {
|
||||
let mut audios = BTreeMap::new();
|
||||
let mut subs = BTreeMap::new();
|
||||
let lines = [
|
||||
"LTextField,Audio1,English Dolby Atmos,Fontstrip_Composite,296,763,275,25,left",
|
||||
"RTextField,Audio2,English Descriptive Audio,Fontstrip_Composite,296,803,275,25,left",
|
||||
"RTextField,Audio3,Spanish 5.1 Dolby Digital,Fontstrip_Composite,296,843,275,25,left",
|
||||
"ATextField,Subtitle0,None,Fontstrip_Composite,1312,843,275,25,left",
|
||||
"HTextField,Subtitle1,English SDH,Fontstrip_Composite,1312,763,275,25,left",
|
||||
"DTextField,Subtitle2,Spanish,Fontstrip_Composite,1312,803,275,25,left",
|
||||
];
|
||||
for s in &lines {
|
||||
collect_textfield(s, &mut audios, &mut subs);
|
||||
}
|
||||
assert_eq!(audios.len(), 3);
|
||||
assert_eq!(audios[&1], "English Dolby Atmos");
|
||||
assert_eq!(audios[&2], "English Descriptive Audio");
|
||||
assert_eq!(audios[&3], "Spanish 5.1 Dolby Digital");
|
||||
// Subtitle0 ("None") is skipped — disable button, not a stream.
|
||||
assert_eq!(subs.len(), 2);
|
||||
assert_eq!(subs[&1], "English SDH");
|
||||
assert_eq!(subs[&2], "Spanish");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_ignores_non_textfield_strings() {
|
||||
let mut audios = BTreeMap::new();
|
||||
let mut subs = BTreeMap::new();
|
||||
for s in [
|
||||
"GraphicButton,SU_Audio",
|
||||
"AudioMenu",
|
||||
"CommentaryMenuAlternateScenes",
|
||||
"PrimaryAudioControl",
|
||||
] {
|
||||
collect_textfield(s, &mut audios, &mut subs);
|
||||
}
|
||||
assert!(audios.is_empty());
|
||||
assert!(subs.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_routes_via_vocab() {
|
||||
let l = make_label(1, "English SDH".to_string(), StreamLabelType::Subtitle);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.qualifier, LabelQualifier::Sdh);
|
||||
assert_eq!(l.purpose, LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_descriptive_audio() {
|
||||
let l = make_label(
|
||||
2,
|
||||
"English Descriptive Audio".to_string(),
|
||||
StreamLabelType::Audio,
|
||||
);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.purpose, LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_commentary() {
|
||||
let l = make_label(
|
||||
3,
|
||||
"English Director's Commentary".to_string(),
|
||||
StreamLabelType::Audio,
|
||||
);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_compound_languages_populate_variant() {
|
||||
let brazilian = make_label(1, "Brazilian Portuguese 5.1".into(), StreamLabelType::Audio);
|
||||
assert_eq!(brazilian.language, "por");
|
||||
assert_eq!(brazilian.variant, "Brazilian");
|
||||
|
||||
let castilian = make_label(1, "Castilian Spanish".into(), StreamLabelType::Audio);
|
||||
assert_eq!(castilian.language, "spa");
|
||||
assert_eq!(castilian.variant, "Castilian");
|
||||
|
||||
let canadian = make_label(
|
||||
1,
|
||||
"Canadian French Dolby Digital".into(),
|
||||
StreamLabelType::Audio,
|
||||
);
|
||||
assert_eq!(canadian.language, "fra");
|
||||
assert_eq!(canadian.variant, "Canadian");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_bare_language_has_empty_variant() {
|
||||
let l = make_label(1, "English Dolby Atmos".into(), StreamLabelType::Audio);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.variant, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_unknown_language_is_empty() {
|
||||
// vocab::lang returns None — make_label converts both fields to "".
|
||||
let l = make_label(1, "Klingon Dolby Atmos".into(), StreamLabelType::Audio);
|
||||
assert_eq!(l.language, "");
|
||||
assert_eq!(l.variant, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_rnib_descriptive_service() {
|
||||
let l = make_label(1, "English RNIB".into(), StreamLabelType::Subtitle);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.qualifier, LabelQualifier::DescriptiveService);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,151 @@
|
||||
//! BD-J jar utilities — common scaffolding for parsers that read
|
||||
//! `/BDMV/JAR/*.jar`.
|
||||
//!
|
||||
//! Composes with [`class_reader`](super::class_reader) for structured
|
||||
//! `.class` access. Used by `dbp` (string-pool scan via constant pool)
|
||||
//! and `deluxe` (bytecode pattern matching) — those parsers express
|
||||
//! "open every top-level jar, look at every .class inside" without
|
||||
//! repeating the zip-archive boilerplate.
|
||||
|
||||
// `try_each_class` is staged for `labels::deluxe`, which needs the
|
||||
// early-return form to short-circuit class iteration on a match.
|
||||
// dead-code allow comes off when deluxe lands.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use super::class_reader::ClassFile;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::io::Cursor;
|
||||
use zip::ZipArchive;
|
||||
|
||||
/// In-memory zip archive: backed by a `Vec<u8>` read from UDF. Owns
|
||||
/// the buffer; callers pass it to [`has_path_prefix`], [`for_each_class`],
|
||||
/// etc.
|
||||
pub type Jar = ZipArchive<Cursor<Vec<u8>>>;
|
||||
|
||||
/// True if `/BDMV/JAR/` contains at least one top-level `.jar` file
|
||||
/// (not under a subdir). Used by `detect()` in parsers whose real
|
||||
/// signal lives inside a jar — they can't open the jar without a
|
||||
/// `SectorSource`, so they use this cheap pre-check and do the real
|
||||
/// `com/<vendor>/` discriminator in `parse()`.
|
||||
pub fn has_any_top_level_jar(udf: &UdfFs) -> bool {
|
||||
let Some(jar_dir) = udf.find_dir("/BDMV/JAR") else {
|
||||
return false;
|
||||
};
|
||||
jar_dir
|
||||
.entries
|
||||
.iter()
|
||||
.any(|e| !e.is_dir && e.name.to_lowercase().ends_with(".jar"))
|
||||
}
|
||||
|
||||
/// Open every top-level `*.jar` entry in `/BDMV/JAR/` and yield each
|
||||
/// `(entry_name, Jar)` to `f`. Returns the first `Some(R)` the callback
|
||||
/// produces, or `None` if every jar was visited without a hit.
|
||||
///
|
||||
/// "Top-level" means entries directly under `/BDMV/JAR/`, not nested
|
||||
/// under a subdir. (Pixelogic, Criterion, Paramount, etc. put their
|
||||
/// data files inside `/BDMV/JAR/<x>/`; dbp and Deluxe put their jar
|
||||
/// directly at `/BDMV/JAR/<name>.jar`.)
|
||||
///
|
||||
/// Entries that fail to read from UDF or that aren't valid zips are
|
||||
/// silently skipped — same defensive shape as the existing dbp parser.
|
||||
pub fn for_each_jar<R, F>(reader: &mut dyn SectorSource, udf: &UdfFs, mut f: F) -> Option<R>
|
||||
where
|
||||
F: FnMut(&str, &mut Jar) -> Option<R>,
|
||||
{
|
||||
let jar_dir = udf.find_dir("/BDMV/JAR")?;
|
||||
for entry in &jar_dir.entries {
|
||||
if entry.is_dir {
|
||||
continue;
|
||||
}
|
||||
if !entry.name.to_lowercase().ends_with(".jar") {
|
||||
continue;
|
||||
}
|
||||
let path = format!("/BDMV/JAR/{}", entry.name);
|
||||
let Ok(bytes) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(mut archive) = ZipArchive::new(Cursor::new(bytes)) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(r) = f(&entry.name, &mut archive) {
|
||||
return Some(r);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// True if any entry in this jar's central directory starts with
|
||||
/// `prefix`. Fast — only reads filenames, never extracts bytes.
|
||||
///
|
||||
/// Used by parsers as a cheap "is this MY framework's jar?" check
|
||||
/// (e.g. `has_path_prefix(archive, "com/dbp/")` for dbp,
|
||||
/// `has_path_prefix(archive, "com/bydeluxe/")` for Deluxe).
|
||||
pub fn has_path_prefix(archive: &mut Jar, prefix: &str) -> bool {
|
||||
for i in 0..archive.len() {
|
||||
if let Ok(f) = archive.by_index(i) {
|
||||
if f.name().starts_with(prefix) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Iterate every `.class` entry in the jar, parse it with
|
||||
/// [`class_reader`], and call `f` with `(entry_name, &ClassFile)`.
|
||||
///
|
||||
/// Entries that fail to read or parse are silently skipped — this is
|
||||
/// label-extraction code, robustness matters more than completeness.
|
||||
/// Callers that need to know which classes failed should use the
|
||||
/// lower-level [`class_reader`] API directly.
|
||||
pub fn for_each_class<F>(archive: &mut Jar, mut f: F)
|
||||
where
|
||||
F: FnMut(&str, &ClassFile),
|
||||
{
|
||||
for i in 0..archive.len() {
|
||||
let Ok(mut entry) = archive.by_index(i) else {
|
||||
continue;
|
||||
};
|
||||
if !entry.name().ends_with(".class") {
|
||||
continue;
|
||||
}
|
||||
let name = entry.name().to_string();
|
||||
let mut bytes = Vec::with_capacity(entry.size() as usize);
|
||||
if std::io::Read::read_to_end(&mut entry, &mut bytes).is_err() {
|
||||
continue;
|
||||
}
|
||||
let Ok(class) = ClassFile::parse(&bytes) else {
|
||||
continue;
|
||||
};
|
||||
f(&name, &class);
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`for_each_class`] but allows the callback to short-circuit
|
||||
/// iteration. Returns the first `Some(R)` the callback produces.
|
||||
pub fn try_each_class<R, F>(archive: &mut Jar, mut f: F) -> Option<R>
|
||||
where
|
||||
F: FnMut(&str, &ClassFile) -> Option<R>,
|
||||
{
|
||||
for i in 0..archive.len() {
|
||||
let Ok(mut entry) = archive.by_index(i) else {
|
||||
continue;
|
||||
};
|
||||
if !entry.name().ends_with(".class") {
|
||||
continue;
|
||||
}
|
||||
let name = entry.name().to_string();
|
||||
let mut bytes = Vec::with_capacity(entry.size() as usize);
|
||||
if std::io::Read::read_to_end(&mut entry, &mut bytes).is_err() {
|
||||
continue;
|
||||
}
|
||||
let Ok(class) = ClassFile::parse(&bytes) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(r) = f(&name, &class) {
|
||||
return Some(r);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
+1270
-26
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,597 @@
|
||||
//! Universal MPLS-based stream labels.
|
||||
//!
|
||||
//! Unlike the framework-specific parsers in this directory (dbp,
|
||||
//! pixelogic, ctrm, criterion, ...), this module is the *floor*:
|
||||
//! every Blu-ray ships with MPLS playlists under `/BDMV/PLAYLIST/`,
|
||||
//! and every MPLS file has an STN table with per-stream ISO 639-2
|
||||
//! language codes plus coding-type / channel-layout / sample-rate
|
||||
//! bytes from the BD spec.
|
||||
//!
|
||||
//! The framework parsers extract richer editorial labels ("English
|
||||
//! Dolby Atmos", "Director's Commentary") when the disc was authored
|
||||
//! with a recognized tool. When none of them match (e.g. a "no BD-J"
|
||||
//! disc, or an authoring framework we haven't catalogued), MPLS still
|
||||
//! gives us language + codec on every stream — enough to render
|
||||
//! something more useful than the bare PID.
|
||||
//!
|
||||
//! Output confidence is Low: MPLS carries language + codec but
|
||||
//! never purpose/qualifier info (no way to tell "Commentary" from
|
||||
//! "Normal" from the STN table alone). Higher-confidence framework
|
||||
//! parsers, when present, always win on the registry's max-by-confidence
|
||||
//! tiebreaker — MPLS is only chosen when nothing else matched.
|
||||
|
||||
use super::{
|
||||
LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType,
|
||||
vocab::{self, LangInfo},
|
||||
};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
/// True iff `/BDMV/PLAYLIST/` exists and contains at least one
|
||||
/// `.mpls` file. Cheap directory walk only — no sector reads.
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
let Some(dir) = udf.find_dir("/BDMV/PLAYLIST") else {
|
||||
return false;
|
||||
};
|
||||
dir.entries
|
||||
.iter()
|
||||
.any(|e| !e.is_dir && has_mpls_extension(&e.name))
|
||||
}
|
||||
|
||||
/// Walk every `*.mpls` in `/BDMV/PLAYLIST/`, parse it, and convert
|
||||
/// each StreamEntry to a [`StreamLabel`]. Streams shared across
|
||||
/// playlists (same PID) are deduped.
|
||||
///
|
||||
/// Returns `None` if no labels could be produced (e.g. no .mpls files
|
||||
/// parsed successfully, or every parsed stream was a type we skip
|
||||
/// like IG / DV EL).
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let playlist_dir = udf.find_dir("/BDMV/PLAYLIST")?;
|
||||
|
||||
// Collect mpls filenames first so we don't hold a borrow on udf
|
||||
// while we call udf.read_file (which takes &self).
|
||||
let mpls_names: Vec<String> = playlist_dir
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && has_mpls_extension(&e.name))
|
||||
.map(|e| e.name.clone())
|
||||
.collect();
|
||||
|
||||
if mpls_names.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut labels: Vec<StreamLabel> = Vec::new();
|
||||
// (stream_type_tag, language, codec_hint, pid) — PID is the
|
||||
// canonical "same physical stream" key; type+lang+codec round
|
||||
// out the rare case where two distinct logical streams happen
|
||||
// to share a PID across playlists with different metadata.
|
||||
let mut seen: Vec<(u8, String, String, u16)> = Vec::new();
|
||||
|
||||
// Global 1-based counters keyed by StreamLabelType. Incremented
|
||||
// only when an entry survives dedup, so stream_numbers are dense
|
||||
// (1, 2, 3, ...) per type across the whole disc — not reset per
|
||||
// playlist. A disc with 2 MPLS files that each list the same
|
||||
// 8 audio streams ends up with audio_1..audio_8, not audio_1..
|
||||
// audio_16 or audio_1..audio_8 with audio_1 duplicated.
|
||||
let mut audio_idx: u16 = 0;
|
||||
let mut sub_idx: u16 = 0;
|
||||
|
||||
for name in &mpls_names {
|
||||
let path = format!("/BDMV/PLAYLIST/{}", name);
|
||||
let Ok(data) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(playlist) = crate::mpls::parse(&data) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
for entry in &playlist.streams {
|
||||
let label_type = match entry.stream_type {
|
||||
2 | 5 => StreamLabelType::Audio, // primary + secondary audio
|
||||
3 => StreamLabelType::Subtitle, // PG subtitle
|
||||
// 1 = primary video, 6 = secondary video, 7 = DV EL
|
||||
// → no StreamLabelType variant for video, skip.
|
||||
// 4 = IG (interactive graphics) — not a user-facing
|
||||
// stream, skip.
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
let language = normalize_language(&entry.language);
|
||||
let name = language_display_name(&language);
|
||||
let codec_hint = build_codec_hint(label_type, entry);
|
||||
|
||||
let type_tag = type_tag(label_type);
|
||||
let key = (type_tag, language.clone(), codec_hint.clone(), entry.pid);
|
||||
if seen.contains(&key) {
|
||||
continue;
|
||||
}
|
||||
seen.push(key);
|
||||
|
||||
let stream_number = match label_type {
|
||||
StreamLabelType::Audio => {
|
||||
audio_idx += 1;
|
||||
audio_idx
|
||||
}
|
||||
StreamLabelType::Subtitle => {
|
||||
sub_idx += 1;
|
||||
sub_idx
|
||||
}
|
||||
};
|
||||
|
||||
labels.push(StreamLabel {
|
||||
stream_number,
|
||||
stream_type: label_type,
|
||||
language,
|
||||
name,
|
||||
purpose: LabelPurpose::Normal,
|
||||
qualifier: LabelQualifier::None,
|
||||
codec_hint,
|
||||
variant: String::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// MPLS gives language + codec but never editorial info (no
|
||||
// commentary/SDH/director's cut). Low confidence means framework
|
||||
// parsers (paramount, criterion, pixelogic, ctrm, dbp, deluxe) always
|
||||
// win when they match. MPLS only gets chosen as the parser when
|
||||
// nothing else fired — exactly the universal-fallback role we want.
|
||||
Some(ParseResult::low(labels))
|
||||
}
|
||||
|
||||
fn has_mpls_extension(name: &str) -> bool {
|
||||
// Case-insensitive ".mpls" suffix. Some discs use uppercase,
|
||||
// some lowercase; UDF filenames preserve case but we don't.
|
||||
let n = name.len();
|
||||
if n < 5 {
|
||||
return false;
|
||||
}
|
||||
name[n - 5..].eq_ignore_ascii_case(".mpls")
|
||||
}
|
||||
|
||||
/// Lowercase + trim the raw 3-char ISO 639-2 code. If the lowered
|
||||
/// string maps via [`vocab::lang`] (it won't for plain "eng" — that
|
||||
/// matcher is for English-name fragments, not codes) use its
|
||||
/// canonical code; otherwise return the trimmed lowercase string.
|
||||
fn normalize_language(raw: &str) -> String {
|
||||
let trimmed = raw.trim().to_ascii_lowercase();
|
||||
if trimmed.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
// vocab::lang() matches free-form English names, not ISO 639-2
|
||||
// codes — so for the typical MPLS payload ("eng", "fra", ...)
|
||||
// it returns None and we keep the trimmed code.
|
||||
if let Some(LangInfo { code, .. }) = vocab::lang(&trimmed) {
|
||||
return code.to_string();
|
||||
}
|
||||
trimmed
|
||||
}
|
||||
|
||||
/// Human-readable English name for an ISO 639-2 code, or empty if
|
||||
/// the code is unknown. Kept inline rather than in vocab because
|
||||
/// vocab is the *reverse* mapping (name → code).
|
||||
pub(crate) fn language_display_name(iso: &str) -> String {
|
||||
match iso {
|
||||
"eng" => "English",
|
||||
"fra" | "fre" => "French",
|
||||
"spa" => "Spanish",
|
||||
"deu" | "ger" => "German",
|
||||
"ita" => "Italian",
|
||||
"jpn" => "Japanese",
|
||||
"zho" | "chi" => "Chinese",
|
||||
"kor" => "Korean",
|
||||
"por" => "Portuguese",
|
||||
"pol" => "Polish",
|
||||
"ces" | "cze" => "Czech",
|
||||
"hun" => "Hungarian",
|
||||
"nld" | "dut" => "Dutch",
|
||||
"ara" => "Arabic",
|
||||
"hin" => "Hindi",
|
||||
"tur" => "Turkish",
|
||||
"tha" => "Thai",
|
||||
"swe" => "Swedish",
|
||||
"nor" => "Norwegian",
|
||||
"dan" => "Danish",
|
||||
"fin" => "Finnish",
|
||||
"heb" => "Hebrew",
|
||||
"rus" => "Russian",
|
||||
"ell" | "gre" => "Greek",
|
||||
"vie" => "Vietnamese",
|
||||
"ind" => "Indonesian",
|
||||
"msa" | "may" => "Malay",
|
||||
"ukr" => "Ukrainian",
|
||||
"ron" | "rum" => "Romanian",
|
||||
"bul" => "Bulgarian",
|
||||
"hrv" => "Croatian",
|
||||
"srp" => "Serbian",
|
||||
"slk" | "slo" => "Slovak",
|
||||
"slv" => "Slovenian",
|
||||
"est" => "Estonian",
|
||||
"lav" => "Latvian",
|
||||
"lit" => "Lithuanian",
|
||||
"isl" | "ice" => "Icelandic",
|
||||
"eus" | "baq" => "Basque",
|
||||
"cat" => "Catalan",
|
||||
"glg" => "Galician",
|
||||
_ => "",
|
||||
}
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Map BD coding_type byte → codec name. Returns empty for unknown
|
||||
/// bytes (the table covers everything the spec defines, but unknown
|
||||
/// values are still possible on malformed discs).
|
||||
pub(crate) fn codec_name(coding_type: u8) -> &'static str {
|
||||
match coding_type {
|
||||
0x02 => "MPEG-2",
|
||||
0x1B => "H.264",
|
||||
0x24 => "HEVC",
|
||||
0x80 => "LPCM",
|
||||
0x81 => "AC-3",
|
||||
0x82 => "DTS",
|
||||
0x83 => "TrueHD",
|
||||
0x84 => "AC-3+",
|
||||
0x85 => "DTS-HD",
|
||||
0x86 => "DTS-HD MA",
|
||||
0x90 => "PG",
|
||||
0x91 => "IG",
|
||||
0xA1 => "AC-3+ Secondary",
|
||||
0xA2 => "DTS-HD Secondary",
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the final `codec_hint`. For audio streams, optionally
|
||||
/// append " <channels>" and/or " <rate>" suffixes. Sample rate is
|
||||
/// only spelled out for non-48k (the universal default).
|
||||
fn build_codec_hint(label_type: StreamLabelType, entry: &crate::mpls::StreamEntry) -> String {
|
||||
let base = codec_name(entry.coding_type);
|
||||
if base.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
if label_type != StreamLabelType::Audio {
|
||||
return base.to_string();
|
||||
}
|
||||
|
||||
let mut out = base.to_string();
|
||||
|
||||
let channels = match entry.audio_format {
|
||||
1 => Some("mono"),
|
||||
3 => Some("2.0"),
|
||||
6 => Some("5.1"),
|
||||
12 => Some("7.1"),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(ch) = channels {
|
||||
out.push(' ');
|
||||
out.push_str(ch);
|
||||
}
|
||||
|
||||
// 1 = 48 kHz (universal default, omit). Only call out higher rates.
|
||||
let rate = match entry.audio_rate {
|
||||
4 => Some("96kHz"),
|
||||
5 => Some("192kHz"),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(r) = rate {
|
||||
out.push(' ');
|
||||
out.push_str(r);
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
/// Dedup-tag for the label type. `u8` instead of `StreamLabelType`
|
||||
/// itself because the enum does not derive `Hash` / `Eq`-by-discriminant
|
||||
/// in a way that we want to couple to (and `==` works fine for the
|
||||
/// linear `Vec::contains` lookup we do).
|
||||
fn type_tag(t: StreamLabelType) -> u8 {
|
||||
match t {
|
||||
StreamLabelType::Audio => 1,
|
||||
StreamLabelType::Subtitle => 2,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::mpls::{Playlist, StreamEntry};
|
||||
|
||||
fn audio_entry(pid: u16, coding: u8, fmt: u8, rate: u8, lang: &str) -> StreamEntry {
|
||||
StreamEntry {
|
||||
stream_type: 2,
|
||||
pid,
|
||||
coding_type: coding,
|
||||
video_format: 0,
|
||||
video_rate: 0,
|
||||
audio_format: fmt,
|
||||
audio_rate: rate,
|
||||
language: lang.to_string(),
|
||||
dynamic_range: 0,
|
||||
color_space: 0,
|
||||
secondary: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn pg_entry(pid: u16, lang: &str) -> StreamEntry {
|
||||
StreamEntry {
|
||||
stream_type: 3,
|
||||
pid,
|
||||
coding_type: 0x90,
|
||||
video_format: 0,
|
||||
video_rate: 0,
|
||||
audio_format: 0,
|
||||
audio_rate: 0,
|
||||
language: lang.to_string(),
|
||||
dynamic_range: 0,
|
||||
color_space: 0,
|
||||
secondary: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn playlist_with(streams: Vec<StreamEntry>) -> Playlist {
|
||||
Playlist {
|
||||
version: "0200".to_string(),
|
||||
play_items: Vec::new(),
|
||||
streams,
|
||||
marks: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Drive the same conversion logic that `parse()` runs on real
|
||||
/// disc data, but starting from already-parsed Playlists so we
|
||||
/// don't have to synthesize valid MPLS bytes.
|
||||
fn labels_from_playlists(playlists: &[Playlist]) -> Vec<StreamLabel> {
|
||||
let mut labels: Vec<StreamLabel> = Vec::new();
|
||||
let mut seen: Vec<(u8, String, String, u16)> = Vec::new();
|
||||
|
||||
for playlist in playlists {
|
||||
let mut audio_idx: u16 = 0;
|
||||
let mut sub_idx: u16 = 0;
|
||||
|
||||
for entry in &playlist.streams {
|
||||
let label_type = match entry.stream_type {
|
||||
2 | 5 => StreamLabelType::Audio,
|
||||
3 => StreamLabelType::Subtitle,
|
||||
_ => continue,
|
||||
};
|
||||
let stream_number = match label_type {
|
||||
StreamLabelType::Audio => {
|
||||
audio_idx += 1;
|
||||
audio_idx
|
||||
}
|
||||
StreamLabelType::Subtitle => {
|
||||
sub_idx += 1;
|
||||
sub_idx
|
||||
}
|
||||
};
|
||||
let language = normalize_language(&entry.language);
|
||||
let name = language_display_name(&language);
|
||||
let codec_hint = build_codec_hint(label_type, entry);
|
||||
let key = (
|
||||
type_tag(label_type),
|
||||
language.clone(),
|
||||
codec_hint.clone(),
|
||||
entry.pid,
|
||||
);
|
||||
if seen.contains(&key) {
|
||||
continue;
|
||||
}
|
||||
seen.push(key);
|
||||
labels.push(StreamLabel {
|
||||
stream_number,
|
||||
stream_type: label_type,
|
||||
language,
|
||||
name,
|
||||
purpose: LabelPurpose::Normal,
|
||||
qualifier: LabelQualifier::None,
|
||||
codec_hint,
|
||||
variant: String::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
labels
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mpls_audio_streams_become_labels() {
|
||||
// Two audio streams: English TrueHD 7.1 48k, French AC-3 5.1 48k.
|
||||
let pl = playlist_with(vec![
|
||||
audio_entry(0x1100, 0x83, 12, 1, "eng"),
|
||||
audio_entry(0x1101, 0x81, 6, 1, "fra"),
|
||||
]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert_eq!(labels.len(), 2);
|
||||
|
||||
// English TrueHD 7.1
|
||||
let a = &labels[0];
|
||||
assert_eq!(a.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(a.stream_number, 1);
|
||||
assert_eq!(a.language, "eng");
|
||||
assert_eq!(a.name, "English");
|
||||
assert_eq!(a.codec_hint, "TrueHD 7.1");
|
||||
assert_eq!(a.purpose, LabelPurpose::Normal);
|
||||
assert_eq!(a.qualifier, LabelQualifier::None);
|
||||
assert_eq!(a.variant, "");
|
||||
|
||||
// French AC-3 5.1
|
||||
let b = &labels[1];
|
||||
assert_eq!(b.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(b.stream_number, 2);
|
||||
assert_eq!(b.language, "fra");
|
||||
assert_eq!(b.name, "French");
|
||||
assert_eq!(b.codec_hint, "AC-3 5.1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mpls_pg_streams_become_subtitle_labels() {
|
||||
let pl = playlist_with(vec![
|
||||
pg_entry(0x1200, "eng"),
|
||||
pg_entry(0x1201, "spa"),
|
||||
pg_entry(0x1202, "fra"),
|
||||
]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert_eq!(labels.len(), 3);
|
||||
for label in &labels {
|
||||
assert_eq!(label.stream_type, StreamLabelType::Subtitle);
|
||||
assert_eq!(label.codec_hint, "PG");
|
||||
}
|
||||
assert_eq!(labels[0].stream_number, 1);
|
||||
assert_eq!(labels[0].language, "eng");
|
||||
assert_eq!(labels[0].name, "English");
|
||||
assert_eq!(labels[1].stream_number, 2);
|
||||
assert_eq!(labels[1].language, "spa");
|
||||
assert_eq!(labels[1].name, "Spanish");
|
||||
assert_eq!(labels[2].stream_number, 3);
|
||||
assert_eq!(labels[2].language, "fra");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_streams_across_playlists() {
|
||||
// Two playlists, same English TrueHD 7.1 PID 0x1100 in both.
|
||||
// Expect one Audio label, not two.
|
||||
let pl1 = playlist_with(vec![
|
||||
audio_entry(0x1100, 0x83, 12, 1, "eng"),
|
||||
audio_entry(0x1101, 0x81, 6, 1, "fra"),
|
||||
]);
|
||||
let pl2 = playlist_with(vec![
|
||||
audio_entry(0x1100, 0x83, 12, 1, "eng"), // duplicate
|
||||
audio_entry(0x1102, 0x82, 6, 1, "deu"), // new
|
||||
]);
|
||||
let labels = labels_from_playlists(&[pl1, pl2]);
|
||||
// Expected: eng@0x1100, fra@0x1101, deu@0x1102 — three uniques.
|
||||
assert_eq!(labels.len(), 3);
|
||||
// PID isn't stored on StreamLabel, so assert on the surviving
|
||||
// language set instead.
|
||||
let mut langs: Vec<String> = labels.iter().map(|l| l.language.clone()).collect();
|
||||
langs.sort();
|
||||
assert_eq!(langs, vec!["deu", "eng", "fra"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coding_type_to_codec_hint_table() {
|
||||
// Spot-check every entry in the spec table. Audio entries
|
||||
// come back bare (no channels/rate set) so codec_hint is the
|
||||
// codec name alone.
|
||||
let cases: &[(u8, &str)] = &[
|
||||
(0x02, "MPEG-2"),
|
||||
(0x1B, "H.264"),
|
||||
(0x24, "HEVC"),
|
||||
(0x80, "LPCM"),
|
||||
(0x81, "AC-3"),
|
||||
(0x82, "DTS"),
|
||||
(0x83, "TrueHD"),
|
||||
(0x84, "AC-3+"),
|
||||
(0x85, "DTS-HD"),
|
||||
(0x86, "DTS-HD MA"),
|
||||
(0x90, "PG"),
|
||||
(0x91, "IG"),
|
||||
(0xA1, "AC-3+ Secondary"),
|
||||
(0xA2, "DTS-HD Secondary"),
|
||||
];
|
||||
for (ct, expected) in cases {
|
||||
assert_eq!(
|
||||
codec_name(*ct),
|
||||
*expected,
|
||||
"coding_type 0x{:02X} should map to {}",
|
||||
ct,
|
||||
expected
|
||||
);
|
||||
}
|
||||
// Unknown bytes return empty.
|
||||
assert_eq!(codec_name(0x00), "");
|
||||
assert_eq!(codec_name(0xFF), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_format_appends_channel_layout() {
|
||||
let mono = audio_entry(1, 0x83, 1, 1, "eng");
|
||||
let stereo = audio_entry(2, 0x83, 3, 1, "eng");
|
||||
let surround_51 = audio_entry(3, 0x83, 6, 1, "eng");
|
||||
let surround_71 = audio_entry(4, 0x83, 12, 1, "eng");
|
||||
let unknown = audio_entry(5, 0x83, 0, 1, "eng");
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &mono),
|
||||
"TrueHD mono"
|
||||
);
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &stereo),
|
||||
"TrueHD 2.0"
|
||||
);
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &surround_51),
|
||||
"TrueHD 5.1"
|
||||
);
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &surround_71),
|
||||
"TrueHD 7.1"
|
||||
);
|
||||
assert_eq!(build_codec_hint(StreamLabelType::Audio, &unknown), "TrueHD");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_rate_only_shows_above_48k() {
|
||||
// 48 kHz (rate=1) is the universal default → not surfaced.
|
||||
let r48 = audio_entry(1, 0x83, 6, 1, "eng");
|
||||
// 96 kHz (rate=4) → surfaced.
|
||||
let r96 = audio_entry(2, 0x83, 6, 4, "eng");
|
||||
// 192 kHz (rate=5) → surfaced.
|
||||
let r192 = audio_entry(3, 0x83, 6, 5, "eng");
|
||||
assert_eq!(build_codec_hint(StreamLabelType::Audio, &r48), "TrueHD 5.1");
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &r96),
|
||||
"TrueHD 5.1 96kHz"
|
||||
);
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &r192),
|
||||
"TrueHD 5.1 192kHz"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_iso_code_passes_through_without_display_name() {
|
||||
// Made-up code: keep the raw lowercase code as `language`,
|
||||
// but `name` is empty because we don't know it.
|
||||
let pl = playlist_with(vec![audio_entry(0x1100, 0x83, 6, 1, "xyz")]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].language, "xyz");
|
||||
assert_eq!(labels[0].name, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ig_and_dv_streams_are_skipped() {
|
||||
// stream_type 4 = IG, 7 = DV EL — both must not surface.
|
||||
let mut ig = pg_entry(0x1400, "eng");
|
||||
ig.stream_type = 4;
|
||||
let mut dv = audio_entry(0x1011, 0x24, 0, 0, "");
|
||||
dv.stream_type = 7;
|
||||
let pl = playlist_with(vec![ig, dv]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert!(labels.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secondary_audio_becomes_audio_label() {
|
||||
// stream_type 5 = secondary audio. The conversion should
|
||||
// still produce an Audio label (the registry's apply path
|
||||
// can ignore secondary if it wants — this module just
|
||||
// surfaces what's there).
|
||||
let mut sec = audio_entry(0x1A00, 0x83, 3, 1, "eng");
|
||||
sec.stream_type = 5;
|
||||
sec.secondary = true;
|
||||
let pl = playlist_with(vec![sec]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(labels[0].codec_hint, "TrueHD 2.0");
|
||||
}
|
||||
}
|
||||
+20
-31
@@ -12,15 +12,15 @@
|
||||
//! sub_com1_idx="23,24,25" />
|
||||
//! ```
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, xml};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "playlists.xml")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let data = super::read_jar_file(reader, udf, "playlists.xml")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
@@ -30,8 +30,8 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
let mut labels = Vec::new();
|
||||
|
||||
// Parse audio streams
|
||||
if let Some(aud) = extract_attr(&feature, "aud") {
|
||||
let com_idx = extract_attr(&feature, "aud_com1_idx").and_then(|s| s.parse::<usize>().ok());
|
||||
if let Some(aud) = xml::attr(&feature, "aud") {
|
||||
let com_idx = xml::attr(&feature, "aud_com1_idx").and_then(|s| s.parse::<usize>().ok());
|
||||
|
||||
for (i, lang) in aud.split(',').enumerate() {
|
||||
let lang = lang.trim();
|
||||
@@ -57,12 +57,12 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
}
|
||||
|
||||
// Parse subtitle streams
|
||||
if let Some(sub) = extract_attr(&feature, "sub") {
|
||||
let forced: Vec<bool> = extract_attr(&feature, "forced_sub")
|
||||
if let Some(sub) = xml::attr(&feature, "sub") {
|
||||
let forced: Vec<bool> = xml::attr(&feature, "forced_sub")
|
||||
.map(|s| s.split(',').map(|f| f.trim() == "1").collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
let com_indices: Vec<usize> = extract_attr(&feature, "sub_com1_idx")
|
||||
let com_indices: Vec<usize> = xml::attr(&feature, "sub_com1_idx")
|
||||
.map(|s| s.split(',').filter_map(|i| i.trim().parse().ok()).collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
@@ -100,32 +100,29 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(labels)
|
||||
// High confidence: paramount's playlists.xml is fully structured
|
||||
// and we extract every documented field.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
|
||||
/// Find the feature playlist element (the one with the most audio tracks).
|
||||
fn find_feature_playlist(xml: &str) -> Option<String> {
|
||||
fn find_feature_playlist(text: &str) -> Option<String> {
|
||||
let mut best: Option<String> = None;
|
||||
let mut best_aud_count = 0;
|
||||
let mut from = 0;
|
||||
|
||||
let mut pos = 0;
|
||||
while let Some(start) = xml[pos..].find("<playlist ") {
|
||||
let abs_start = pos + start;
|
||||
let end = match xml[abs_start..].find("/>") {
|
||||
Some(p) => abs_start + p + 2,
|
||||
None => break,
|
||||
};
|
||||
let element = &xml[abs_start..end];
|
||||
while let Some((start, end)) = xml::find_element(text, "playlist", from) {
|
||||
let element = &text[start..end];
|
||||
|
||||
// Prefer name="Feature" explicitly
|
||||
if let Some(name) = extract_attr(element, "name") {
|
||||
// Prefer name="Feature" explicitly.
|
||||
if let Some(name) = xml::attr(element, "name") {
|
||||
if name.eq_ignore_ascii_case("Feature") {
|
||||
return Some(element.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise pick the one with the most audio streams
|
||||
if let Some(aud) = extract_attr(element, "aud") {
|
||||
// Otherwise pick the one with the most audio streams.
|
||||
if let Some(aud) = xml::attr(element, "aud") {
|
||||
let count = aud.split(',').count();
|
||||
if count > best_aud_count {
|
||||
best_aud_count = count;
|
||||
@@ -133,15 +130,7 @@ fn find_feature_playlist(xml: &str) -> Option<String> {
|
||||
}
|
||||
}
|
||||
|
||||
pos = end;
|
||||
from = end;
|
||||
}
|
||||
best
|
||||
}
|
||||
|
||||
/// Extract an XML attribute value from an element string.
|
||||
fn extract_attr(element: &str, name: &str) -> Option<String> {
|
||||
let needle = format!("{}=\"", name);
|
||||
let start = element.find(&needle)? + needle.len();
|
||||
let end = element[start..].find('"')? + start;
|
||||
Some(element[start..end].to_string())
|
||||
}
|
||||
|
||||
+102
-23
@@ -5,9 +5,13 @@
|
||||
//!
|
||||
//! Token format: `{lang}_{codec?}_{purpose?}_{region?}_`
|
||||
|
||||
use super::{vocab, LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{
|
||||
Confidence, LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, text,
|
||||
vocab,
|
||||
};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
/// Known audio codec tokens
|
||||
const AUDIO_CODECS: &[&str] = &["MLP", "AC3", "DTS", "DDL", "WAV", "AC"];
|
||||
@@ -20,9 +24,18 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "bluray_project.bin")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let data = super::read_jar_file(reader, udf, "bluray_project.bin")?;
|
||||
let strings = extract_strings(&data);
|
||||
// min_len=4 matches the prior local extract_strings impl. The token
|
||||
// grammar is `{lang3}_{codec?}_{purpose?}_{region?}_` so the
|
||||
// shortest meaningful run is 4 chars (lang + underscore).
|
||||
let strings = text::extract_ascii_strings(&data, 4);
|
||||
|
||||
// Tracked across all parse_token calls in this run: did any stream
|
||||
// hit an unrecognized token component (skip-unknown path)? If yes
|
||||
// we downgrade confidence to Medium — the labels are still valid
|
||||
// but the corpus surfaced something we don't catalogue.
|
||||
let saw_unknown = AtomicBool::new(false);
|
||||
|
||||
let mut labels = Vec::new();
|
||||
let mut in_feature = false;
|
||||
@@ -50,7 +63,7 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(label) = parse_token(s) {
|
||||
if let Some(label) = parse_token_inner(s, Some(&saw_unknown)) {
|
||||
match label.stream_type {
|
||||
StreamLabelType::Audio => {
|
||||
audio_num += 1;
|
||||
@@ -73,10 +86,15 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(labels)
|
||||
let confidence = if saw_unknown.load(Ordering::Relaxed) {
|
||||
Confidence::Medium
|
||||
} else {
|
||||
Confidence::High
|
||||
};
|
||||
Some(ParseResult { labels, confidence })
|
||||
}
|
||||
|
||||
fn parse_token(s: &str) -> Option<StreamLabel> {
|
||||
fn parse_token_inner(s: &str, saw_unknown: Option<&AtomicBool>) -> Option<StreamLabel> {
|
||||
let clean = s.trim().trim_start_matches('\t').trim_end_matches('_');
|
||||
let parts: Vec<&str> = clean.split('_').collect();
|
||||
if parts.len() < 2 {
|
||||
@@ -127,7 +145,18 @@ fn parse_token(s: &str) -> Option<StreamLabel> {
|
||||
} else if part.starts_with("PGStream") {
|
||||
is_subtitle = true;
|
||||
} else {
|
||||
return None;
|
||||
// Unknown token component — skip this single part rather
|
||||
// than discarding the entire stream record. Pre-refactor
|
||||
// behavior was `return None` here, which silently dropped
|
||||
// any stream containing a single uncatalogued token (e.g.
|
||||
// a new codec ID or framework variant). Better to surface
|
||||
// what we know than discard a whole stream over one part,
|
||||
// but flag the parse as Medium-confidence so callers know
|
||||
// some data was elided.
|
||||
tracing::debug!(part = %part, "pixelogic: unrecognized token component, skipping");
|
||||
if let Some(flag) = saw_unknown {
|
||||
flag.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,22 +182,72 @@ fn parse_token(s: &str) -> Option<StreamLabel> {
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_strings(data: &[u8]) -> Vec<String> {
|
||||
let mut strings = Vec::new();
|
||||
let mut current = String::new();
|
||||
// extract_strings removed — replaced by super::text::extract_ascii_strings(data, 4).
|
||||
|
||||
for &b in data {
|
||||
if (0x20..0x7f).contains(&b) {
|
||||
current.push(b as char);
|
||||
} else {
|
||||
if current.len() > 3 {
|
||||
strings.push(current.clone());
|
||||
}
|
||||
current.clear();
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_token_basic_audio() {
|
||||
let l = parse_token_inner("eng_MLP_", None).unwrap();
|
||||
assert_eq!(l.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.codec_hint, "TrueHD");
|
||||
assert_eq!(l.purpose, LabelPurpose::Normal);
|
||||
}
|
||||
if current.len() > 3 {
|
||||
strings.push(current);
|
||||
|
||||
#[test]
|
||||
fn parse_token_basic_subtitle_sdh() {
|
||||
let l = parse_token_inner("eng_SDH_", None).unwrap();
|
||||
assert_eq!(l.stream_type, StreamLabelType::Subtitle);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.qualifier, LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_commentary() {
|
||||
let l = parse_token_inner("eng_MLP_ACOM_", None).unwrap();
|
||||
assert_eq!(l.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(l.purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_descriptive() {
|
||||
let l = parse_token_inner("eng_AC3_ADES_", None).unwrap();
|
||||
assert_eq!(l.purpose, LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_with_region() {
|
||||
let l = parse_token_inner("eng_MLP_US_", None).unwrap();
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.variant, "US");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_unknown_component_does_not_kill_stream() {
|
||||
// Regression: pre-refactor, an unrecognized token part returned
|
||||
// None for the whole stream, silently dropping it. New
|
||||
// behavior: skip the unknown part, surface what we know.
|
||||
let l = parse_token_inner("eng_MLP_FUTUREFLAG_FOR_", None).unwrap();
|
||||
assert_eq!(l.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.codec_hint, "TrueHD");
|
||||
assert_eq!(l.qualifier, LabelQualifier::Forced);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_no_audio_or_subtitle_signal_returns_none() {
|
||||
// A token that has only a language and an unknown part with
|
||||
// no audio/subtitle classifier should still return None —
|
||||
// there's no way to file it as a stream.
|
||||
assert!(parse_token_inner("eng_UNKNOWN_", None).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_rejects_non_lang_prefix() {
|
||||
assert!(parse_token_inner("XX_MLP_", None).is_none());
|
||||
assert!(parse_token_inner("ENG_MLP_", None).is_none()); // uppercase not accepted as ISO 639-2
|
||||
}
|
||||
strings
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
//! Text-extraction helpers used by parsers that scan binary blobs for
|
||||
//! embedded label strings.
|
||||
//!
|
||||
//! Promoted from two near-duplicate implementations:
|
||||
//! - `pixelogic::extract_strings` (`bluray_project.bin`, min_len=4)
|
||||
//! - `dbp::extract_printable` (`.class` files in jars, min_len=5)
|
||||
//!
|
||||
//! Single implementation, threshold passed in. Callers that have a
|
||||
//! more structured parse path (e.g. `class_reader` for .class) should
|
||||
//! prefer that — this helper is for genuinely unstructured input.
|
||||
|
||||
/// Walk `data`, emit every maximal run of printable-ASCII bytes
|
||||
/// (`0x20..=0x7E`) whose length is at least `min_len`.
|
||||
///
|
||||
/// Non-printable bytes (including `\t`, `\n`, NUL) terminate the
|
||||
/// current run. Output strings are guaranteed valid UTF-8 (they're
|
||||
/// pure 7-bit ASCII). Strings shorter than `min_len` are dropped.
|
||||
pub fn extract_ascii_strings(data: &[u8], min_len: usize) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
let mut current = String::new();
|
||||
for &b in data {
|
||||
if (0x20..=0x7E).contains(&b) {
|
||||
current.push(b as char);
|
||||
} else if current.len() >= min_len {
|
||||
out.push(std::mem::take(&mut current));
|
||||
} else {
|
||||
current.clear();
|
||||
}
|
||||
}
|
||||
if current.len() >= min_len {
|
||||
out.push(current);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn extracts_simple_runs() {
|
||||
let got = extract_ascii_strings(b"hello\0world\0", 3);
|
||||
assert_eq!(got, vec!["hello", "world"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn applies_minimum_length() {
|
||||
let got = extract_ascii_strings(b"hi\0ok\0longer\0", 5);
|
||||
assert_eq!(got, vec!["longer"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn treats_tab_and_newline_as_separators() {
|
||||
// \t (0x09) and \n (0x0A) are below 0x20, so they break runs.
|
||||
let got = extract_ascii_strings(b"alpha\tbeta\ngamma", 3);
|
||||
assert_eq!(got, vec!["alpha", "beta", "gamma"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handles_trailing_run_without_terminator() {
|
||||
// Run at the very end of the buffer should still be emitted.
|
||||
let got = extract_ascii_strings(b"prefix\0tail", 3);
|
||||
assert_eq!(got, vec!["prefix", "tail"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_high_bit_bytes() {
|
||||
// 0x80+ is non-printable per this helper's definition.
|
||||
let mut buf = b"good".to_vec();
|
||||
buf.push(0xC3);
|
||||
buf.push(0xA9);
|
||||
buf.extend_from_slice(b"more");
|
||||
let got = extract_ascii_strings(&buf, 3);
|
||||
assert_eq!(got, vec!["good", "more"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_returns_empty() {
|
||||
let got = extract_ascii_strings(&[], 1);
|
||||
assert!(got.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn min_len_zero_emits_singletons() {
|
||||
// Pathological but well-defined.
|
||||
let got = extract_ascii_strings(b"a\0b", 0);
|
||||
assert_eq!(got, vec!["a", "b"]);
|
||||
}
|
||||
}
|
||||
+432
-9
@@ -1,19 +1,40 @@
|
||||
//! Shared label vocabulary — values we are 100% confident about.
|
||||
//! Shared label vocabulary — canonical mappings used by ≥1 label parser.
|
||||
//!
|
||||
//! Labels come from BD-J authoring tool files (bluray_project.bin,
|
||||
//! playlists.xml, menu_base.prop, etc.) — NOT from BD spec fields.
|
||||
//! This is NOT for MPLS/CLPI/STN data. Those follow the BD spec directly.
|
||||
//! playlists.xml, menu_base.prop, .class string pools, etc.) — NOT
|
||||
//! from BD spec fields. This module is the central, regression-tested
|
||||
//! source of truth for:
|
||||
//!
|
||||
//! Rules:
|
||||
//! - Only map values we are 100% certain about (published codec names).
|
||||
//! - Unknown codes (csp, eda, cf, etc.) pass through raw from disc.
|
||||
//! - The app/CLI handles display text, not the lib.
|
||||
//! - Codec brand name aliases (`MLP` → `TrueHD`).
|
||||
//! - English / multi-word language name → ISO 639-2 code.
|
||||
//! - English text → [`LabelPurpose`] (Commentary / Descriptive / etc.).
|
||||
//! - English text → [`LabelQualifier`] (SDH / Forced / Descriptive Service).
|
||||
//!
|
||||
//! Rules of engagement (carried over from
|
||||
//! `(internal)/memory/feedback_label_data_rules.md`):
|
||||
//!
|
||||
//! 1. Only map values we are 100% certain about — published codec
|
||||
//! names, well-known ISO 639-2 mappings, vendor-documented purpose
|
||||
//! keywords.
|
||||
//! 2. Unknown codes / unrecognized phrases pass through raw or return
|
||||
//! `None`. We never guess.
|
||||
//! 3. Matching is case-insensitive and word-boundary-aware where
|
||||
//! relevant (so "Commenter" doesn't match "commentary"). Anchoring
|
||||
//! on whole tokens is the responsibility of this module — callers
|
||||
//! pass raw text, we handle it.
|
||||
//!
|
||||
//! This module is NOT for BD spec STN codec IDs; those decode in
|
||||
//! `mpls.rs` separately.
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier};
|
||||
|
||||
// ── Codec aliases ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Map a codec identifier found in label data to its display name.
|
||||
///
|
||||
/// These are well-known codec identifiers used across multiple BD-J
|
||||
/// authoring tools. NOT BD spec STN codec IDs — those are decoded
|
||||
/// separately in mpls.rs.
|
||||
/// authoring tools. Unknown codes pass through unchanged so callers
|
||||
/// can still surface vendor-specific tokens we haven't catalogued.
|
||||
pub fn codec(code: &str) -> &str {
|
||||
match code {
|
||||
"MLP" => "TrueHD",
|
||||
@@ -25,3 +46,405 @@ pub fn codec(code: &str) -> &str {
|
||||
_ => code,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Language: English / multi-word names → ISO 639-2 ─────────────────────────
|
||||
|
||||
/// Result of [`lang`] — ISO code + human-readable regional variant.
|
||||
///
|
||||
/// `code` is ISO 639-2 (always 3 lowercase letters).
|
||||
/// `variant` is the regional dialect as a human-readable English word
|
||||
/// (`"Brazilian"`, `"Castilian"`, `"Canadian"`, `"Simplified"`, ...)
|
||||
/// or `""` when the input names just a bare language without
|
||||
/// dialect ("Spanish" → variant=""). The variant matches the
|
||||
/// convention pixelogic / ctrm / criterion already use for their
|
||||
/// `StreamLabel::variant` field: a short display token the UI can
|
||||
/// surface verbatim.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct LangInfo {
|
||||
pub code: &'static str,
|
||||
pub variant: &'static str,
|
||||
}
|
||||
|
||||
/// Map a free-form language label fragment to an ISO 639-2 code AND
|
||||
/// (where applicable) its regional variant.
|
||||
///
|
||||
/// Handles both bare English names ("English", "Spanish") and the
|
||||
/// multi-word vendor variants we've seen in the corpus ("Brazilian
|
||||
/// Portuguese", "Castilian Spanish", "Canadian French"). Match is
|
||||
/// case-insensitive; longer compound phrases win over their bare
|
||||
/// counterparts (so "Brazilian Portuguese" returns
|
||||
/// `LangInfo { code: "por", variant: "Brazilian" }`, not consumed by
|
||||
/// the bare "Portuguese" entry).
|
||||
///
|
||||
/// Bare-name matches return `variant: ""`.
|
||||
///
|
||||
/// Returns `None` for unrecognized input — callers decide whether to
|
||||
/// fall back to MPLS spec codes, pass through raw, or drop the stream.
|
||||
/// Never guesses.
|
||||
///
|
||||
/// Why the variant: the prior `lang() -> Option<&str>` shape silently
|
||||
/// dropped regional dialect info. "Brazilian Portuguese 5.1" became
|
||||
/// `language="por", variant=""` — UI displayed plain "Portuguese"
|
||||
/// even though the disc had explicitly labeled this stream Brazilian.
|
||||
/// Capturing the variant here parallels how pixelogic and ctrm
|
||||
/// populate `StreamLabel::variant` from their own region tables.
|
||||
pub fn lang(text: &str) -> Option<LangInfo> {
|
||||
let lower = text.to_lowercase();
|
||||
// Multi-word compounds first — longest-match wins.
|
||||
for (needle, code, variant) in COMPOUND_LANGS {
|
||||
if lower.contains(needle) {
|
||||
return Some(LangInfo { code, variant });
|
||||
}
|
||||
}
|
||||
// Bare names: word-boundary match (avoid "english" inside "englishman"
|
||||
// or any other accidental substring).
|
||||
for (needle, code) in BARE_LANGS {
|
||||
if has_word(&lower, needle) {
|
||||
return Some(LangInfo { code, variant: "" });
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
const COMPOUND_LANGS: &[(&str, &str, &str)] = &[
|
||||
("brazilian portuguese", "por", "Brazilian"),
|
||||
("euro portuguese", "por", "European"),
|
||||
("european portuguese", "por", "European"),
|
||||
("castilian spanish", "spa", "Castilian"),
|
||||
("latin american spanish", "spa", "Latin American"),
|
||||
("latin spanish", "spa", "Latin American"),
|
||||
("canadian french", "fra", "Canadian"),
|
||||
("parisian french", "fra", "Parisian"),
|
||||
("australian english", "eng", "Australian"),
|
||||
("austrailian english", "eng", "Australian"), // disc-corpus typo, keep matching
|
||||
("british english", "eng", "British"),
|
||||
("simplified chinese", "zho", "Simplified"),
|
||||
("traditional chinese", "zho", "Traditional"),
|
||||
("mandarin chinese", "zho", "Mandarin"),
|
||||
("cantonese chinese", "zho", "Cantonese"),
|
||||
];
|
||||
|
||||
const BARE_LANGS: &[(&str, &str)] = &[
|
||||
("english", "eng"),
|
||||
("spanish", "spa"),
|
||||
("french", "fra"),
|
||||
("german", "deu"),
|
||||
("italian", "ita"),
|
||||
("japanese", "jpn"),
|
||||
("chinese", "zho"),
|
||||
("mandarin", "zho"),
|
||||
("cantonese", "zho"),
|
||||
("portuguese", "por"),
|
||||
("polish", "pol"),
|
||||
("czech", "ces"),
|
||||
("hungarian", "hun"),
|
||||
("dutch", "nld"),
|
||||
("korean", "kor"),
|
||||
("arabic", "ara"),
|
||||
("hindi", "hin"),
|
||||
("turkish", "tur"),
|
||||
("thai", "tha"),
|
||||
("swedish", "swe"),
|
||||
("norwegian", "nor"),
|
||||
("danish", "dan"),
|
||||
("finnish", "fin"),
|
||||
("hebrew", "heb"),
|
||||
("russian", "rus"),
|
||||
("greek", "ell"),
|
||||
("vietnamese", "vie"),
|
||||
("indonesian", "ind"),
|
||||
("malay", "msa"),
|
||||
("ukrainian", "ukr"),
|
||||
("romanian", "ron"),
|
||||
("bulgarian", "bul"),
|
||||
("croatian", "hrv"),
|
||||
("serbian", "srp"),
|
||||
("slovak", "slk"),
|
||||
("slovenian", "slv"),
|
||||
("estonian", "est"),
|
||||
("latvian", "lav"),
|
||||
("lithuanian", "lit"),
|
||||
("icelandic", "isl"),
|
||||
("basque", "eus"),
|
||||
("catalan", "cat"),
|
||||
("galician", "glg"),
|
||||
];
|
||||
|
||||
// ── Purpose ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Classify a free-form English label string into a [`LabelPurpose`].
|
||||
///
|
||||
/// Recognized keywords (case-insensitive, word-boundary matched):
|
||||
/// - "commentary", "director's commentary" → `Commentary`
|
||||
/// - "descriptive", "description", "audio description", "described" → `Descriptive`
|
||||
/// - "score", "music only" → `Score`
|
||||
/// - "ime" (alternate music for closing themes etc.) → `Ime`
|
||||
/// - anything else → `Normal`
|
||||
///
|
||||
/// Word-boundary matching means "Commentary track" matches but
|
||||
/// "Commenter Pro audio" does not.
|
||||
pub fn purpose(text: &str) -> LabelPurpose {
|
||||
let lower = text.to_lowercase();
|
||||
// Multi-word compounds first — they're more specific.
|
||||
if lower.contains("audio description") || lower.contains("descriptive service") {
|
||||
return LabelPurpose::Descriptive;
|
||||
}
|
||||
if lower.contains("music only") {
|
||||
return LabelPurpose::Score;
|
||||
}
|
||||
if has_word(&lower, "commentary") {
|
||||
return LabelPurpose::Commentary;
|
||||
}
|
||||
if has_word(&lower, "descriptive")
|
||||
|| has_word(&lower, "description")
|
||||
|| has_word(&lower, "described")
|
||||
{
|
||||
return LabelPurpose::Descriptive;
|
||||
}
|
||||
if has_word(&lower, "score") {
|
||||
return LabelPurpose::Score;
|
||||
}
|
||||
if has_word(&lower, "ime") {
|
||||
return LabelPurpose::Ime;
|
||||
}
|
||||
LabelPurpose::Normal
|
||||
}
|
||||
|
||||
// ── Qualifier ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Classify a free-form English label string into a [`LabelQualifier`].
|
||||
///
|
||||
/// Recognized keywords (case-insensitive, word-boundary matched):
|
||||
/// - "sdh", "captions" → `Sdh`
|
||||
/// - "forced", "forced narrative" → `Forced`
|
||||
/// - "rnib", "descriptive service" → `DescriptiveService`
|
||||
/// - anything else → `None`
|
||||
///
|
||||
/// SDH (Subtitles for the Deaf and Hard of hearing) wins over Forced
|
||||
/// when both keywords are present, because an SDH track is its own
|
||||
/// stream regardless of whether the player flags it as "forced".
|
||||
pub fn qualifier(text: &str) -> LabelQualifier {
|
||||
let lower = text.to_lowercase();
|
||||
if has_word(&lower, "sdh") || has_word(&lower, "captions") {
|
||||
return LabelQualifier::Sdh;
|
||||
}
|
||||
if lower.contains("descriptive service") || has_word(&lower, "rnib") {
|
||||
return LabelQualifier::DescriptiveService;
|
||||
}
|
||||
if has_word(&lower, "forced") {
|
||||
return LabelQualifier::Forced;
|
||||
}
|
||||
LabelQualifier::None
|
||||
}
|
||||
|
||||
// ── Internal: word-boundary matching ────────────────────────────────────────
|
||||
|
||||
/// True if `needle` appears in `haystack` surrounded by non-alphanumeric
|
||||
/// boundaries (or string ends). `haystack` is assumed lowercase already.
|
||||
///
|
||||
/// This is the load-bearing primitive for `lang` / `purpose` /
|
||||
/// `qualifier`: bare-token matchers MUST use it, otherwise we match
|
||||
/// "english" inside "englishman" and "sdh" inside "lambdash". The
|
||||
/// existing parsers used `.contains()` and got lucky on the corpus;
|
||||
/// vocab guarantees the boundary.
|
||||
fn has_word(haystack: &str, needle: &str) -> bool {
|
||||
if needle.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let bytes = haystack.as_bytes();
|
||||
let nb = needle.as_bytes();
|
||||
let mut i = 0;
|
||||
while i + nb.len() <= bytes.len() {
|
||||
if &bytes[i..i + nb.len()] == nb {
|
||||
let before = if i == 0 { None } else { Some(bytes[i - 1]) };
|
||||
let after = bytes.get(i + nb.len()).copied();
|
||||
let bound = |c: Option<u8>| match c {
|
||||
None => true,
|
||||
Some(b) => !b.is_ascii_alphanumeric(),
|
||||
};
|
||||
if bound(before) && bound(after) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn codec_known_aliases() {
|
||||
assert_eq!(codec("MLP"), "TrueHD");
|
||||
assert_eq!(codec("AC3"), "Dolby Digital");
|
||||
assert_eq!(codec("AC"), "Dolby Digital");
|
||||
assert_eq!(codec("DDL"), "Dolby Digital Plus");
|
||||
assert_eq!(codec("atmos"), "Dolby Atmos");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_unknown_passes_through() {
|
||||
assert_eq!(codec("FX9"), "FX9");
|
||||
assert_eq!(codec(""), "");
|
||||
}
|
||||
|
||||
fn li(code: &'static str, variant: &'static str) -> LangInfo {
|
||||
LangInfo { code, variant }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_bare_names_have_empty_variant() {
|
||||
assert_eq!(lang("English"), Some(li("eng", "")));
|
||||
assert_eq!(lang("english"), Some(li("eng", "")));
|
||||
assert_eq!(lang("Spanish 5.1 Dolby Digital"), Some(li("spa", "")));
|
||||
assert_eq!(lang("japanese"), Some(li("jpn", "")));
|
||||
assert_eq!(lang("Italian"), Some(li("ita", "")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_compounds_carry_variant() {
|
||||
assert_eq!(
|
||||
lang("Brazilian Portuguese 5.1"),
|
||||
Some(li("por", "Brazilian"))
|
||||
);
|
||||
assert_eq!(lang("Castilian Spanish"), Some(li("spa", "Castilian")));
|
||||
assert_eq!(
|
||||
lang("Canadian French Dolby Digital"),
|
||||
Some(li("fra", "Canadian"))
|
||||
);
|
||||
assert_eq!(
|
||||
lang("Latin American Spanish"),
|
||||
Some(li("spa", "Latin American"))
|
||||
);
|
||||
assert_eq!(lang("Simplified Chinese"), Some(li("zho", "Simplified")));
|
||||
assert_eq!(lang("British English"), Some(li("eng", "British")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_compounds_win_over_bare() {
|
||||
// Brazilian Portuguese must map to (por, Brazilian) via the
|
||||
// compound rule, not be intercepted by bare "portuguese"
|
||||
// (which would yield (por, "") and lose the variant).
|
||||
assert_eq!(lang("Brazilian Portuguese").unwrap().variant, "Brazilian");
|
||||
assert_eq!(lang("Canadian French").unwrap().variant, "Canadian");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_unknown_returns_none() {
|
||||
assert_eq!(lang("Klingon Dolby Atmos"), None);
|
||||
assert_eq!(lang(""), None);
|
||||
assert_eq!(lang("eng"), None); // 3-letter codes are not English names
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_word_boundary_avoids_substring_false_positive() {
|
||||
// No false positive — "engineering" must NOT match "english".
|
||||
assert_eq!(lang("Audio Engineering Demo"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_recognizes_commentary() {
|
||||
assert_eq!(purpose("English Commentary"), LabelPurpose::Commentary);
|
||||
assert_eq!(purpose("Director's Commentary"), LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_recognizes_descriptive() {
|
||||
assert_eq!(
|
||||
purpose("English Descriptive Audio"),
|
||||
LabelPurpose::Descriptive
|
||||
);
|
||||
assert_eq!(purpose("Audio Description"), LabelPurpose::Descriptive);
|
||||
assert_eq!(purpose("Described Video"), LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_descriptive_service_routes_to_descriptive() {
|
||||
// "Descriptive Service" is qualifier territory but the purpose
|
||||
// implication is Descriptive — vocab::purpose treats it as such.
|
||||
assert_eq!(
|
||||
purpose("English Descriptive Service"),
|
||||
LabelPurpose::Descriptive
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_word_boundary_avoids_commenter_false_positive() {
|
||||
// "Commenter Pro audio" does NOT match "commentary" — the
|
||||
// existing dbp/ctrm hand-rolls would have. Vocab is stricter.
|
||||
assert_eq!(purpose("Commenter Pro audio track"), LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_recognizes_score() {
|
||||
assert_eq!(purpose("Music Only"), LabelPurpose::Score);
|
||||
assert_eq!(purpose("Isolated Score"), LabelPurpose::Score);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_unknown_is_normal() {
|
||||
assert_eq!(purpose("English Dolby Atmos"), LabelPurpose::Normal);
|
||||
assert_eq!(purpose(""), LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_recognizes_sdh() {
|
||||
assert_eq!(qualifier("English SDH"), LabelQualifier::Sdh);
|
||||
assert_eq!(qualifier("English Captions"), LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_recognizes_forced() {
|
||||
assert_eq!(qualifier("English Forced"), LabelQualifier::Forced);
|
||||
assert_eq!(qualifier("Forced Narrative"), LabelQualifier::Forced);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_recognizes_descriptive_service() {
|
||||
assert_eq!(
|
||||
qualifier("English RNIB"),
|
||||
LabelQualifier::DescriptiveService
|
||||
);
|
||||
assert_eq!(
|
||||
qualifier("English Descriptive Service"),
|
||||
LabelQualifier::DescriptiveService
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_sdh_wins_over_forced_when_both_present() {
|
||||
// SDH track is its own stream regardless of forced flag.
|
||||
assert_eq!(qualifier("English Forced SDH"), LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_unknown_is_none() {
|
||||
assert_eq!(qualifier("English"), LabelQualifier::None);
|
||||
assert_eq!(qualifier(""), LabelQualifier::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_word_basic() {
|
||||
assert!(has_word("english forced", "english"));
|
||||
assert!(has_word("english forced", "forced"));
|
||||
assert!(has_word("english", "english"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_word_rejects_substring() {
|
||||
assert!(!has_word("engineering", "english"));
|
||||
assert!(!has_word("englishman", "english"));
|
||||
assert!(!has_word("aenglish", "english"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_word_punctuation_boundary() {
|
||||
// "(SDH)" is a valid boundary — parentheses count as non-alphanum.
|
||||
assert!(has_word("english (sdh)", "sdh"));
|
||||
assert!(has_word("commentary,extra,info", "commentary"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,452 @@
|
||||
//! Tolerant XML scraping helpers — promoted from two near-duplicate
|
||||
//! hand-rolls in `paramount.rs` (attribute extraction) and `criterion.rs`
|
||||
//! (tag-text extraction).
|
||||
//!
|
||||
//! These are NOT a full XML parser. They handle the subset of XML the
|
||||
//! BD-J authoring tools we've seen actually emit: ASCII tag/attr
|
||||
//! names, no entity references inside label strings, optional XML
|
||||
//! namespaces. Hardening goals over the prior `find("<tag>")` /
|
||||
//! `find(r#"name=""#)` matchers:
|
||||
//!
|
||||
//! 1. **Case-insensitive** tag and attribute names — vendors casing
|
||||
//! is inconsistent across authoring-tool revisions.
|
||||
//! 2. **Namespace-aware** — strip an optional `ns:` prefix so
|
||||
//! `<ns:playlist>` and `<playlist>` both match.
|
||||
//! 3. **Whitespace-tolerant** — multiple/tab/newline characters
|
||||
//! around `=` between attribute name and value; whitespace inside
|
||||
//! the opening tag.
|
||||
//! 4. **Quote-style tolerant** — both `"value"` and `'value'`.
|
||||
//! 5. **Self-closing tag handling** — `<tag />` and `<tag/>` both
|
||||
//! work; [`text`] returns `Some("")` for empty content.
|
||||
//!
|
||||
//! Out of scope (intentionally simple): XML entity decoding
|
||||
//! (`&`, `<`, etc.), CDATA sections, comments, processing
|
||||
//! instructions, DTD declarations. None of the BD-J authored disc
|
||||
//! data we've observed exercises any of those — labels are plain
|
||||
//! ASCII/Latin-1 in attribute values.
|
||||
|
||||
/// Extract the value of attribute `name` from one XML element
|
||||
/// fragment (e.g. `<playlist name="Feature" id="00222" />`).
|
||||
///
|
||||
/// Returns the raw attribute text (no entity decoding) or `None` if
|
||||
/// the attribute isn't present. Empty string for `name=""` is
|
||||
/// represented as `Some("")`.
|
||||
pub fn attr(element: &str, name: &str) -> Option<String> {
|
||||
let bytes = element.as_bytes();
|
||||
let name_lower = name.to_ascii_lowercase();
|
||||
let name_bytes = name_lower.as_bytes();
|
||||
let mut i = 0;
|
||||
while i + name_bytes.len() < bytes.len() {
|
||||
// Find the next position where `name=` could start. We need
|
||||
// a word boundary before the name (whitespace or `<` or `:`).
|
||||
if i > 0 && is_name_char(bytes[i - 1]) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if !slice_eq_ignore_case(&bytes[i..i + name_bytes.len()], name_bytes) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
let after_name = i + name_bytes.len();
|
||||
// The character immediately after the name must not be a
|
||||
// name-continuation (otherwise we matched a prefix like
|
||||
// `lang_id` when looking for `lang`).
|
||||
if after_name < bytes.len() && is_name_char(bytes[after_name]) {
|
||||
i = after_name;
|
||||
continue;
|
||||
}
|
||||
// Walk past whitespace, then `=`, then more whitespace, then
|
||||
// the opening quote.
|
||||
let mut j = after_name;
|
||||
while j < bytes.len() && is_ws(bytes[j]) {
|
||||
j += 1;
|
||||
}
|
||||
if j >= bytes.len() || bytes[j] != b'=' {
|
||||
i = j.max(i + 1);
|
||||
continue;
|
||||
}
|
||||
j += 1; // past '='
|
||||
while j < bytes.len() && is_ws(bytes[j]) {
|
||||
j += 1;
|
||||
}
|
||||
if j >= bytes.len() {
|
||||
return None;
|
||||
}
|
||||
let quote = bytes[j];
|
||||
if quote != b'"' && quote != b'\'' {
|
||||
// Unquoted attribute values aren't part of well-formed
|
||||
// XML (HTML5 allows them, XML doesn't). Skip.
|
||||
i = j;
|
||||
continue;
|
||||
}
|
||||
let value_start = j + 1;
|
||||
let close = bytes[value_start..].iter().position(|&b| b == quote)?;
|
||||
let value = &element[value_start..value_start + close];
|
||||
return Some(value.to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Extract the trimmed text content of the first occurrence of
|
||||
/// `<tag>...</tag>` in `xml`. Returns `None` if the tag isn't found
|
||||
/// or its opening tag is malformed.
|
||||
///
|
||||
/// Whitespace around the inner text is stripped. Self-closing
|
||||
/// `<tag />` yields `Some("")`. Nested same-name tags are NOT
|
||||
/// handled — the first close encountered wins (this matches the
|
||||
/// prior behavior in criterion.rs).
|
||||
pub fn text(xml: &str, tag: &str) -> Option<String> {
|
||||
let (open_end, body_start) = find_open_tag(xml, tag, 0)?;
|
||||
// Self-closing — already consumed in find_open_tag if `/>`.
|
||||
if open_end == body_start {
|
||||
// Means find_open_tag returned the same offset twice for
|
||||
// self-closing form. (Not currently the case in our impl,
|
||||
// but defensive.)
|
||||
return Some(String::new());
|
||||
}
|
||||
// For self-closing tags, body_start is past `/>` and we have no
|
||||
// content. Detect by checking the char at body_start - 1 was `/`.
|
||||
if body_start >= 2 && &xml[body_start - 2..body_start] == "/>" {
|
||||
return Some(String::new());
|
||||
}
|
||||
// Find the matching close tag. Case-insensitive + namespace-aware.
|
||||
let close_start = find_close_tag(xml, tag, body_start)?;
|
||||
Some(xml[body_start..close_start].trim().to_string())
|
||||
}
|
||||
|
||||
/// Locate the next `<tag>` opening AND its closing `</tag>` in
|
||||
/// `xml`, starting at byte offset `from`. Returns `(element_start,
|
||||
/// element_end)` — `element_start` is the `<` of the opening tag,
|
||||
/// `element_end` is one past the `>` of the closing tag. Useful for
|
||||
/// iterating over repeated elements like `<playlist>` blocks in
|
||||
/// `paramount`.
|
||||
///
|
||||
/// Self-closing elements return the same offset for body_end as the
|
||||
/// element_end (i.e. `element_end - element_start` includes only the
|
||||
/// `<tag .../>` text).
|
||||
pub fn find_element(xml: &str, tag: &str, from: usize) -> Option<(usize, usize)> {
|
||||
let bytes = xml.as_bytes();
|
||||
let mut i = from;
|
||||
while i < bytes.len() {
|
||||
if bytes[i] != b'<' {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
// Try matching tag name at i+1 (after `<`).
|
||||
let after_lt = i + 1;
|
||||
if !matches_tag_name_at(bytes, after_lt, tag) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
// Found an open tag at offset i. Walk to find the closing `>`
|
||||
// of the open tag itself.
|
||||
let mut j = after_lt;
|
||||
// Skip past the tag name (and optional namespace prefix).
|
||||
while j < bytes.len() && (is_name_char(bytes[j]) || bytes[j] == b':') {
|
||||
j += 1;
|
||||
}
|
||||
// Walk attributes — track quoting state.
|
||||
let mut self_closing = false;
|
||||
while j < bytes.len() {
|
||||
match bytes[j] {
|
||||
b'>' => {
|
||||
j += 1;
|
||||
break;
|
||||
}
|
||||
b'/' if j + 1 < bytes.len() && bytes[j + 1] == b'>' => {
|
||||
self_closing = true;
|
||||
j += 2;
|
||||
break;
|
||||
}
|
||||
b'"' | b'\'' => {
|
||||
let q = bytes[j];
|
||||
j += 1;
|
||||
while j < bytes.len() && bytes[j] != q {
|
||||
j += 1;
|
||||
}
|
||||
if j < bytes.len() {
|
||||
j += 1;
|
||||
}
|
||||
}
|
||||
_ => j += 1,
|
||||
}
|
||||
}
|
||||
if self_closing {
|
||||
return Some((i, j));
|
||||
}
|
||||
// Find matching close. Doesn't handle nested same-name; OK
|
||||
// for our authoring-tool subset.
|
||||
let close_start = find_close_tag(xml, tag, j)?;
|
||||
let close_end = find_byte(bytes, b'>', close_start)? + 1;
|
||||
return Some((i, close_end));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
// ── Internal helpers ───────────────────────────────────────────────────────
|
||||
|
||||
/// True if `bytes[start..]` opens a tag named `tag`, allowing an
|
||||
/// optional `ns:` namespace prefix. Comparison is case-insensitive.
|
||||
/// The character after the tag name must not be a name-continuation
|
||||
/// (so `<player>` doesn't match `<play>`).
|
||||
fn matches_tag_name_at(bytes: &[u8], start: usize, tag: &str) -> bool {
|
||||
let tag_lower = tag.to_ascii_lowercase();
|
||||
let tag_bytes = tag_lower.as_bytes();
|
||||
// Skip optional `prefix:` (one or more name chars + `:`).
|
||||
let mut name_start = start;
|
||||
let mut scan = start;
|
||||
while scan < bytes.len() && is_name_char(bytes[scan]) {
|
||||
scan += 1;
|
||||
}
|
||||
if scan < bytes.len() && bytes[scan] == b':' {
|
||||
name_start = scan + 1;
|
||||
}
|
||||
if name_start + tag_bytes.len() > bytes.len() {
|
||||
return false;
|
||||
}
|
||||
if !slice_eq_ignore_case(&bytes[name_start..name_start + tag_bytes.len()], tag_bytes) {
|
||||
return false;
|
||||
}
|
||||
// Boundary: char after the tag name must be `>`, `/`, whitespace.
|
||||
let after = name_start + tag_bytes.len();
|
||||
if after >= bytes.len() {
|
||||
return false;
|
||||
}
|
||||
matches!(bytes[after], b'>' | b'/' | b' ' | b'\t' | b'\n' | b'\r')
|
||||
}
|
||||
|
||||
/// Find the offset of the next `</tag>` (or `</ns:tag>`) in `xml`
|
||||
/// starting at `from`. Case-insensitive; returns the offset of the
|
||||
/// `<`. None if not found.
|
||||
fn find_close_tag(xml: &str, tag: &str, from: usize) -> Option<usize> {
|
||||
let bytes = xml.as_bytes();
|
||||
let mut i = from;
|
||||
while i + 2 < bytes.len() {
|
||||
if bytes[i] == b'<' && bytes[i + 1] == b'/' {
|
||||
// Check tag name (with optional namespace).
|
||||
if matches_tag_name_at(bytes, i + 2, tag) {
|
||||
return Some(i);
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Find the open tag of `<tag>` in `xml` starting at `from`. Returns
|
||||
/// `(after_open_lt, after_open_gt)` — the offsets are: just past
|
||||
/// the `<` of the open tag, and just past the `>` of the open tag.
|
||||
fn find_open_tag(xml: &str, tag: &str, from: usize) -> Option<(usize, usize)> {
|
||||
let bytes = xml.as_bytes();
|
||||
let (elem_start, _) = find_element(xml, tag, from)?;
|
||||
let after_lt = elem_start + 1;
|
||||
// Find the `>` that ends the open tag (handling quoted attrs).
|
||||
let mut j = after_lt;
|
||||
while j < bytes.len() {
|
||||
match bytes[j] {
|
||||
b'>' => return Some((after_lt, j + 1)),
|
||||
b'/' if j + 1 < bytes.len() && bytes[j + 1] == b'>' => {
|
||||
return Some((after_lt, j + 2));
|
||||
}
|
||||
b'"' | b'\'' => {
|
||||
let q = bytes[j];
|
||||
j += 1;
|
||||
while j < bytes.len() && bytes[j] != q {
|
||||
j += 1;
|
||||
}
|
||||
if j < bytes.len() {
|
||||
j += 1;
|
||||
}
|
||||
}
|
||||
_ => j += 1,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn find_byte(bytes: &[u8], target: u8, from: usize) -> Option<usize> {
|
||||
bytes[from..]
|
||||
.iter()
|
||||
.position(|&b| b == target)
|
||||
.map(|p| p + from)
|
||||
}
|
||||
|
||||
/// True if `c` can be part of an XML name token (rough). We accept
|
||||
/// alphanumerics, `_`, `-`, `.`.
|
||||
fn is_name_char(c: u8) -> bool {
|
||||
c.is_ascii_alphanumeric() || c == b'_' || c == b'-' || c == b'.'
|
||||
}
|
||||
|
||||
fn is_ws(c: u8) -> bool {
|
||||
matches!(c, b' ' | b'\t' | b'\n' | b'\r')
|
||||
}
|
||||
|
||||
fn slice_eq_ignore_case(a: &[u8], b_lower: &[u8]) -> bool {
|
||||
if a.len() != b_lower.len() {
|
||||
return false;
|
||||
}
|
||||
a.iter()
|
||||
.zip(b_lower.iter())
|
||||
.all(|(&x, &y)| x.to_ascii_lowercase() == y)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn attr_basic() {
|
||||
assert_eq!(
|
||||
attr(r#"<playlist name="Feature" id="00222" />"#, "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
assert_eq!(
|
||||
attr(r#"<playlist name="Feature" id="00222" />"#, "id"),
|
||||
Some("00222".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_case_insensitive_name() {
|
||||
assert_eq!(
|
||||
attr(r#"<playlist Name="Feature" />"#, "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
assert_eq!(
|
||||
attr(r#"<playlist NAME="Feature" />"#, "Name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_accepts_single_quotes() {
|
||||
assert_eq!(
|
||||
attr(r#"<playlist name='Feature' />"#, "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_whitespace_around_equals() {
|
||||
assert_eq!(
|
||||
attr(r#"<playlist name = "Feature" />"#, "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
assert_eq!(
|
||||
attr("<playlist name\n=\n\"Feature\" />", "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_missing_returns_none() {
|
||||
assert_eq!(attr(r#"<playlist name="X" />"#, "id"), None);
|
||||
assert_eq!(attr("", "name"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_no_substring_false_positive() {
|
||||
// Looking for "lang" should NOT match "lang_id" or
|
||||
// "language" because of the name-char boundary check.
|
||||
assert_eq!(attr(r#"<x lang_id="fra" language="eng" />"#, "lang"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_empty_value() {
|
||||
assert_eq!(attr(r#"<x name="" id="1" />"#, "name"), Some("".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_basic() {
|
||||
assert_eq!(text("<x>hello</x>", "x"), Some("hello".into()));
|
||||
assert_eq!(
|
||||
text("<x> hello world </x>", "x"),
|
||||
Some("hello world".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_case_insensitive_tag() {
|
||||
assert_eq!(text("<X>foo</X>", "x"), Some("foo".into()));
|
||||
assert_eq!(text("<Foo>bar</foo>", "foo"), Some("bar".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_namespace_prefix() {
|
||||
assert_eq!(text("<ns:tag>value</ns:tag>", "tag"), Some("value".into()));
|
||||
assert_eq!(text("<foo:Bar>v</foo:Bar>", "bar"), Some("v".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_self_closing() {
|
||||
assert_eq!(text("<x/>", "x"), Some("".into()));
|
||||
assert_eq!(text("<x />", "x"), Some("".into()));
|
||||
assert_eq!(text("<x attr=\"y\" />", "x"), Some("".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_with_attrs() {
|
||||
assert_eq!(
|
||||
text(r#"<x id="1" name="y">hello</x>"#, "x"),
|
||||
Some("hello".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_missing_close_returns_none() {
|
||||
assert_eq!(text("<x>hello", "x"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_skips_inner_tags_naively() {
|
||||
// Limitation noted: nested same-name tags aren't handled.
|
||||
// Different-name nesting works (we just return everything
|
||||
// between the open and close).
|
||||
assert_eq!(
|
||||
text("<x><y>nested</y></x>", "x"),
|
||||
Some("<y>nested</y>".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_basic() {
|
||||
let xml = r#"<x /> <y attr="1">body</y>"#;
|
||||
let (s, e) = find_element(xml, "y", 0).unwrap();
|
||||
assert_eq!(&xml[s..e], r#"<y attr="1">body</y>"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_self_closing() {
|
||||
let xml = r#"<x />"#;
|
||||
let (s, e) = find_element(xml, "x", 0).unwrap();
|
||||
assert_eq!(&xml[s..e], "<x />");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_handles_quoted_gt_in_attr() {
|
||||
// A `>` inside a quoted attribute value should not terminate
|
||||
// the open tag prematurely.
|
||||
let xml = r#"<x attr="foo>bar">body</x>"#;
|
||||
let (s, e) = find_element(xml, "x", 0).unwrap();
|
||||
assert_eq!(&xml[s..e], r#"<x attr="foo>bar">body</x>"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_iteration() {
|
||||
let xml = "<p>a</p><p>b</p><p>c</p>";
|
||||
let mut positions = Vec::new();
|
||||
let mut from = 0;
|
||||
while let Some((s, e)) = find_element(xml, "p", from) {
|
||||
positions.push(&xml[s..e]);
|
||||
from = e;
|
||||
}
|
||||
assert_eq!(positions, vec!["<p>a</p>", "<p>b</p>", "<p>c</p>"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_with_namespace() {
|
||||
let xml = r#"<root><ns:item id="1" /></root>"#;
|
||||
let (s, e) = find_element(xml, "item", 0).unwrap();
|
||||
assert_eq!(&xml[s..e], r#"<ns:item id="1" />"#);
|
||||
}
|
||||
}
|
||||
+130
-25
@@ -6,28 +6,32 @@
|
||||
//! # Quick Start
|
||||
//!
|
||||
//! ```no_run
|
||||
//! use libfreemkv::{DriveSession, Disc, ScanOptions, find_drive};
|
||||
//! use std::path::Path;
|
||||
//! use libfreemkv::{Drive, Disc, ScanOptions, find_drive};
|
||||
//!
|
||||
//! let device = find_drive().expect("no optical drive found");
|
||||
//! let mut session = DriveSession::open(Path::new(&device)).unwrap();
|
||||
//! let disc = Disc::scan(&mut session, &ScanOptions::default()).unwrap();
|
||||
//! let mut drive = find_drive().expect("no optical drive found");
|
||||
//! drive.wait_ready().unwrap();
|
||||
//! drive.init().unwrap();
|
||||
//! let disc = Disc::scan(&mut drive, &ScanOptions::default()).unwrap();
|
||||
//!
|
||||
//! for title in &disc.titles {
|
||||
//! println!("{} -- {} streams", title.duration_display(), title.streams.len());
|
||||
//! }
|
||||
//!
|
||||
//! // Read content (decrypted automatically if AACS keys available)
|
||||
//! let mut reader = disc.open_title(&mut session, 0).unwrap();
|
||||
//! while let Some(unit) = reader.read_unit().unwrap() {
|
||||
//! // 6144 bytes of decrypted content per unit
|
||||
//! // Stream via PES pipeline
|
||||
//! let opts = libfreemkv::InputOptions::default();
|
||||
//! let mut input = libfreemkv::input("disc://", &opts).unwrap();
|
||||
//! let title = input.info().clone();
|
||||
//! let mut output = libfreemkv::output("mkv://Movie.mkv", &title).unwrap();
|
||||
//! while let Ok(Some(frame)) = input.read() {
|
||||
//! output.write(&frame).unwrap();
|
||||
//! }
|
||||
//! output.finish().unwrap();
|
||||
//! ```
|
||||
//!
|
||||
//! # Architecture
|
||||
//!
|
||||
//! ```text
|
||||
//! DriveSession -- open, identify, unlock, read sectors
|
||||
//! Drive -- open, identify, unlock, read sectors
|
||||
//! ├── ScsiTransport -- SG_IO (Linux), IOKit (macOS)
|
||||
//! ├── DriveProfile -- per-drive unlock parameters (206 bundled)
|
||||
//! ├── DriveId -- INQUIRY + GET_CONFIG identification
|
||||
@@ -70,45 +74,146 @@
|
||||
pub mod aacs;
|
||||
pub(crate) mod clpi;
|
||||
pub mod css;
|
||||
pub mod decrypt;
|
||||
pub mod disc;
|
||||
pub mod drive;
|
||||
pub mod error;
|
||||
pub mod event;
|
||||
pub mod halt;
|
||||
pub(crate) mod identity;
|
||||
pub(crate) mod ifo;
|
||||
pub mod io;
|
||||
pub mod keydb;
|
||||
pub(crate) mod labels;
|
||||
pub mod labels;
|
||||
pub(crate) mod mpls;
|
||||
pub mod mux;
|
||||
pub mod pes;
|
||||
pub(crate) mod platform;
|
||||
pub(crate) mod profile;
|
||||
pub mod profile;
|
||||
pub mod progress;
|
||||
pub mod scsi;
|
||||
pub(crate) mod sector;
|
||||
pub mod sector;
|
||||
pub(crate) mod speed;
|
||||
pub(crate) mod udf;
|
||||
pub mod verify;
|
||||
|
||||
pub use drive::{find_drive, find_drives, resolve_device, DriveSession};
|
||||
// Re-export verify types at the crate root for ergonomic imports.
|
||||
pub use verify::{SectorRange, SectorStatus, VerifyResult, verify_title};
|
||||
|
||||
// ─── Drive lifecycle ────────────────────────────────────────────────────────
|
||||
//
|
||||
// `Drive::open(path)` → `wait_ready()` → `init()` → `Disc::scan()`. `Drive`
|
||||
// owns the SCSI session; `DriveCapture` etc. let advanced callers introspect
|
||||
// drive identity / profile data for sharing.
|
||||
pub use drive::capture::{
|
||||
CapturedFeature, DriveCapture, capture_drive_data, mask_bytes, mask_string,
|
||||
};
|
||||
pub use drive::{Drive, DriveStatus, find_drive};
|
||||
|
||||
// ─── Errors ─────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// All fallible APIs return `Result<T, Error>`. `Error` is a typed enum with a
|
||||
// numeric `code()`; **no English text in the library** — applications map
|
||||
// codes to localized messages. See `error.rs` for the full taxonomy.
|
||||
pub use error::{Error, Result};
|
||||
|
||||
// ─── 0.18 primitives ────────────────────────────────────────────────────────
|
||||
//
|
||||
// One-bit cooperative cancellation token, shared by every long-running loop
|
||||
// in libfreemkv (sweep, patch, mux). Replaces the ad-hoc `Arc<AtomicBool>`
|
||||
// flags scattered across 0.17 (`DiscStream::set_halt`, autorip's
|
||||
// `HALT_FLAGS` registry). Clone it cheaply; pass it by value into each
|
||||
// component; poll `is_cancelled()` inside the loop body.
|
||||
pub use halt::Halt;
|
||||
|
||||
// Generic bounded producer/consumer primitive used by sweep, patch, and
|
||||
// mux to overlap reads with writes via a dedicated consumer thread.
|
||||
// `Pipeline::spawn(name, depth, sink)` spawns a named consumer; `pipe.send(item)`
|
||||
// pushes one item with back-pressure; `pipe.finish()` joins the
|
||||
// consumer and surfaces its `close()` output. Callers implement `Sink`
|
||||
// to define per-item behaviour and end-of-stream finalisation.
|
||||
//
|
||||
// `DEFAULT_PIPELINE_DEPTH` (=4) is for callers without specific needs;
|
||||
// most should use READ_PIPELINE_DEPTH or WRITE_PIPELINE_DEPTH instead.
|
||||
// Patch uses `WRITE_THROUGH_DEPTH` (=1). Returning `Flow::Stop` from
|
||||
// `apply` ends the consumer cleanly (still calls `close()`).
|
||||
pub use io::pipeline::{
|
||||
DEFAULT_PIPELINE_DEPTH, Flow, Pipeline, READ_PIPELINE_DEPTH, Sink, WRITE_PIPELINE_DEPTH,
|
||||
WRITE_THROUGH_DEPTH,
|
||||
};
|
||||
|
||||
// ─── Drive events (low-level callbacks) ─────────────────────────────────────
|
||||
pub use event::{Event, EventKind};
|
||||
pub use identity::DriveId;
|
||||
pub use profile::DriveProfile;
|
||||
// Platform trait is pub(crate) -- callers use DriveSession, not Platform directly
|
||||
// Platform trait is pub(crate) — callers use Drive, not Platform directly.
|
||||
|
||||
// ─── Decryption (AACS / CSS) ────────────────────────────────────────────────
|
||||
//
|
||||
// `Disc::scan()` resolves keys and stores them on `Disc`; in most flows you
|
||||
// don't touch `DecryptKeys` directly — `DiscStream::new(reader, title, keys, …)`
|
||||
// accepts whatever `Disc::decrypt_keys()` returned. `decrypt_sectors()` is
|
||||
// for callers that operate on raw sector buffers (e.g. ISO patching).
|
||||
pub use decrypt::{DecryptKeys, decrypt_sectors};
|
||||
|
||||
// ─── Disc structure ─────────────────────────────────────────────────────────
|
||||
//
|
||||
// `Disc::scan()` produces a fully-populated `Disc` (titles, streams, AACS
|
||||
// state). `Disc::identify()` is the fast path — UDF only, no playlist parse,
|
||||
// for displaying disc name + format quickly while a full scan runs in the
|
||||
// background. The codec / channel / resolution enums are the canonical
|
||||
// structured representation; never compare against display strings.
|
||||
// Note: `disc::Stream` here is the codec enum (audio / video / sub kind)
|
||||
// — not the `pes::Stream` trait re-exported below as `PesStream`. Two
|
||||
// different concepts, the same short name; the trait gets the `Pes`
|
||||
// prefix at the crate root to keep both addressable.
|
||||
pub use disc::{
|
||||
AacsState, AudioStream, Clip, Codec, ColorSpace, ContentFormat, ContentReader, Disc,
|
||||
DiscFormat, DiscTitle, Extent, HdrFormat, KeySource, ScanOptions, Stream, SubtitleStream,
|
||||
VideoStream,
|
||||
AacsState, AudioChannels, AudioStream, Clip, Codec, ColorSpace, ContentFormat, DamageSeverity,
|
||||
Disc, DiscFormat, DiscId, DiscTitle, Extent, FrameRate, HdrFormat, KeySource, LabelPurpose,
|
||||
LabelQualifier, PatchOptions, PatchOutcome, Resolution, SampleRate, ScanOptions, Stream,
|
||||
SubtitleStream, SweepOptions, VideoStream, classify_damage,
|
||||
};
|
||||
pub use mux::DiscOptions;
|
||||
|
||||
// ─── Streams ────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// All stream types implement `pes::Stream` — read PES frames from a source,
|
||||
// write PES frames to a sink. Pick the right type at construction:
|
||||
//
|
||||
// - `DiscStream` — physical drive or ISO (any `SectorSource`). Read-only.
|
||||
// - `MkvStream` — Matroska container. Read on `open()`, write on `create()`.
|
||||
// - `M2tsStream` — Blu-ray Transport Stream. Read on `open()`, write on `create()`.
|
||||
// - `NetworkStream` — TCP. Read on `listen()`, write on `connect()`.
|
||||
// - `NullStream` — write-only black-hole sink. Useful for benchmarks.
|
||||
// - `StdioStream` — pipe to/from stdin/stdout. Read or write.
|
||||
//
|
||||
// Most consumers use the URL resolvers (`input()` / `output()`) which pick
|
||||
// the right type from a scheme:// URL. Direct construction is for callers
|
||||
// that need to wire custom readers (e.g. autorip's drive-session reuse).
|
||||
// The trait is re-exported as `PesStream` here to disambiguate from
|
||||
// `disc::Stream` (the codec-kind enum re-exported above), which would
|
||||
// otherwise collide at the crate root.
|
||||
pub use pes::PesFrame;
|
||||
pub use pes::Stream as PesStream;
|
||||
|
||||
pub use mux::DiscStream;
|
||||
pub use mux::IOStream;
|
||||
pub use mux::IsoStream;
|
||||
pub use mux::M2tsStream;
|
||||
pub use mux::MkvStream;
|
||||
pub use mux::NetworkStream;
|
||||
pub use mux::NullStream;
|
||||
pub use mux::StdioStream;
|
||||
pub use mux::{open_input, open_output, parse_url, InputOptions};
|
||||
pub use scsi::ScsiTransport;
|
||||
pub use sector::SectorReader;
|
||||
pub use mux::{InputOptions, StreamUrl, input, output, parse_url};
|
||||
|
||||
// ─── Lower-level surfaces ───────────────────────────────────────────────────
|
||||
//
|
||||
// `ScsiTransport` is the platform-abstraction trait Drive uses; expose for
|
||||
// out-of-tree platform backends. `SectorSource` / `SectorSink` are the
|
||||
// direction-typed read/write traits; `FileSectorSource` and `FileSectorSink`
|
||||
// are the ISO-on-disk implementations. [`DecryptingSectorSource`] is the
|
||||
// single decrypt-on-read decorator (AACS / CSS / none) — wrap any
|
||||
// `SectorSource` to get plaintext sectors out.
|
||||
pub use scsi::{DriveInfo, ScsiSense, ScsiTransport, drive_has_disc, list_drives};
|
||||
pub use sector::{
|
||||
DecryptingSectorSource, FileSectorSink, FileSectorSource, SectorSink, SectorSource,
|
||||
};
|
||||
pub use speed::DriveSpeed;
|
||||
pub use udf::{read_filesystem, UdfFs};
|
||||
pub use udf::{UdfFs, read_filesystem};
|
||||
|
||||
+1
-1
@@ -256,7 +256,7 @@ pub fn parse(data: &[u8]) -> Result<Playlist> {
|
||||
|
||||
// Parse PlayListMark section
|
||||
let mut marks = Vec::new();
|
||||
if mark_start > 0 && mark_start + 4 < data.len() {
|
||||
if mark_start > 0 && mark_start + 4 <= data.len() {
|
||||
let ms = &data[mark_start..];
|
||||
if ms.len() >= 6 {
|
||||
let num_marks = u16::from_be_bytes([ms[4], ms[5]]) as usize;
|
||||
|
||||
+193
-267
@@ -1,15 +1,15 @@
|
||||
//! AC3 (Dolby Digital) / EAC3 (Dolby Digital Plus) frame parser.
|
||||
//!
|
||||
//! AC3 frames are self-contained and always start with syncword 0x0B77.
|
||||
//! Each PES packet typically contains exactly one AC3 frame.
|
||||
//! All AC3 frames are effectively keyframes (no inter-frame dependencies).
|
||||
//!
|
||||
//! E-AC-3 shares the same syncword but uses bsid >= 11 (typically 16).
|
||||
//! Frame size is derived from the frmsiz field instead of fscod/frmsizecod.
|
||||
//! Buffers across PES boundaries so frames that span two PES packets
|
||||
//! are emitted complete, not truncated.
|
||||
|
||||
use super::{pts_to_ns, CodecParser, Frame, PesPacket};
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
pub struct Ac3Parser;
|
||||
pub struct Ac3Parser {
|
||||
/// Leftover bytes from previous PES (incomplete frame at end).
|
||||
buf: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Default for Ac3Parser {
|
||||
fn default() -> Self {
|
||||
@@ -19,19 +19,24 @@ impl Default for Ac3Parser {
|
||||
|
||||
impl Ac3Parser {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
Self {
|
||||
buf: Vec::with_capacity(4096),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CodecParser for Ac3Parser {
|
||||
fn parse(&mut self, pes: &PesPacket) -> Vec<Frame> {
|
||||
if pes.data.len() < 2 {
|
||||
if pes.data.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let pts_ns = pes.pts.map(pts_to_ns).unwrap_or(0);
|
||||
|
||||
let data = &pes.data;
|
||||
// Prepend leftover from previous PES
|
||||
self.buf.extend_from_slice(&pes.data);
|
||||
|
||||
let data = &self.buf;
|
||||
let mut frames = Vec::new();
|
||||
let mut pos = 0;
|
||||
|
||||
@@ -44,44 +49,54 @@ impl CodecParser for Ac3Parser {
|
||||
|
||||
let remaining = &data[start..];
|
||||
|
||||
// Need at least 6 bytes to inspect bsid / frame size fields
|
||||
if remaining.len() < 6 {
|
||||
// Emit whatever remains as a single frame
|
||||
frames.push(Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: remaining.to_vec(),
|
||||
});
|
||||
// Not enough data to determine frame size — keep for next PES
|
||||
break;
|
||||
}
|
||||
|
||||
let bsid = get_bsid(remaining);
|
||||
|
||||
if bsid >= 11 {
|
||||
// E-AC-3 frame size from frmsiz field (bytes 2-3)
|
||||
let frame_size = eac3_frame_size(remaining);
|
||||
|
||||
let end = start + frame_size.min(data.len() - start);
|
||||
frames.push(Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: data[start..end].to_vec(),
|
||||
});
|
||||
pos = end;
|
||||
let frame_size = if bsid >= 11 {
|
||||
eac3_frame_size(remaining)
|
||||
} else {
|
||||
// AC-3: emit everything from syncword to next syncword (or end)
|
||||
let next_sync = find_ac3_sync(&data[start + 2..]).map(|o| start + 2 + o);
|
||||
let end = next_sync.unwrap_or(data.len());
|
||||
frames.push(Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: data[start..end].to_vec(),
|
||||
});
|
||||
pos = end;
|
||||
ac3_frame_size(remaining)
|
||||
};
|
||||
|
||||
if frame_size == 0 || frame_size > 8192 {
|
||||
// Invalid frame size — skip this sync word
|
||||
pos = start + 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
if start + frame_size > data.len() {
|
||||
// Incomplete frame — keep for next PES
|
||||
break;
|
||||
}
|
||||
|
||||
frames.push(Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: data[start..start + frame_size].to_vec(),
|
||||
});
|
||||
pos = start + frame_size;
|
||||
}
|
||||
|
||||
// Keep unconsumed data for next call
|
||||
// `pos` points to the start of unconsumed data (either a partial sync or leftover)
|
||||
let keep_from = if pos < data.len() {
|
||||
// Find the last sync word position in the unconsumed region
|
||||
find_ac3_sync(&data[pos..])
|
||||
.map(|o| pos + o)
|
||||
.unwrap_or(data.len())
|
||||
} else {
|
||||
data.len()
|
||||
};
|
||||
|
||||
if keep_from < data.len() {
|
||||
self.buf = data[keep_from..].to_vec();
|
||||
} else {
|
||||
self.buf.clear();
|
||||
}
|
||||
|
||||
// If we found no syncword at all, return empty — the data is not valid AC3.
|
||||
frames
|
||||
}
|
||||
|
||||
@@ -97,263 +112,174 @@ fn find_ac3_sync(data: &[u8]) -> Option<usize> {
|
||||
|
||||
/// Extract bsid from an AC-3/E-AC-3 frame starting at the syncword.
|
||||
/// bsid is at byte 5, bits 7..3.
|
||||
/// AC-3: bsid <= 10, E-AC-3: bsid >= 11 (typically 16).
|
||||
pub fn get_bsid(data: &[u8]) -> u8 {
|
||||
debug_assert!(data.len() >= 6);
|
||||
if data.len() < 6 {
|
||||
return 0;
|
||||
}
|
||||
(data[5] >> 3) & 0x1F
|
||||
}
|
||||
|
||||
/// Calculate E-AC-3 frame size in bytes from the frmsiz field.
|
||||
/// frmsiz is at bits [2:0] of byte 2 concatenated with byte 3.
|
||||
/// Frame size = (frmsiz + 1) * 2 bytes.
|
||||
pub fn eac3_frame_size(data: &[u8]) -> usize {
|
||||
debug_assert!(data.len() >= 4);
|
||||
let frmsiz = ((data[2] as usize & 0x07) << 8) | (data[3] as usize);
|
||||
fn eac3_frame_size(data: &[u8]) -> usize {
|
||||
if data.len() < 4 {
|
||||
return 0;
|
||||
}
|
||||
let frmsiz = ((data[2] as usize & 0x07) << 8) | data[3] as usize;
|
||||
(frmsiz + 1) * 2
|
||||
}
|
||||
|
||||
/// Calculate AC-3 frame size in bytes from fscod and frmsizecod.
|
||||
fn ac3_frame_size(data: &[u8]) -> usize {
|
||||
if data.len() < 5 {
|
||||
return 0;
|
||||
}
|
||||
let fscod = (data[4] >> 6) & 0x03;
|
||||
let frmsizecod = (data[4] & 0x3F) as usize;
|
||||
if frmsizecod >= AC3_FRAME_SIZES.len() {
|
||||
return 0;
|
||||
}
|
||||
let words = AC3_FRAME_SIZES[frmsizecod];
|
||||
match fscod {
|
||||
0 => words[0] * 2,
|
||||
1 => words[1] * 2,
|
||||
2 => words[2] * 2,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// AC-3 frame size table: [frmsizecod] -> [48kHz words, 44.1kHz words, 32kHz words]
|
||||
const AC3_FRAME_SIZES: [[usize; 3]; 38] = [
|
||||
[64, 69, 96],
|
||||
[64, 70, 96],
|
||||
[80, 87, 120],
|
||||
[80, 88, 120],
|
||||
[96, 104, 144],
|
||||
[96, 105, 144],
|
||||
[112, 121, 168],
|
||||
[112, 122, 168],
|
||||
[128, 139, 192],
|
||||
[128, 140, 192],
|
||||
[160, 174, 240],
|
||||
[160, 175, 240],
|
||||
[192, 208, 288],
|
||||
[192, 209, 288],
|
||||
[224, 243, 336],
|
||||
[224, 244, 336],
|
||||
[256, 278, 384],
|
||||
[256, 279, 384],
|
||||
[320, 348, 480],
|
||||
[320, 349, 480],
|
||||
[384, 417, 576],
|
||||
[384, 418, 576],
|
||||
[448, 487, 672],
|
||||
[448, 488, 672],
|
||||
[512, 557, 768],
|
||||
[512, 558, 768],
|
||||
[640, 696, 960],
|
||||
[640, 697, 960],
|
||||
[768, 835, 1152],
|
||||
[768, 836, 1152],
|
||||
[896, 975, 1344],
|
||||
[896, 976, 1344],
|
||||
[1024, 1114, 1536],
|
||||
[1024, 1115, 1536],
|
||||
[1152, 1253, 1728],
|
||||
[1152, 1254, 1728],
|
||||
[1280, 1393, 1920],
|
||||
[1280, 1394, 1920],
|
||||
];
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::mux::ts::PesPacket;
|
||||
|
||||
fn make_pes(data: Vec<u8>, pts: Option<i64>) -> PesPacket {
|
||||
PesPacket {
|
||||
pid: 0x1100,
|
||||
pts,
|
||||
dts: None,
|
||||
data,
|
||||
}
|
||||
fn make_ac3_frame(fscod: u8, frmsizecod: u8) -> Vec<u8> {
|
||||
let size = AC3_FRAME_SIZES[frmsizecod as usize][fscod as usize] * 2;
|
||||
let mut frame = vec![0u8; size];
|
||||
frame[0] = 0x0B;
|
||||
frame[1] = 0x77;
|
||||
frame[4] = (fscod << 6) | frmsizecod;
|
||||
frame[5] = 0x08 << 3; // bsid = 8 (AC-3)
|
||||
frame
|
||||
}
|
||||
|
||||
/// Build a minimal AC-3 header (bsid <= 10).
|
||||
fn make_ac3_header(bsid: u8) -> Vec<u8> {
|
||||
// 0x0B 0x77 <byte2> <byte3> <byte4> <byte5=bsid>
|
||||
let byte5 = (bsid & 0x1F) << 3;
|
||||
vec![0x0B, 0x77, 0x00, 0x00, 0x00, byte5, 0xAA, 0xBB]
|
||||
}
|
||||
|
||||
/// Build a minimal E-AC-3 header with the given bsid and frmsiz.
|
||||
/// frmsiz encodes frame size: frame_bytes = (frmsiz + 1) * 2.
|
||||
fn make_eac3_header(bsid: u8, frmsiz: u16, payload_fill: u8) -> Vec<u8> {
|
||||
let byte2 = (frmsiz >> 8) as u8 & 0x07;
|
||||
let byte3 = (frmsiz & 0xFF) as u8;
|
||||
let byte5 = (bsid & 0x1F) << 3;
|
||||
let frame_size = (frmsiz as usize + 1) * 2;
|
||||
let mut data = vec![0x0B, 0x77, byte2, byte3, 0x00, byte5];
|
||||
// Pad to full frame size
|
||||
while data.len() < frame_size {
|
||||
data.push(payload_fill);
|
||||
}
|
||||
data.truncate(frame_size);
|
||||
data
|
||||
}
|
||||
|
||||
// --- syncword detection ---
|
||||
|
||||
#[test]
|
||||
fn find_ac3_sync_at_start() {
|
||||
let data = [0x0B, 0x77, 0x01, 0x02, 0x03];
|
||||
assert_eq!(find_ac3_sync(&data), Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_ac3_sync_with_garbage_prefix() {
|
||||
let data = [0xFF, 0xFE, 0x0B, 0x77, 0x01, 0x02];
|
||||
assert_eq!(find_ac3_sync(&data), Some(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_ac3_sync_none() {
|
||||
let data = [0x0B, 0x78, 0x00, 0x00];
|
||||
assert_eq!(find_ac3_sync(&data), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_ac3_sync_empty() {
|
||||
let data: [u8; 0] = [];
|
||||
assert_eq!(find_ac3_sync(&data), None);
|
||||
}
|
||||
|
||||
// --- bsid detection ---
|
||||
|
||||
#[test]
|
||||
fn bsid_ac3() {
|
||||
let header = make_ac3_header(8);
|
||||
assert_eq!(get_bsid(&header), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bsid_eac3() {
|
||||
let header = make_eac3_header(16, 99, 0x00);
|
||||
assert_eq!(get_bsid(&header), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bsid_boundary_10() {
|
||||
let header = make_ac3_header(10);
|
||||
assert_eq!(get_bsid(&header), 10);
|
||||
// bsid 10 should be treated as AC-3 (<= 10)
|
||||
assert!(get_bsid(&header) <= 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bsid_boundary_11() {
|
||||
let header = make_eac3_header(11, 3, 0x00);
|
||||
assert_eq!(get_bsid(&header), 11);
|
||||
// bsid 11 should be treated as E-AC-3 (>= 11)
|
||||
assert!(get_bsid(&header) >= 11);
|
||||
}
|
||||
|
||||
// --- E-AC-3 frame size calculation ---
|
||||
|
||||
#[test]
|
||||
fn eac3_frame_size_basic() {
|
||||
// frmsiz = 99 → frame_size = (99+1)*2 = 200 bytes
|
||||
let header = make_eac3_header(16, 99, 0xDD);
|
||||
assert_eq!(eac3_frame_size(&header), 200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eac3_frame_size_min() {
|
||||
// frmsiz = 0 → frame_size = (0+1)*2 = 2 bytes
|
||||
let data = [0x0B, 0x77, 0x00, 0x00, 0x00, 0x80];
|
||||
assert_eq!(eac3_frame_size(&data), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eac3_frame_size_large() {
|
||||
// frmsiz = 0x7FF (max 11-bit) → (2047+1)*2 = 4096
|
||||
let data = [0x0B, 0x77, 0x07, 0xFF, 0x00, 0x80];
|
||||
assert_eq!(eac3_frame_size(&data), 4096);
|
||||
}
|
||||
|
||||
// --- parse: E-AC-3 frame extraction ---
|
||||
|
||||
#[test]
|
||||
fn parse_eac3_single_frame() {
|
||||
let mut parser = Ac3Parser::new();
|
||||
// frmsiz = 9 → frame_size = 20 bytes
|
||||
let data = make_eac3_header(16, 9, 0xCC);
|
||||
assert_eq!(data.len(), 20);
|
||||
let pes = make_pes(data.clone(), Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data.len(), 20);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
assert!(frames[0].keyframe);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_eac3_frame_with_garbage_prefix() {
|
||||
let mut parser = Ac3Parser::new();
|
||||
let mut data = vec![0xFF, 0xFE]; // garbage
|
||||
data.extend_from_slice(&make_eac3_header(16, 4, 0xAA)); // frmsiz=4 → 10 bytes
|
||||
let pes = make_pes(data, Some(0));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data[0], 0x0B);
|
||||
assert_eq!(frames[0].data[1], 0x77);
|
||||
assert_eq!(frames[0].data.len(), 10);
|
||||
}
|
||||
|
||||
// --- parse syncword → frame extracted (AC-3) ---
|
||||
|
||||
#[test]
|
||||
fn parse_syncword() {
|
||||
let mut parser = Ac3Parser::new();
|
||||
|
||||
// AC3 frame starting with syncword (bsid=8)
|
||||
let data = make_ac3_header(8);
|
||||
let pes = make_pes(data.clone(), Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data, data);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_syncword_with_garbage_prefix() {
|
||||
let mut parser = Ac3Parser::new();
|
||||
|
||||
// Garbage bytes before syncword
|
||||
let mut data = vec![0xFF, 0xFE];
|
||||
data.extend_from_slice(&make_ac3_header(8));
|
||||
let pes = make_pes(data, Some(0));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data[0], 0x0B);
|
||||
assert_eq!(frames[0].data[1], 0x77);
|
||||
}
|
||||
|
||||
// --- all frames are keyframes ---
|
||||
|
||||
#[test]
|
||||
fn all_keyframes() {
|
||||
let mut parser = Ac3Parser::new();
|
||||
|
||||
for i in 0..5u8 {
|
||||
let mut data = make_ac3_header(8);
|
||||
data.push(i);
|
||||
let pes = make_pes(data, Some(90000 * i as i64));
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert!(frames[0].keyframe, "AC3 frame {} should be a keyframe", i);
|
||||
}
|
||||
}
|
||||
|
||||
// --- codec_private is None ---
|
||||
|
||||
#[test]
|
||||
fn codec_private_none() {
|
||||
let parser = Ac3Parser::new();
|
||||
assert!(parser.codec_private().is_none());
|
||||
}
|
||||
|
||||
// --- empty / too-short PES ---
|
||||
|
||||
#[test]
|
||||
fn parse_empty_pes() {
|
||||
let mut parser = Ac3Parser::new();
|
||||
let pes = make_pes(Vec::new(), Some(0));
|
||||
let frames = parser.parse(&pes);
|
||||
assert!(frames.is_empty());
|
||||
let pes = PesPacket {
|
||||
pid: 0,
|
||||
pts: None,
|
||||
dts: None,
|
||||
data: vec![],
|
||||
};
|
||||
assert!(parser.parse(&pes).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_single_byte_pes() {
|
||||
fn parse_single_frame() {
|
||||
let mut parser = Ac3Parser::new();
|
||||
let pes = make_pes(vec![0x0B], Some(0));
|
||||
let frames = parser.parse(&pes);
|
||||
assert!(frames.is_empty());
|
||||
}
|
||||
|
||||
// --- PTS conversion ---
|
||||
|
||||
#[test]
|
||||
fn pts_conversion() {
|
||||
let mut parser = Ac3Parser::new();
|
||||
let data = make_ac3_header(8);
|
||||
// 45000 ticks = 0.5 seconds → 500_000_000 ns
|
||||
let pes = make_pes(data, Some(45000));
|
||||
let frame_data = make_ac3_frame(0, 2); // 48kHz, 80 words = 160 bytes
|
||||
let pes = PesPacket {
|
||||
pid: 0,
|
||||
pts: Some(90000),
|
||||
dts: None,
|
||||
data: frame_data.clone(),
|
||||
};
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].pts_ns, 500_000_000);
|
||||
assert_eq!(frames[0].data.len(), 160);
|
||||
}
|
||||
|
||||
// --- None PTS ---
|
||||
#[test]
|
||||
fn parse_frame_spanning_two_pes() {
|
||||
let mut parser = Ac3Parser::new();
|
||||
let frame_data = make_ac3_frame(0, 2); // 160 bytes
|
||||
let mid = 80;
|
||||
|
||||
// First PES: first half of frame
|
||||
let pes1 = PesPacket {
|
||||
pid: 0,
|
||||
pts: Some(90000),
|
||||
dts: None,
|
||||
data: frame_data[..mid].to_vec(),
|
||||
};
|
||||
let frames1 = parser.parse(&pes1);
|
||||
assert!(frames1.is_empty(), "partial frame should not emit");
|
||||
|
||||
// Second PES: second half
|
||||
let pes2 = PesPacket {
|
||||
pid: 0,
|
||||
pts: Some(93000),
|
||||
dts: None,
|
||||
data: frame_data[mid..].to_vec(),
|
||||
};
|
||||
let frames2 = parser.parse(&pes2);
|
||||
assert_eq!(frames2.len(), 1);
|
||||
assert_eq!(frames2[0].data.len(), 160);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_pts() {
|
||||
fn skip_garbage_before_sync() {
|
||||
let mut parser = Ac3Parser::new();
|
||||
let data = make_ac3_header(8);
|
||||
let pes = make_pes(data, None);
|
||||
let frame_data = make_ac3_frame(0, 2);
|
||||
let mut data = vec![0xDE, 0xAD, 0xBE, 0xEF]; // garbage
|
||||
data.extend_from_slice(&frame_data);
|
||||
let pes = PesPacket {
|
||||
pid: 0,
|
||||
pts: None,
|
||||
dts: None,
|
||||
data,
|
||||
};
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].pts_ns, 0);
|
||||
assert_eq!(frames[0].data.len(), 160);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ac3_frame_size_table() {
|
||||
// fscod=0 (48kHz), frmsizecod=0: 64 words = 128 bytes
|
||||
assert_eq!(ac3_frame_size(&[0x0B, 0x77, 0, 0, 0x00, 0x40]), 128);
|
||||
// fscod=0 (48kHz), frmsizecod=2: 80 words = 160 bytes
|
||||
assert_eq!(ac3_frame_size(&[0x0B, 0x77, 0, 0, 0x02, 0x40]), 160);
|
||||
}
|
||||
}
|
||||
|
||||
+131
-198
@@ -2,16 +2,17 @@
|
||||
//!
|
||||
//! DTS core syncword: 0x7FFE8001 (32 bits).
|
||||
//! DTS-HD MA/HRA extension syncword: 0x64582025 (32 bits), appears after the core frame.
|
||||
//! The extension contains high-resolution audio data and is appended to the core frame.
|
||||
//! All frames are keyframes (no inter-frame dependencies).
|
||||
//! Each PES packet = one frame.
|
||||
//! Buffers across PES boundaries so frames spanning two PES packets
|
||||
//! are emitted complete.
|
||||
|
||||
use super::{pts_to_ns, CodecParser, Frame, PesPacket};
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
/// DTS-HD extension syncword bytes.
|
||||
const DTS_CORE_SYNC: [u8; 4] = [0x7F, 0xFE, 0x80, 0x01];
|
||||
const DTS_HD_EXT_SYNC: [u8; 4] = [0x64, 0x58, 0x20, 0x25];
|
||||
|
||||
pub struct DtsParser;
|
||||
pub struct DtsParser {
|
||||
buf: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Default for DtsParser {
|
||||
fn default() -> Self {
|
||||
@@ -21,7 +22,9 @@ impl Default for DtsParser {
|
||||
|
||||
impl DtsParser {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
Self {
|
||||
buf: Vec::with_capacity(32768),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,31 +35,77 @@ impl CodecParser for DtsParser {
|
||||
}
|
||||
let pts_ns = pes.pts.map(pts_to_ns).unwrap_or(0);
|
||||
|
||||
let data = &pes.data;
|
||||
self.buf.extend_from_slice(&pes.data);
|
||||
|
||||
// Look for a DTS-HD extension substream after the core.
|
||||
// If found, include both core + extension in the output frame.
|
||||
let frame_data = match find_dts_hd_ext_sync(data) {
|
||||
Some(ext_offset) => {
|
||||
let ext = &data[ext_offset..];
|
||||
if ext.len() >= 9 {
|
||||
let ext_size = dts_hd_ext_frame_size(ext);
|
||||
let total_end = ext_offset + ext_size;
|
||||
let end = total_end.min(data.len());
|
||||
data[..end].to_vec()
|
||||
} else {
|
||||
// Extension header too short to parse size; include all data.
|
||||
data.to_vec()
|
||||
let data = &self.buf;
|
||||
let mut frames = Vec::new();
|
||||
let mut pos = 0;
|
||||
|
||||
while pos < data.len() {
|
||||
// Find DTS core sync
|
||||
let start = match find_sync(&data[pos..], &DTS_CORE_SYNC) {
|
||||
Some(offset) => pos + offset,
|
||||
None => break,
|
||||
};
|
||||
|
||||
// Need at least 10 bytes for core header to get frame size
|
||||
if start + 10 > data.len() {
|
||||
break;
|
||||
}
|
||||
|
||||
let core_size = dts_core_frame_size(&data[start..]);
|
||||
if core_size == 0 || core_size > 32768 {
|
||||
pos = start + 4;
|
||||
continue;
|
||||
}
|
||||
|
||||
if start + core_size > data.len() {
|
||||
// Incomplete core frame
|
||||
break;
|
||||
}
|
||||
|
||||
// Check for DTS-HD extension after core
|
||||
let mut total_size = core_size;
|
||||
if start + core_size + 4 <= data.len() {
|
||||
if let Some(0) = find_sync(
|
||||
&data[start + core_size..start + core_size + 4],
|
||||
&DTS_HD_EXT_SYNC,
|
||||
) {
|
||||
let ext = &data[start + core_size..];
|
||||
if ext.len() >= 9 {
|
||||
let ext_size = dts_hd_ext_frame_size(ext);
|
||||
if start + core_size + ext_size <= data.len() {
|
||||
total_size = core_size + ext_size;
|
||||
}
|
||||
// If ext incomplete, just emit core
|
||||
}
|
||||
}
|
||||
}
|
||||
None => data.to_vec(),
|
||||
|
||||
frames.push(Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: data[start..start + total_size].to_vec(),
|
||||
});
|
||||
pos = start + total_size;
|
||||
}
|
||||
|
||||
// Keep unconsumed data
|
||||
let keep_from = if pos < data.len() {
|
||||
find_sync(&data[pos..], &DTS_CORE_SYNC)
|
||||
.map(|o| pos + o)
|
||||
.unwrap_or(data.len())
|
||||
} else {
|
||||
data.len()
|
||||
};
|
||||
|
||||
vec![Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: frame_data,
|
||||
}]
|
||||
if keep_from < data.len() {
|
||||
self.buf = data[keep_from..].to_vec();
|
||||
} else {
|
||||
self.buf.clear();
|
||||
}
|
||||
|
||||
frames
|
||||
}
|
||||
|
||||
fn codec_private(&self) -> Option<Vec<u8>> {
|
||||
@@ -64,30 +113,40 @@ impl CodecParser for DtsParser {
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the DTS-HD extension syncword (0x64582025) in data.
|
||||
/// Returns the byte offset of the sync, or None.
|
||||
pub fn find_dts_hd_ext_sync(data: &[u8]) -> Option<usize> {
|
||||
fn find_sync(data: &[u8], pattern: &[u8; 4]) -> Option<usize> {
|
||||
if data.len() < 4 {
|
||||
return None;
|
||||
}
|
||||
(0..=data.len() - 4).find(|&i| {
|
||||
data[i] == DTS_HD_EXT_SYNC[0]
|
||||
&& data[i + 1] == DTS_HD_EXT_SYNC[1]
|
||||
&& data[i + 2] == DTS_HD_EXT_SYNC[2]
|
||||
&& data[i + 3] == DTS_HD_EXT_SYNC[3]
|
||||
})
|
||||
(0..=data.len() - 4).find(|&i| data[i..i + 4] == *pattern)
|
||||
}
|
||||
|
||||
/// Calculate DTS-HD extension frame size from the extension header.
|
||||
/// The size field is at bytes 6-8 of the extension:
|
||||
/// ((ext[6] & 0x1F) << 11) | (ext[7] << 3) | (ext[8] >> 5) + 1
|
||||
/// DTS core frame size from header bits.
|
||||
/// fsize is at bits 46-59 (14 bits) of the header: bytes 5-7.
|
||||
fn dts_core_frame_size(data: &[u8]) -> usize {
|
||||
if data.len() < 10 {
|
||||
return 0;
|
||||
}
|
||||
// fsize field: 14 bits starting at bit 46
|
||||
// byte 5 bits 1-0, byte 6 all 8, byte 7 bits 7-4
|
||||
let fsize =
|
||||
((data[5] as usize & 0x03) << 12) | ((data[6] as usize) << 4) | ((data[7] as usize) >> 4);
|
||||
fsize + 1
|
||||
}
|
||||
|
||||
/// DTS-HD extension frame size from extension header.
|
||||
pub fn dts_hd_ext_frame_size(ext: &[u8]) -> usize {
|
||||
debug_assert!(ext.len() >= 9);
|
||||
if ext.len() < 9 {
|
||||
return 0;
|
||||
}
|
||||
let raw =
|
||||
((ext[6] as usize & 0x1F) << 11) | ((ext[7] as usize) << 3) | ((ext[8] as usize) >> 5);
|
||||
raw + 1
|
||||
}
|
||||
|
||||
pub fn find_dts_hd_ext_sync(data: &[u8]) -> Option<usize> {
|
||||
find_sync(data, &DTS_HD_EXT_SYNC)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -102,164 +161,16 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a DTS core frame with given payload size.
|
||||
fn make_dts_core(payload_len: usize) -> Vec<u8> {
|
||||
let mut data = vec![0x7F, 0xFE, 0x80, 0x01];
|
||||
data.resize(4 + payload_len, 0xAA);
|
||||
fn make_dts_core(size: usize) -> Vec<u8> {
|
||||
let fsize = size - 1;
|
||||
let mut data = vec![0u8; size];
|
||||
data[0..4].copy_from_slice(&DTS_CORE_SYNC);
|
||||
data[5] = (data[5] & 0xFC) | ((fsize >> 12) & 0x03) as u8;
|
||||
data[6] = ((fsize >> 4) & 0xFF) as u8;
|
||||
data[7] = (data[7] & 0x0F) | (((fsize & 0x0F) << 4) as u8);
|
||||
data
|
||||
}
|
||||
|
||||
/// Build a DTS-HD extension header + payload.
|
||||
/// ext_size is the value to encode (frame size = ext_size + 1 reported by dts_hd_ext_frame_size,
|
||||
/// but we encode raw = ext_size so that dts_hd_ext_frame_size returns ext_size + 1).
|
||||
fn make_dts_hd_ext(raw_size_field: usize, payload_fill: u8) -> Vec<u8> {
|
||||
let total = raw_size_field + 1; // the size dts_hd_ext_frame_size will return
|
||||
let byte6 = ((raw_size_field >> 11) & 0x1F) as u8;
|
||||
let byte7 = ((raw_size_field >> 3) & 0xFF) as u8;
|
||||
let byte8 = ((raw_size_field & 0x07) << 5) as u8;
|
||||
let mut data = vec![0x64, 0x58, 0x20, 0x25, 0x00, 0x00, byte6, byte7, byte8];
|
||||
while data.len() < total {
|
||||
data.push(payload_fill);
|
||||
}
|
||||
data.truncate(total);
|
||||
data
|
||||
}
|
||||
|
||||
// --- DTS-HD extension sync detection ---
|
||||
|
||||
#[test]
|
||||
fn find_ext_sync_at_offset() {
|
||||
let mut data = vec![0x7F, 0xFE, 0x80, 0x01, 0x00, 0x00];
|
||||
data.extend_from_slice(&[0x64, 0x58, 0x20, 0x25]);
|
||||
assert_eq!(find_dts_hd_ext_sync(&data), Some(6));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_ext_sync_none() {
|
||||
let data = vec![0x7F, 0xFE, 0x80, 0x01, 0x00, 0x00];
|
||||
assert_eq!(find_dts_hd_ext_sync(&data), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_ext_sync_at_start() {
|
||||
let data = vec![0x64, 0x58, 0x20, 0x25, 0x00];
|
||||
assert_eq!(find_dts_hd_ext_sync(&data), Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_ext_sync_too_short() {
|
||||
let data = vec![0x64, 0x58, 0x20];
|
||||
assert_eq!(find_dts_hd_ext_sync(&data), None);
|
||||
}
|
||||
|
||||
// --- DTS-HD extension frame size ---
|
||||
|
||||
#[test]
|
||||
fn ext_frame_size_basic() {
|
||||
// raw_size_field = 100 → frame size = 101
|
||||
let ext = make_dts_hd_ext(100, 0xBB);
|
||||
assert_eq!(dts_hd_ext_frame_size(&ext), 101);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ext_frame_size_zero() {
|
||||
// raw_size_field = 0 → frame size = 1
|
||||
let ext = vec![0x64, 0x58, 0x20, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00];
|
||||
assert_eq!(dts_hd_ext_frame_size(&ext), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ext_frame_size_large() {
|
||||
// raw = 0x1F << 11 | 0xFF << 3 | 0x07 = 0xFFFF = 65535
|
||||
// frame_size = 65536
|
||||
let ext = vec![0x64, 0x58, 0x20, 0x25, 0x00, 0x00, 0x1F, 0xFF, 0xFF];
|
||||
// byte6=0x1F, byte7=0xFF, byte8=0xFF
|
||||
// (0x1F << 11) | (0xFF << 3) | (0xFF >> 5) = 63488 | 2040 | 7 = 65535
|
||||
assert_eq!(dts_hd_ext_frame_size(&ext), 65536);
|
||||
}
|
||||
|
||||
// --- parse: core + extension frame ---
|
||||
|
||||
#[test]
|
||||
fn parse_core_plus_extension() {
|
||||
let mut parser = DtsParser::new();
|
||||
let core = make_dts_core(20); // 24 bytes total
|
||||
let ext = make_dts_hd_ext(50, 0xCC); // 51 bytes
|
||||
let mut data = core.clone();
|
||||
data.extend_from_slice(&ext);
|
||||
|
||||
let pes = make_pes(data.clone(), Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
// Frame should include core (24) + extension (51) = 75 bytes
|
||||
assert_eq!(frames[0].data.len(), 24 + 51);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
assert!(frames[0].keyframe);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_core_only() {
|
||||
let mut parser = DtsParser::new();
|
||||
let data = make_dts_core(10);
|
||||
let pes = make_pes(data.clone(), Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_core_plus_extension_truncated_at_buffer_end() {
|
||||
let mut parser = DtsParser::new();
|
||||
let core = make_dts_core(4); // 8 bytes
|
||||
// Extension claims 200 bytes but we only provide 20
|
||||
let ext = make_dts_hd_ext(199, 0xDD); // wants 200 bytes
|
||||
let mut data = core;
|
||||
// Only append partial extension (first 20 bytes)
|
||||
data.extend_from_slice(&ext[..20.min(ext.len())]);
|
||||
|
||||
let total_len = data.len();
|
||||
let pes = make_pes(data, Some(0));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
// Should be clamped to actual data length
|
||||
assert_eq!(frames[0].data.len(), total_len);
|
||||
}
|
||||
|
||||
// --- basic tests (carried over) ---
|
||||
|
||||
#[test]
|
||||
fn parse_basic_frame() {
|
||||
let mut parser = DtsParser::new();
|
||||
let data = vec![0x7F, 0xFE, 0x80, 0x01, 0xAA, 0xBB, 0xCC];
|
||||
let pes = make_pes(data.clone(), Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data, data);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_keyframes() {
|
||||
let mut parser = DtsParser::new();
|
||||
for i in 0..3 {
|
||||
let data = vec![0x7F, 0xFE, 0x80, 0x01, i];
|
||||
let pes = make_pes(data, Some(90000 * i as i64));
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert!(frames[0].keyframe, "DTS frame should always be keyframe");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_private_none() {
|
||||
let parser = DtsParser::new();
|
||||
assert!(parser.codec_private().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_empty_pes() {
|
||||
let mut parser = DtsParser::new();
|
||||
@@ -268,11 +179,33 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_pts() {
|
||||
fn parse_single_frame() {
|
||||
let mut parser = DtsParser::new();
|
||||
let pes = make_pes(vec![0x7F, 0xFE, 0x80, 0x01], None);
|
||||
let frame = make_dts_core(512);
|
||||
let pes = make_pes(frame, Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].pts_ns, 0);
|
||||
assert_eq!(frames[0].data.len(), 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_frame_spanning_two_pes() {
|
||||
let mut parser = DtsParser::new();
|
||||
let frame = make_dts_core(512);
|
||||
let mid = 256;
|
||||
|
||||
let pes1 = make_pes(frame[..mid].to_vec(), Some(90000));
|
||||
assert!(parser.parse(&pes1).is_empty());
|
||||
|
||||
let pes2 = make_pes(frame[mid..].to_vec(), Some(93000));
|
||||
let frames = parser.parse(&pes2);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data.len(), 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_private_none() {
|
||||
let parser = DtsParser::new();
|
||||
assert!(parser.codec_private().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
+9
-20
@@ -7,26 +7,15 @@
|
||||
//! For MKV: codec ID "S_VOBSUB".
|
||||
//! All frames are keyframes (each is a complete bitmap).
|
||||
|
||||
use super::{pts_to_ns, CodecParser, Frame, PesPacket};
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
pub struct DvdSubParser {
|
||||
/// Pre-formatted VobSub .idx palette header for codec_private.
|
||||
codec_data: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl Default for DvdSubParser {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl DvdSubParser {
|
||||
pub fn new() -> Self {
|
||||
Self { codec_data: None }
|
||||
}
|
||||
|
||||
/// Create a parser with pre-formatted codec private data (palette header).
|
||||
pub fn with_codec_data(codec_data: Option<Vec<u8>>) -> Self {
|
||||
pub fn new(codec_data: Option<Vec<u8>>) -> Self {
|
||||
Self { codec_data }
|
||||
}
|
||||
}
|
||||
@@ -87,7 +76,7 @@ pub fn format_palette(palette: &[[u8; 4]]) -> Vec<u8> {
|
||||
let mut parts: Vec<String> = Vec::with_capacity(palette.len());
|
||||
for color in palette {
|
||||
let [r, g, b] = ycbcr_to_rgb(color);
|
||||
parts.push(format!("{:02x}{:02x}{:02x}", r, g, b));
|
||||
parts.push(format!("{r:02x}{g:02x}{b:02x}"));
|
||||
}
|
||||
let line = format!("palette: {}\n", parts.join(", "));
|
||||
line.into_bytes()
|
||||
@@ -109,7 +98,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn passthrough_data() {
|
||||
let mut parser = DvdSubParser::new();
|
||||
let mut parser = DvdSubParser::new(None);
|
||||
let sub_data = vec![0x00, 0x0A, 0x00, 0x08, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05];
|
||||
let pes = make_pes(sub_data.clone(), Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
@@ -124,7 +113,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn always_keyframe() {
|
||||
let mut parser = DvdSubParser::new();
|
||||
let mut parser = DvdSubParser::new(None);
|
||||
for i in 0..3u8 {
|
||||
let data = vec![0x00, i, 0x00, i + 1];
|
||||
let pes = make_pes(data, Some(90000 * i as i64));
|
||||
@@ -139,21 +128,21 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn empty_pes_returns_no_frames() {
|
||||
let mut parser = DvdSubParser::new();
|
||||
let mut parser = DvdSubParser::new(None);
|
||||
let pes = make_pes(Vec::new(), Some(0));
|
||||
assert!(parser.parse(&pes).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_private_none_by_default() {
|
||||
let parser = DvdSubParser::new();
|
||||
let parser = DvdSubParser::new(None);
|
||||
assert!(parser.codec_private().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_private_returns_palette_when_set() {
|
||||
let palette_data = b"palette: 000000, ffffff\n".to_vec();
|
||||
let parser = DvdSubParser::with_codec_data(Some(palette_data.clone()));
|
||||
let parser = DvdSubParser::new(Some(palette_data.clone()));
|
||||
let cp = parser.codec_private();
|
||||
assert!(cp.is_some());
|
||||
assert_eq!(cp.unwrap(), palette_data);
|
||||
@@ -161,7 +150,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn no_pts_defaults_to_zero() {
|
||||
let mut parser = DvdSubParser::new();
|
||||
let mut parser = DvdSubParser::new(None);
|
||||
let pes = make_pes(vec![0x01, 0x02], None);
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
|
||||
+11
-8
@@ -4,7 +4,7 @@
|
||||
//! Detects keyframes (IDR slices).
|
||||
//! Each PES packet = one access unit = one frame.
|
||||
|
||||
use super::{pts_to_ns, CodecParser, Frame, PesPacket};
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
/// H.264 NAL unit types we care about.
|
||||
const NAL_SLICE_IDR: u8 = 5;
|
||||
@@ -111,12 +111,12 @@ impl CodecParser for H264Parser {
|
||||
// pictureParameterSetNALUnit = pps
|
||||
|
||||
let mut record = vec![
|
||||
1, // configurationVersion
|
||||
sps[1], // profile
|
||||
sps[2], // compatibility
|
||||
sps[3], // level
|
||||
0xFF, // 6 bits reserved (111111) + 2 bits lengthSizeMinusOne (11 = 3)
|
||||
0xE1, // 3 bits reserved (111) + 5 bits numSPS (1)
|
||||
1, // configurationVersion
|
||||
sps[1], // profile
|
||||
sps[2], // compatibility
|
||||
sps[3], // level
|
||||
0xFF, // 6 bits reserved (111111) + 2 bits lengthSizeMinusOne (11 = 3)
|
||||
0xE1, // 3 bits reserved (111) + 5 bits numSPS (1)
|
||||
(sps.len() >> 8) as u8,
|
||||
sps.len() as u8,
|
||||
];
|
||||
@@ -180,6 +180,9 @@ pub fn find_start_code(data: &[u8], from: usize) -> Option<usize> {
|
||||
if data.len() < from + 3 {
|
||||
return None;
|
||||
}
|
||||
// Range excludes last 2 bytes since we read 3 bytes at each position.
|
||||
// data.len()-2 as exclusive upper bound means last checked index is data.len()-3,
|
||||
// which accesses data[len-3], data[len-2], data[len-1] — all valid.
|
||||
(from..data.len() - 2).find(|&i| data[i] == 0x00 && data[i + 1] == 0x00 && data[i + 2] == 0x01)
|
||||
}
|
||||
|
||||
@@ -260,7 +263,7 @@ mod tests {
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01]);
|
||||
data.push(0x67); // SPS
|
||||
data.extend_from_slice(&[0x42, 0x00, 0x1E, 0xAB, 0xCD]); // profile=0x42, compat=0x00, level=0x1E
|
||||
// PPS: 00 00 01 [68 <payload>]
|
||||
// PPS: 00 00 01 [68 <payload>]
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01]);
|
||||
data.push(0x68); // PPS
|
||||
data.extend_from_slice(&[0xCE, 0x01]);
|
||||
|
||||
+25
-41
@@ -5,7 +5,7 @@
|
||||
//! Each PES packet = one access unit = one frame.
|
||||
|
||||
use super::h264::{find_start_code, skip_start_code};
|
||||
use super::{pts_to_ns, CodecParser, Frame, PesPacket};
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
// HEVC NAL unit types
|
||||
const NAL_VPS: u8 = 32;
|
||||
@@ -52,8 +52,9 @@ impl CodecParser for HevcParser {
|
||||
let pts_ns = pes.dts.or(pes.pts).map(pts_to_ns).unwrap_or(0);
|
||||
let data = &pes.data;
|
||||
let mut keyframe = false;
|
||||
let mut frame_data = Vec::new();
|
||||
|
||||
// Scan NAL units
|
||||
// Single-pass NAL scan: extract params, detect keyframes, build length-prefixed output
|
||||
let mut pos = 0;
|
||||
while let Some(sc_pos) = find_start_code(data, pos) {
|
||||
if let Some(nal_start) = skip_start_code(data, sc_pos) {
|
||||
@@ -68,45 +69,28 @@ impl CodecParser for HevcParser {
|
||||
let nal_type = (data[nal_start] >> 1) & 0x3F;
|
||||
|
||||
match nal_type {
|
||||
NAL_VPS => self.vps = Some(data[nal_start..end].to_vec()),
|
||||
NAL_SPS => self.sps = Some(data[nal_start..end].to_vec()),
|
||||
NAL_PPS => self.pps = Some(data[nal_start..end].to_vec()),
|
||||
NAL_VPS => {
|
||||
self.vps = Some(data[nal_start..end].to_vec());
|
||||
}
|
||||
NAL_SPS => {
|
||||
self.sps = Some(data[nal_start..end].to_vec());
|
||||
}
|
||||
NAL_PPS => {
|
||||
self.pps = Some(data[nal_start..end].to_vec());
|
||||
}
|
||||
NAL_AUD => {} // Skip access unit delimiters
|
||||
t if (NAL_BLA_W_LP..=NAL_RSV_IRAP_VCL23).contains(&t) => {
|
||||
keyframe = true;
|
||||
let nal = &data[nal_start..end];
|
||||
frame_data.extend_from_slice(&(nal.len() as u32).to_be_bytes());
|
||||
frame_data.extend_from_slice(nal);
|
||||
}
|
||||
_ => {
|
||||
// All other NAL types (slices, SEI, DV RPU, etc.) pass through
|
||||
let nal = &data[nal_start..end];
|
||||
frame_data.extend_from_slice(&(nal.len() as u32).to_be_bytes());
|
||||
frame_data.extend_from_slice(nal);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
pos = next;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert Annex B to length-prefixed NALUs.
|
||||
// Skip VPS/SPS/PPS/AUD — they're in codecPrivate.
|
||||
let mut frame_data = Vec::new();
|
||||
let mut pos = 0;
|
||||
while let Some(sc_pos) = find_start_code(&pes.data, pos) {
|
||||
if let Some(nal_start) = skip_start_code(&pes.data, sc_pos) {
|
||||
let next = find_start_code(&pes.data, nal_start).unwrap_or(pes.data.len());
|
||||
let mut end = next;
|
||||
while end > nal_start && pes.data[end - 1] == 0x00 {
|
||||
end -= 1;
|
||||
}
|
||||
|
||||
if nal_start < pes.data.len() {
|
||||
let nal_type = (pes.data[nal_start] >> 1) & 0x3F;
|
||||
// Skip parameter sets and AUD
|
||||
if nal_type != NAL_VPS
|
||||
&& nal_type != NAL_SPS
|
||||
&& nal_type != NAL_PPS
|
||||
&& nal_type != NAL_AUD
|
||||
{
|
||||
let nal = &pes.data[nal_start..end];
|
||||
let len = nal.len() as u32;
|
||||
frame_data.extend_from_slice(&len.to_be_bytes());
|
||||
frame_data.extend_from_slice(nal);
|
||||
}
|
||||
}
|
||||
pos = next;
|
||||
@@ -138,7 +122,7 @@ impl CodecParser for HevcParser {
|
||||
|
||||
// Minimal HEVCDecoderConfigurationRecord header
|
||||
record.push(1); // configurationVersion
|
||||
// General profile space, tier flag, profile IDC from SPS
|
||||
// General profile space, tier flag, profile IDC from SPS
|
||||
if sps.len() > 3 {
|
||||
record.push(sps[1]); // general_profile_space + general_tier_flag + general_profile_idc
|
||||
} else {
|
||||
@@ -170,7 +154,7 @@ impl CodecParser for HevcParser {
|
||||
record.push(0xFC);
|
||||
// chromaFormat (6 + 2 bits)
|
||||
record.push(0xFC | 1); // 4:2:0
|
||||
// bitDepthLumaMinus8 (5 + 3 bits)
|
||||
// bitDepthLumaMinus8 (5 + 3 bits)
|
||||
record.push(0xF8);
|
||||
// bitDepthChromaMinus8 (5 + 3 bits)
|
||||
record.push(0xF8);
|
||||
@@ -178,7 +162,7 @@ impl CodecParser for HevcParser {
|
||||
record.extend_from_slice(&[0, 0]);
|
||||
// constantFrameRate + numTemporalLayers + temporalIdNested + lengthSizeMinusOne
|
||||
record.push(0x03); // lengthSizeMinusOne = 3 (4 bytes)
|
||||
// numOfArrays
|
||||
// numOfArrays
|
||||
record.push(3); // VPS, SPS, PPS
|
||||
|
||||
// VPS array
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
//! For MKV: codec ID "A_PCM/INT/BIG" (BD) or "A_PCM/INT/LIT" (DVD).
|
||||
//! All frames are keyframes (uncompressed audio).
|
||||
|
||||
use super::{pts_to_ns, CodecParser, Frame, PesPacket};
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
/// BD LPCM header size in bytes.
|
||||
const BD_LPCM_HEADER_SIZE: usize = 4;
|
||||
|
||||
+6
-12
@@ -31,9 +31,11 @@ pub struct Frame {
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Convert 90kHz PTS to nanoseconds.
|
||||
/// Convert 90kHz PTS to nanoseconds (round to nearest).
|
||||
pub fn pts_to_ns(pts: i64) -> i64 {
|
||||
pts * 100_000 / 9
|
||||
// pts * 1_000_000_000 / 90_000 = pts * 100_000 / 9
|
||||
// Add half-divisor for rounding: (pts * 100_000 + 4) / 9
|
||||
(pts * 100_000 + 4) / 9
|
||||
}
|
||||
|
||||
/// Trait for codec-specific elementary stream parsers.
|
||||
@@ -77,18 +79,10 @@ impl CodecParser for PassthroughParser {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create the appropriate parser for a codec.
|
||||
pub fn parser_for_codec(codec: Codec) -> Box<dyn CodecParser> {
|
||||
parser_for_codec_with_data(codec, None)
|
||||
}
|
||||
|
||||
/// Create the appropriate parser for a codec, with optional codec private data.
|
||||
///
|
||||
/// For DvdSub, `codec_data` should be the pre-formatted VobSub .idx palette header.
|
||||
pub fn parser_for_codec_with_data(
|
||||
codec: Codec,
|
||||
codec_data: Option<Vec<u8>>,
|
||||
) -> Box<dyn CodecParser> {
|
||||
pub fn parser_for_codec(codec: Codec, codec_data: Option<Vec<u8>>) -> Box<dyn CodecParser> {
|
||||
match codec {
|
||||
Codec::H264 => Box::new(h264::H264Parser::new()),
|
||||
Codec::Hevc => Box::new(hevc::HevcParser::new()),
|
||||
@@ -99,7 +93,7 @@ pub fn parser_for_codec_with_data(
|
||||
Codec::TrueHd => Box::new(truehd::TrueHdParser::new()),
|
||||
Codec::Pgs => Box::new(pgs::PgsParser::new()),
|
||||
Codec::Lpcm => Box::new(lpcm::LpcmParser::new()),
|
||||
Codec::DvdSub => Box::new(dvdsub::DvdSubParser::with_codec_data(codec_data)),
|
||||
Codec::DvdSub => Box::new(dvdsub::DvdSubParser::new(codec_data)),
|
||||
_ => Box::new(PassthroughParser::new(true)),
|
||||
}
|
||||
}
|
||||
|
||||
+67
-17
@@ -9,7 +9,7 @@
|
||||
//! - Sequence extension: 00 00 01 B5
|
||||
//! - Picture header: 00 00 01 00
|
||||
|
||||
use super::{pts_to_ns, CodecParser, Frame};
|
||||
use super::{CodecParser, Frame, pts_to_ns};
|
||||
use crate::mux::ts::PesPacket;
|
||||
|
||||
/// Sequence header start code suffix.
|
||||
@@ -50,6 +50,8 @@ const ASPECT_RATIOS: [(u8, u8); 5] = [
|
||||
pub struct Mpeg2Parser {
|
||||
/// Raw bytes of the last seen sequence header (+ sequence extension if found).
|
||||
seq_header: Option<Vec<u8>>,
|
||||
/// Whether we've captured the sequence extension (B5) already.
|
||||
has_extension: bool,
|
||||
}
|
||||
|
||||
impl Default for Mpeg2Parser {
|
||||
@@ -60,7 +62,10 @@ impl Default for Mpeg2Parser {
|
||||
|
||||
impl Mpeg2Parser {
|
||||
pub fn new() -> Self {
|
||||
Self { seq_header: None }
|
||||
Self {
|
||||
seq_header: None,
|
||||
has_extension: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract resolution from a captured sequence header.
|
||||
@@ -105,23 +110,68 @@ impl CodecParser for Mpeg2Parser {
|
||||
|
||||
match code {
|
||||
SEQ_HEADER_CODE => {
|
||||
// Capture sequence header: from start code to next start code
|
||||
// (or to the sequence extension if present).
|
||||
// MPEG-2 sequence header: 00 00 01 B3 + variable data.
|
||||
// Base header: 8 bytes after start code = 12 bytes total.
|
||||
// Then possibly 64 intra quantizer values (bit-packed from bit 63).
|
||||
// Then possibly 64 non-intra quantizer values.
|
||||
// Then extensions (00 00 01 B5).
|
||||
//
|
||||
// Capture to the next start code within this PES data.
|
||||
// If no next start code exists (extension in next PES), capture
|
||||
// just the sequence header without extensions.
|
||||
let hdr_start = sc;
|
||||
let hdr_end = find_start_code(data, sc + 4).unwrap_or(data.len());
|
||||
let next_sc = find_start_code(data, sc + 4);
|
||||
let hdr_end = match next_sc {
|
||||
Some(next) if next + 3 < data.len() => {
|
||||
let mut end = next;
|
||||
// Include B5 extensions
|
||||
while end + 3 < data.len() && data[end + 3] == SEQ_EXT_CODE {
|
||||
end = find_start_code(data, end + 4).unwrap_or(data.len());
|
||||
}
|
||||
end
|
||||
}
|
||||
_ => {
|
||||
// No next start code in this PES — calculate exact header size.
|
||||
// Bit 62: load_intra_quantiser_matrix
|
||||
// Bit 62+1+512: load_non_intra_quantiser_matrix (if intra present)
|
||||
// Bit 62+1: load_non_intra_quantiser_matrix (if intra absent)
|
||||
if sc + 12 > data.len() {
|
||||
data.len()
|
||||
} else {
|
||||
let mut bits = 63u32; // bits consumed so far
|
||||
let intra = (data[sc + 11] & 0x02) != 0;
|
||||
if intra {
|
||||
bits += 64 * 8;
|
||||
}
|
||||
// Non-intra flag is at current bit position
|
||||
let byte_pos = (bits / 8) as usize;
|
||||
let bit_pos = 7 - (bits % 8) as u8;
|
||||
if sc + 4 + byte_pos < data.len() {
|
||||
let non_intra = (data[sc + 4 + byte_pos] >> bit_pos) & 1 != 0;
|
||||
bits += 1;
|
||||
if non_intra {
|
||||
bits += 64 * 8;
|
||||
}
|
||||
}
|
||||
let total_bytes = 4 + bits.div_ceil(8) as usize;
|
||||
(sc + total_bytes).min(data.len())
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let mut seq_data = data[hdr_start..hdr_end].to_vec();
|
||||
|
||||
// Check if sequence extension follows immediately.
|
||||
if hdr_end + 3 < data.len() && data[hdr_end + 3] == SEQ_EXT_CODE {
|
||||
let ext_end = find_start_code(data, hdr_end + 4).unwrap_or(data.len());
|
||||
seq_data.extend_from_slice(&data[hdr_end..ext_end]);
|
||||
}
|
||||
|
||||
self.seq_header = Some(seq_data);
|
||||
// Sequence header implies I-frame follows.
|
||||
self.seq_header = Some(data[hdr_start..hdr_end].to_vec());
|
||||
keyframe = true;
|
||||
pos = sc + 4;
|
||||
pos = if next_sc.is_some() { hdr_end } else { sc + 4 };
|
||||
}
|
||||
SEQ_EXT_CODE if self.seq_header.is_some() && !self.has_extension => {
|
||||
// Sequence extension appears after seq header (may be in next PES).
|
||||
// Append it to the stored seq_header.
|
||||
let ext_end = find_start_code(data, sc + 4).unwrap_or(data.len());
|
||||
if let Some(ref mut hdr) = self.seq_header {
|
||||
hdr.extend_from_slice(&data[sc..ext_end]);
|
||||
}
|
||||
self.has_extension = true;
|
||||
pos = ext_end;
|
||||
}
|
||||
PICTURE_CODE => {
|
||||
// Picture header: bytes after start code contain temporal_reference
|
||||
@@ -365,7 +415,7 @@ mod tests {
|
||||
// Sequence extension: 00 00 01 B5 [ext data]
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01, SEQ_EXT_CODE]);
|
||||
data.extend_from_slice(&[0x14, 0x8A, 0x00, 0x01, 0x00, 0x00]); // ext payload
|
||||
// Picture header follows.
|
||||
// Picture header follows.
|
||||
data.extend_from_slice(&make_picture_header(PICTURE_TYPE_I));
|
||||
data.extend_from_slice(&[0xFF; 4]);
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
//! Each PES packet contains one or more segments.
|
||||
//! All segments are keyframes (no inter-segment dependencies).
|
||||
|
||||
use super::{pts_to_ns, CodecParser, Frame, PesPacket};
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
pub struct PgsParser;
|
||||
|
||||
|
||||
+196
-41
@@ -1,14 +1,25 @@
|
||||
//! Dolby TrueHD / Atmos elementary stream parser.
|
||||
//!
|
||||
//! TrueHD major sync: 0xF8726FBA at a 4-byte aligned position.
|
||||
//! Access units consist of a major sync followed by minor syncs.
|
||||
//! An embedded AC3 core is in substream 0 for backward compatibility.
|
||||
//! All access units are keyframes.
|
||||
//! Each PES packet = one access unit.
|
||||
//! BD-TS TrueHD PES packets contain interleaved AC-3 + TrueHD access units.
|
||||
//! Access units span PES boundaries — must buffer and reassemble.
|
||||
//!
|
||||
//! TrueHD access unit header (4 bytes):
|
||||
//! [0..1] upper 4 bits = parity, lower 12 bits = length in 2-byte words
|
||||
//! [2..3] timing value
|
||||
//! [4..] substream data (major sync 0xF8726FBA may appear at offset 4)
|
||||
//!
|
||||
//! AC-3 frames (interleaved, same PID): start with sync word 0x0B77.
|
||||
//! We skip AC-3 frames and only emit TrueHD access units.
|
||||
|
||||
use super::{pts_to_ns, CodecParser, Frame, PesPacket};
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
pub struct TrueHdParser;
|
||||
/// Duration of one TrueHD access unit in nanoseconds (1/1200 second).
|
||||
const AU_DURATION_NS: i64 = 833_333;
|
||||
|
||||
pub struct TrueHdParser {
|
||||
buf: Vec<u8>,
|
||||
next_pts_ns: i64,
|
||||
}
|
||||
|
||||
impl Default for TrueHdParser {
|
||||
fn default() -> Self {
|
||||
@@ -18,7 +29,58 @@ impl Default for TrueHdParser {
|
||||
|
||||
impl TrueHdParser {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
Self {
|
||||
buf: Vec::with_capacity(32768),
|
||||
next_pts_ns: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Skip an AC-3 frame starting at the current buffer position.
|
||||
/// Returns number of bytes consumed, or 0 if not enough data.
|
||||
fn skip_ac3_frame(&self) -> usize {
|
||||
if self.buf.len() < 6 {
|
||||
return 0;
|
||||
}
|
||||
// AC-3 frame size from frmsizcod + fscod
|
||||
// Byte 4: [fscod:2][frmsizecod:6]
|
||||
let fscod = (self.buf[4] >> 6) & 0x03;
|
||||
let frmsizecod = (self.buf[4] & 0x3F) as usize;
|
||||
// Frame size in 16-bit words per fscod (simplified table for common rates)
|
||||
let frame_words = match fscod {
|
||||
0 => {
|
||||
// 48 kHz
|
||||
static SIZES: [usize; 38] = [
|
||||
64, 64, 80, 80, 96, 96, 112, 112, 128, 128, 160, 160, 192, 192, 224, 224, 256,
|
||||
256, 320, 320, 384, 384, 448, 448, 512, 512, 640, 640, 768, 768, 896, 896,
|
||||
1024, 1024, 1152, 1152, 1280, 1280,
|
||||
];
|
||||
SIZES.get(frmsizecod).copied().unwrap_or(0)
|
||||
}
|
||||
1 => {
|
||||
// 44.1 kHz
|
||||
static SIZES: [usize; 38] = [
|
||||
69, 70, 87, 88, 104, 105, 121, 122, 139, 140, 174, 175, 208, 209, 243, 244,
|
||||
278, 279, 348, 349, 417, 418, 487, 488, 557, 558, 696, 697, 835, 836, 975, 976,
|
||||
1114, 1115, 1253, 1254, 1393, 1394,
|
||||
];
|
||||
SIZES.get(frmsizecod).copied().unwrap_or(0)
|
||||
}
|
||||
2 => {
|
||||
// 32 kHz
|
||||
static SIZES: [usize; 38] = [
|
||||
96, 96, 120, 120, 144, 144, 168, 168, 192, 192, 240, 240, 288, 288, 336, 336,
|
||||
384, 384, 480, 480, 576, 576, 672, 672, 768, 768, 960, 960, 1152, 1152, 1344,
|
||||
1344, 1536, 1536, 1728, 1728, 1920, 1920,
|
||||
];
|
||||
SIZES.get(frmsizecod).copied().unwrap_or(0)
|
||||
}
|
||||
_ => 0,
|
||||
};
|
||||
let frame_bytes = frame_words * 2;
|
||||
if frame_bytes == 0 || self.buf.len() < frame_bytes {
|
||||
return 0;
|
||||
}
|
||||
frame_bytes
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,12 +89,62 @@ impl CodecParser for TrueHdParser {
|
||||
if pes.data.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let pts_ns = pes.pts.map(pts_to_ns).unwrap_or(0);
|
||||
vec![Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: pes.data.clone(),
|
||||
}]
|
||||
|
||||
if let Some(pts) = pes.pts {
|
||||
self.next_pts_ns = pts_to_ns(pts);
|
||||
}
|
||||
|
||||
self.buf.extend_from_slice(&pes.data);
|
||||
|
||||
let mut frames = Vec::new();
|
||||
|
||||
loop {
|
||||
if self.buf.len() < 4 {
|
||||
break;
|
||||
}
|
||||
|
||||
// AC-3 frame (interleaved): starts with sync word 0x0B77
|
||||
if self.buf[0] == 0x0B && self.buf[1] == 0x77 {
|
||||
let skip = self.skip_ac3_frame();
|
||||
if skip == 0 {
|
||||
break; // incomplete AC-3 frame, wait for more data
|
||||
}
|
||||
self.buf.drain(..skip);
|
||||
continue;
|
||||
}
|
||||
|
||||
// TrueHD access unit: lower 12 bits of first 2 bytes = length in words
|
||||
let unit_words = (((self.buf[0] as usize) << 8) | self.buf[1] as usize) & 0xFFF;
|
||||
if unit_words == 0 {
|
||||
self.buf.drain(..2);
|
||||
continue;
|
||||
}
|
||||
let unit_bytes = unit_words * 2;
|
||||
if unit_bytes > 32768 {
|
||||
// Likely misaligned — try to resync by scanning for AC-3 sync or
|
||||
// a valid TrueHD length
|
||||
self.buf.drain(..2);
|
||||
continue;
|
||||
}
|
||||
if self.buf.len() < unit_bytes {
|
||||
break; // incomplete access unit, wait for more data
|
||||
}
|
||||
|
||||
let is_major_sync = unit_bytes >= 8
|
||||
&& (u32::from_be_bytes([self.buf[4], self.buf[5], self.buf[6], self.buf[7]])
|
||||
& 0xFFFF_FFFE)
|
||||
== 0xF872_6FBA;
|
||||
|
||||
frames.push(Frame {
|
||||
pts_ns: self.next_pts_ns,
|
||||
keyframe: is_major_sync,
|
||||
data: self.buf[..unit_bytes].to_vec(),
|
||||
});
|
||||
self.buf.drain(..unit_bytes);
|
||||
self.next_pts_ns += AU_DURATION_NS;
|
||||
}
|
||||
|
||||
frames
|
||||
}
|
||||
|
||||
fn codec_private(&self) -> Option<Vec<u8>> {
|
||||
@@ -54,35 +166,21 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_basic_frame() {
|
||||
let mut parser = TrueHdParser::new();
|
||||
// TrueHD major sync: F8 72 6F BA (at 4-byte aligned position) + payload
|
||||
let data = vec![0xF8, 0x72, 0x6F, 0xBA, 0x01, 0x02, 0x03, 0x04];
|
||||
let pes = make_pes(data.clone(), Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data, data);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
fn make_truehd_unit(size_bytes: usize) -> Vec<u8> {
|
||||
let words = size_bytes / 2;
|
||||
let mut data = vec![0u8; size_bytes];
|
||||
data[0] = ((words >> 8) & 0x0F) as u8;
|
||||
data[1] = (words & 0xFF) as u8;
|
||||
data
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_keyframes() {
|
||||
let mut parser = TrueHdParser::new();
|
||||
for i in 0..3 {
|
||||
let data = vec![0xF8, 0x72, 0x6F, 0xBA, i];
|
||||
let pes = make_pes(data, Some(90000 * i as i64));
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert!(frames[0].keyframe, "TrueHD frame should always be keyframe");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_private_none() {
|
||||
let parser = TrueHdParser::new();
|
||||
assert!(parser.codec_private().is_none());
|
||||
fn make_ac3_frame() -> Vec<u8> {
|
||||
// Minimal AC-3 frame: sync 0x0B77, fscod=0 (48kHz), frmsizecod=0 (64 words = 128 bytes)
|
||||
let mut data = vec![0u8; 128];
|
||||
data[0] = 0x0B;
|
||||
data[1] = 0x77;
|
||||
data[4] = 0x00; // fscod=0, frmsizecod=0
|
||||
data
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -91,4 +189,61 @@ mod tests {
|
||||
let pes = make_pes(Vec::new(), Some(0));
|
||||
assert!(parser.parse(&pes).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_single_unit() {
|
||||
let mut parser = TrueHdParser::new();
|
||||
let unit = make_truehd_unit(200);
|
||||
let pes = make_pes(unit, Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data.len(), 200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_unit_spanning_two_pes() {
|
||||
let mut parser = TrueHdParser::new();
|
||||
let unit = make_truehd_unit(200);
|
||||
let mid = 100;
|
||||
|
||||
let pes1 = make_pes(unit[..mid].to_vec(), Some(90000));
|
||||
assert!(parser.parse(&pes1).is_empty());
|
||||
|
||||
let pes2 = make_pes(unit[mid..].to_vec(), Some(93000));
|
||||
let frames = parser.parse(&pes2);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data.len(), 200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_multiple_units_incrementing_pts() {
|
||||
let mut parser = TrueHdParser::new();
|
||||
let mut data = make_truehd_unit(100);
|
||||
data.extend_from_slice(&make_truehd_unit(120));
|
||||
let pes = make_pes(data, Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 2);
|
||||
assert_eq!(frames[0].data.len(), 100);
|
||||
assert_eq!(frames[1].data.len(), 120);
|
||||
assert_eq!(frames[1].pts_ns - frames[0].pts_ns, AU_DURATION_NS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skip_interleaved_ac3() {
|
||||
let mut parser = TrueHdParser::new();
|
||||
let ac3 = make_ac3_frame();
|
||||
let truehd = make_truehd_unit(200);
|
||||
let mut data = ac3;
|
||||
data.extend_from_slice(&truehd);
|
||||
let pes = make_pes(data, Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data.len(), 200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_private_none() {
|
||||
let parser = TrueHdParser::new();
|
||||
assert!(parser.codec_private().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user