mirror of
https://github.com/freemkv/libfreemkv.git
synced 2026-10-07 18:13:10 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ea15d212de | ||
|
|
00673c8ec3 | ||
|
|
bfefb4cb5b | ||
|
|
1b95193517 | ||
|
|
7dcac44136 | ||
|
|
4da559e39f | ||
|
|
eeca250f69 | ||
|
|
c51b3181f2 | ||
|
|
2a31a47434 | ||
|
|
b08662f95c | ||
|
|
4594196b6e | ||
|
|
9d6974df3e | ||
|
|
3394a5b3fe | ||
|
|
e2c7e20329 | ||
|
|
ff22385a61 | ||
|
|
d39d3933f5 | ||
|
|
31dc3d35c3 | ||
|
|
f7165a2020 | ||
|
|
d836f8b1e7 | ||
|
|
8919de5b03 | ||
|
|
0dc1f12108 | ||
|
|
6abc25a7e8 | ||
|
|
4d22cf632f | ||
|
|
611ad77586 | ||
|
|
60590193e3 | ||
|
|
5f24843d3c | ||
|
|
ecf9a4d01e | ||
|
|
8ec767339e | ||
|
|
335e8b68cc | ||
|
|
1137b4ea1f | ||
|
|
7788c560d1 | ||
|
|
84e0c2ca69 | ||
|
|
f4a881b25d | ||
|
|
1edda7fedb | ||
|
|
3dd71a0650 | ||
|
|
579c94bd23 | ||
|
|
3b2a68fe7a | ||
|
|
5b56f112a0 | ||
|
|
f27e4c3088 | ||
|
|
b3576f2ae6 | ||
|
|
39f2d0e992 | ||
|
|
50867516b8 | ||
|
|
193e7c1680 | ||
|
|
b6463729e8 | ||
|
|
dffee56102 | ||
|
|
d8f27cdee0 | ||
|
|
fc13268dc6 | ||
|
|
d9b55f02d0 | ||
|
|
35d66d2059 | ||
|
|
d703ce439b | ||
|
|
6112a26d15 | ||
|
|
2a33364253 | ||
|
|
fa3872ddcc | ||
|
|
c427389f36 | ||
|
|
6980f562df | ||
|
|
e3b2c9d850 | ||
|
|
a383200ef1 | ||
|
|
51d15f551e | ||
|
|
523af461a5 | ||
|
|
695b65149e | ||
|
|
d3f9560689 | ||
|
|
1fa5a7d27f | ||
|
|
55219070f1 | ||
|
|
a34c419521 | ||
|
|
b179846f5d | ||
|
|
e110e80e6e | ||
|
|
597fa34099 | ||
|
|
52eeb949e3 | ||
|
|
6ca62b8411 | ||
|
|
04195c27d8 | ||
|
|
5a8f8e54e1 | ||
|
|
1a2c830cf6 | ||
|
|
637c2bfe36 | ||
|
|
0d28357b4d | ||
|
|
5495332f07 | ||
|
|
e22fc6fd47 | ||
|
|
5b98c13e47 | ||
|
|
8f8f1a62a2 | ||
|
|
e3cfd27942 | ||
|
|
aa12bdad62 | ||
|
|
ef3895cdc5 | ||
|
|
2dcf969ac8 | ||
|
|
005f887bf9 | ||
|
|
d7243a6044 | ||
|
|
f1926c38dc | ||
|
|
4709a73c80 | ||
|
|
55bd1ee868 | ||
|
|
5285e8b6ec | ||
|
|
c5af61e46b | ||
|
|
ed73c1ab68 | ||
|
|
5ce2df0888 | ||
|
|
ea591a36a5 | ||
|
|
368e10486e | ||
|
|
a99c4f8487 | ||
|
|
4df470e572 | ||
|
|
a0b13941fe | ||
|
|
275c9da009 | ||
|
|
da183722fa | ||
|
|
4a5424eabe | ||
|
|
293d89c586 | ||
|
|
4840f5134d | ||
|
|
11c8211605 | ||
|
|
1038beaf06 | ||
|
|
f1df57196c | ||
|
|
8ba9dc1c0b | ||
|
|
49b131f39a | ||
|
|
8238ec4ce7 | ||
|
|
4fd5d27df8 | ||
|
|
1fbe272832 | ||
|
|
6bd635725e | ||
|
|
09b77b4dea | ||
|
|
688058b3e8 | ||
|
|
c96bac7977 | ||
|
|
c32acff3e4 | ||
|
|
b58e2d9873 | ||
|
|
a9e802c1c2 | ||
|
|
a876ce846b | ||
|
|
058fd8396f | ||
|
|
5ee28c08b9 | ||
|
|
764230b1eb | ||
|
|
7fb2e07aed | ||
|
|
376aadb335 | ||
|
|
586495b2ee | ||
|
|
72f2224efe | ||
|
|
0bca7a11bd | ||
|
|
72ab714c1e | ||
|
|
d7fb1b35ed | ||
|
|
06be4defd2 | ||
|
|
a832bad697 | ||
|
|
26e2d847d5 | ||
|
|
4442fa2df6 | ||
|
|
23dc55661c | ||
|
|
fbdb50c79f | ||
|
|
7cc74f0087 | ||
|
|
3aa1e528c5 | ||
|
|
7c6b0f82ab | ||
|
|
4226a53e73 | ||
|
|
92f34a289e | ||
|
|
fdbe469d50 | ||
|
|
4ec75a03f2 | ||
|
|
99236ffd55 | ||
|
|
9e3327a569 | ||
|
|
0081955686 | ||
|
|
3244fdd683 | ||
|
|
ef5487e5a5 | ||
|
|
e7dffa63d2 | ||
|
|
e5989c8270 | ||
|
|
e1c938b82a | ||
|
|
ed801a708b | ||
|
|
d94451954c | ||
|
|
385c9f094c | ||
|
|
9f2a13739d | ||
|
|
d9ce69bc9d | ||
|
|
7cd2c937ed | ||
|
|
107524da5e | ||
|
|
055a3c5276 | ||
|
|
50b04790ce | ||
|
|
a00b2c884b | ||
|
|
44e2c64d7e | ||
|
|
d6535b8f57 | ||
|
|
90ab00ed45 | ||
|
|
6bfd7dfd13 | ||
|
|
b9a7f601d4 | ||
|
|
cf0a61f8b4 | ||
|
|
f98f07b2d3 | ||
|
|
b53454fa09 | ||
|
|
f28b6ee6d9 | ||
|
|
d9d778fa64 | ||
|
|
bbfb887a35 | ||
|
|
9884346c24 | ||
|
|
766b7c6636 | ||
|
|
925c30686b | ||
|
|
f52e4c5d22 | ||
|
|
8d67790d2e | ||
|
|
246a439990 | ||
|
|
bd19041648 | ||
|
|
198268b725 | ||
|
|
31424c8203 | ||
|
|
c747ecc589 | ||
|
|
283a561c12 | ||
|
|
2667d68675 | ||
|
|
57f2c22e30 | ||
|
|
5f3545d244 | ||
|
|
6e17ef0859 | ||
|
|
6ec97af104 | ||
|
|
3a6c1aa5a3 | ||
|
|
1ba3264747 | ||
|
|
ae2909fe8d | ||
|
|
6a3d19a453 | ||
|
|
a35596d2d1 | ||
|
|
318f654fed | ||
|
|
5e82ef65de | ||
|
|
8534607329 | ||
|
|
97e1a4cad3 | ||
|
|
3f6bf33de5 | ||
|
|
359301e8fc | ||
|
|
bfa527162a | ||
|
|
45defd47f3 | ||
|
|
fe8933dad2 | ||
|
|
0a3c1b7b70 | ||
|
|
2bcdc97341 | ||
|
|
4863e9c545 | ||
|
|
03db038dd9 | ||
|
|
95001ba2c3 | ||
|
|
a1f4dff6f6 | ||
|
|
7dd5001d45 | ||
|
|
67fe93c0b8 | ||
|
|
5c4f575c46 | ||
|
|
e2cb7a78bb | ||
|
|
a6f1bd19bc | ||
|
|
9e3d0f1383 | ||
|
|
1e6eb0698d | ||
|
|
bd744171e4 | ||
|
|
7de9d4d42c | ||
|
|
9ac0c2fcdf | ||
|
|
3aa27d5d1b | ||
|
|
5083f70cff | ||
|
|
477cf7ee0e | ||
|
|
0b079e27ce | ||
|
|
5f69fd2a44 | ||
|
|
befe6ef69b | ||
|
|
29491d6eaf | ||
|
|
2e602f7e37 | ||
|
|
a126f31b70 | ||
|
|
6ff10df83a | ||
|
|
52b8522a75 | ||
|
|
13eb4331b6 | ||
|
|
1a3f7ce676 | ||
|
|
e1a1f730be | ||
|
|
646c22ae93 | ||
|
|
2f6092a686 | ||
|
|
e5a90a6567 | ||
|
|
afce1031d5 | ||
|
|
2c0bff1303 | ||
|
|
2067ebe485 | ||
|
|
2ce4c20221 | ||
|
|
da3004b120 | ||
|
|
b5ed97801b | ||
|
|
ae76aaf0fa | ||
|
|
2cd4fbead7 | ||
|
|
ebffc6eb88 | ||
|
|
424d3cd4f2 | ||
|
|
d2905ba7bb | ||
|
|
b4de5d343d | ||
|
|
8beeac7df9 | ||
|
|
b33f41e219 | ||
|
|
e85e20f436 | ||
|
|
7bc54be8d3 | ||
|
|
6d9083743b | ||
|
|
c49a68054f | ||
|
|
4bd38787c6 | ||
|
|
b4951b1c5b | ||
|
|
92dc145864 | ||
|
|
36b65f526a | ||
|
|
ca8ebf418f | ||
|
|
34182d956a | ||
|
|
0f967a2084 | ||
|
|
0341995c0d | ||
|
|
fe0ec0c5cf | ||
|
|
022657d481 | ||
|
|
fc8eca44e1 | ||
|
|
43836865be | ||
|
|
5e22199441 | ||
|
|
761b77bbd9 | ||
|
|
cc05ef2a3a | ||
|
|
e27b82ce5b | ||
|
|
8af47e4c19 |
@@ -14,6 +14,10 @@ jobs:
|
||||
with:
|
||||
components: clippy, rustfmt
|
||||
- run: cargo fmt --check
|
||||
# libfreemkv is a library — Cargo.lock is gitignored. --locked
|
||||
# would always fail on a fresh runner because there's no committed
|
||||
# lockfile to lock against. The binary crates (freemkv, autorip,
|
||||
# bdemu) track Cargo.lock and DO use --locked.
|
||||
- run: cargo clippy -- -D warnings
|
||||
|
||||
test:
|
||||
@@ -21,7 +25,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
- run: cargo test
|
||||
- run: cargo test --tests
|
||||
|
||||
check-macos:
|
||||
runs-on: macos-latest
|
||||
|
||||
@@ -28,6 +28,8 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
# libfreemkv is a library — Cargo.lock isn't tracked, so --locked
|
||||
# would always fail (no lockfile to lock against on a fresh runner).
|
||||
- run: cargo test
|
||||
|
||||
publish:
|
||||
|
||||
@@ -4,3 +4,4 @@ Cargo.lock
|
||||
*.swo
|
||||
.DS_Store
|
||||
.cargo/
|
||||
.claude/worktrees/
|
||||
|
||||
+1534
File diff suppressed because it is too large
Load Diff
+25
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "libfreemkv"
|
||||
version = "0.13.0"
|
||||
version = "0.25.5"
|
||||
edition = "2024"
|
||||
rust-version = "1.86"
|
||||
license = "AGPL-3.0-only"
|
||||
@@ -24,6 +24,27 @@ rand = "0.8"
|
||||
cmac = "0.7"
|
||||
zip = { version = "2", default-features = false, features = ["deflate"] }
|
||||
base64 = "0.22.1"
|
||||
# Trace-level instrumentation for Disc::copy + SgIoTransport::execute. Permitted
|
||||
# under project docs ("Acceptable strings: debug/trace logging"). Consumers (autorip)
|
||||
# wire a tracing subscriber and pipe events into the JSONL debug log.
|
||||
tracing = "0.1"
|
||||
# Bounded MPSC channel with kernel-wakeup send_timeout. Used by `io::pipeline`
|
||||
# so the halt-aware send/finish loops can BLOCK on consumer drain instead of
|
||||
# polling — the 50 ms poll cadence of the previous mpsc-based impl capped mux
|
||||
# throughput at ~1 MB/s (see (internal)/memory/
|
||||
# feedback_send_with_halt_poll_throttle.md, 0.21.7).
|
||||
crossbeam-channel = "0.5"
|
||||
# Persistent work-stealing thread pool for parallel AACS unit
|
||||
# decryption. Per-call std::thread::scope spawned fresh OS threads
|
||||
# and that overhead dominated for typical batch sizes (60 units).
|
||||
# rayon's global pool initialises once on first use.
|
||||
rayon = "1"
|
||||
# SIMD-accelerated bytestring search. Drives the HEVC/H.264 start-code
|
||||
# scan in `mux::codec::h264::find_start_code` — naive byte-by-byte
|
||||
# walk is ~500 MB/s single-thread on x86_64; memchr's vectorised
|
||||
# `memmem::find` for the 3-byte `00 00 01` needle hits ~5 GB/s on
|
||||
# AVX2-capable hosts.
|
||||
memchr = "2"
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
libc = "0.2"
|
||||
@@ -31,6 +52,9 @@ libc = "0.2"
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
[[bench]]
|
||||
name = "sgio_read"
|
||||
harness = false
|
||||
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
# libfreemkv — Feature List
|
||||
|
||||
## v0.10.10 (current)
|
||||
|
||||
### Done
|
||||
- [x] Drive access: open, identify, unlock, firmware upload, speed calibration, eject
|
||||
- [x] 206 bundled drive profiles (MediaTek MT1959 A + B variants)
|
||||
- [x] SCSI transport: Linux SG_IO, macOS IOKit, Windows SPTI
|
||||
- [x] UDF 2.50 filesystem parser (metadata partitions, Blu-ray profile)
|
||||
- [x] MPLS playlist parser (play items, STN table, secondary streams)
|
||||
- [x] CLPI clip info parser (EP map, sector extents)
|
||||
- [x] AACS 1.0 decryption (4 VUK paths: KEYDB, media key, processing key, device key)
|
||||
- [x] AACS 2.0 SCSI handshake (P-256/SHA-256 ECDH, bus decryption, read data key)
|
||||
- [x] DVD IFO parser (VMG, VTS, PGC chains, cell addresses)
|
||||
- [x] DVD CSS decryption (bus auth, disc key via player keys, title key, sector descramble)
|
||||
- [x] CSS Stevenson plaintext attack for ISO key recovery
|
||||
- [x] MPEG-2 PS demuxer (DVD Program Stream with PES extraction)
|
||||
- [x] MPEG-2 video codec parser (sequence header, quantizer matrices, keyframe detection)
|
||||
- [x] KEYDB.cfg download, verify, save (raw TCP, zero HTTP deps)
|
||||
- [x] Content reading with adaptive batch sizing, error recovery, 12+ MB/s
|
||||
- [x] Stream labels: 5 BD-J format parsers (Paramount, Criterion, Pixelogic, CTRM, Deluxe)
|
||||
- [x] MKV muxer: 14 codec parsers (H.264, HEVC, MPEG-2, AC-3, DTS, TrueHD, PGS, DVD Sub, VC-1, LPCM, +4 more)
|
||||
- [x] SectorReader trait: decouples disc scanning from SCSI
|
||||
- [x] 7 stream types: Disc, ISO, MKV, M2TS, Network, Stdio, Null
|
||||
- [x] PES pipeline with unified Stream trait (any source → any dest)
|
||||
- [x] FMKV metadata header for M2TS and network streams
|
||||
- [x] Numeric error codes only (no English text in library)
|
||||
- [x] Event system for progress callbacks
|
||||
|
||||
### Planned
|
||||
- [ ] Windows testing on real hardware
|
||||
- [ ] Pioneer Renesas platform support (48 drives, need GET_CONFIG 010C)
|
||||
- [ ] TranscodeStream (ffmpeg integration)
|
||||
- [ ] ISO write with BD-compliant UDF structure
|
||||
@@ -0,0 +1,20 @@
|
||||
# libfreemkv — local dev helper.
|
||||
# Mirrors the cross-crate scripts in (internal)/scripts/test-all.sh
|
||||
# but scoped to this single crate.
|
||||
|
||||
.PHONY: test build check ci clean
|
||||
|
||||
test:
|
||||
cargo test --tests
|
||||
|
||||
build:
|
||||
cargo build --release
|
||||
|
||||
check:
|
||||
cargo fmt --check
|
||||
cargo clippy --all-targets -- -D warnings
|
||||
|
||||
ci: check build test
|
||||
|
||||
clean:
|
||||
cargo clean
|
||||
@@ -18,7 +18,7 @@ Part of the [freemkv](https://github.com/freemkv) project.
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
libfreemkv = "0.11"
|
||||
libfreemkv = "0.25"
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
@@ -53,27 +53,47 @@ output.finish()?;
|
||||
|
||||
### Multi-pass recovery rip
|
||||
|
||||
For damaged discs, the library offers a two-stage rip model: fast sweep with zero-fill and a ddrescue-format mapfile, then targeted retry of bad ranges. See [`docs/rip-recovery.md`](docs/rip-recovery.md) for the full architecture.
|
||||
For damaged discs the library exposes two flat verbs — `Disc::sweep` for the
|
||||
forward Pass 1 and `Disc::patch` for retrying bad ranges. The library never
|
||||
loops; the multipass policy is the caller's job. See
|
||||
[`docs/rip-recovery.md`](docs/rip-recovery.md).
|
||||
|
||||
```rust
|
||||
use libfreemkv::disc::{CopyOptions, PatchOptions};
|
||||
use libfreemkv::{SweepOptions, PatchOptions};
|
||||
use libfreemkv::disc::{mapfile, mapfile_path_for};
|
||||
use std::path::Path;
|
||||
|
||||
// Pass 1: disc → ISO. Fast 64 KB reads, skip-forward on failure,
|
||||
// zero-fill bad blocks, write a sidecar .mapfile.
|
||||
let mut result = disc.copy(
|
||||
&mut drive,
|
||||
Path::new("disc.iso"),
|
||||
&CopyOptions { skip_on_error: true, skip_forward: true, ..Default::default() },
|
||||
)?;
|
||||
let iso = Path::new("disc.iso");
|
||||
|
||||
// Pass 2..N: retry bad ranges with full drive recovery.
|
||||
// Idempotent — call as many times as you want.
|
||||
while result.bytes_unreadable + result.bytes_pending > 0 {
|
||||
let pr = disc.patch(&mut drive, Path::new("disc.iso"), &PatchOptions::default())?;
|
||||
if pr.bytes_recovered_this_pass == 0 { break; }
|
||||
// Pass 1: disc → ISO. Skip-on-error, zero-fill, write the sidecar mapfile.
|
||||
disc.sweep(&mut drive, iso, &SweepOptions {
|
||||
decrypt: true,
|
||||
resume: false,
|
||||
batch_sectors: None,
|
||||
skip_on_error: true,
|
||||
progress: None,
|
||||
halt: None,
|
||||
})?;
|
||||
|
||||
// Pass 2..N: retry every non-finished range. Idempotent.
|
||||
loop {
|
||||
let map = mapfile::Mapfile::load(&mapfile_path_for(iso))?;
|
||||
let stats = map.stats();
|
||||
if stats.bytes_pending + stats.bytes_unreadable == 0 { break; }
|
||||
|
||||
let outcome = disc.patch(&mut drive, iso, &PatchOptions {
|
||||
decrypt: true,
|
||||
block_sectors: None,
|
||||
full_recovery: true,
|
||||
reverse: true,
|
||||
wedged_threshold: 50,
|
||||
progress: None,
|
||||
halt: None,
|
||||
})?;
|
||||
if outcome.bytes_recovered_this_pass == 0 { break; }
|
||||
}
|
||||
|
||||
// Then mux from the ISO via the normal stream pipeline (no drive involvement).
|
||||
// Mux from the ISO via the normal stream pipeline (no drive involvement).
|
||||
```
|
||||
|
||||
## What It Does
|
||||
@@ -84,7 +104,7 @@ while result.bytes_unreadable + result.bytes_pending > 0 {
|
||||
- **Stream labels** — 5 BD-J format parsers (Paramount, Criterion, Pixelogic, CTRM, Deluxe)
|
||||
- **AACS decryption** — transparent key resolution and content decrypt (1.0 + 2.0 bus decryption)
|
||||
- **KEYDB updates** — download, verify, save from any HTTP URL (zero deps, raw TCP)
|
||||
- **Content reading** — adaptive batch reads with automatic decryption and error recovery
|
||||
- **Content reading** — adaptive batch reads with automatic decryption
|
||||
- **Stream I/O** — unified stream pipeline for reading and writing any format
|
||||
|
||||
### Streams
|
||||
@@ -92,21 +112,21 @@ while result.bytes_unreadable + result.bytes_pending > 0 {
|
||||
| Stream | Input | Output | Transport |
|
||||
|--------|-------|--------|-----------|
|
||||
| DiscStream | Yes | -- | Optical drive via SCSI |
|
||||
| IsoStream | Yes | Yes | Blu-ray ISO image file |
|
||||
| IsoStream | Yes | -- | Blu-ray ISO image file (read via stream pipeline; written via `Disc::sweep()`) |
|
||||
| MkvStream | Yes | Yes | Matroska container |
|
||||
| M2tsStream | Yes | Yes | BD transport stream with FMKV metadata header |
|
||||
| NetworkStream | Yes (listen) | Yes (connect) | TCP with FMKV metadata header |
|
||||
| StdioStream | Yes (stdin) | Yes (stdout) | Raw byte pipe |
|
||||
| NullStream | -- | Yes | Discard sink (byte counter for benchmarks) |
|
||||
|
||||
Streams implement `IOStream` (byte-level) and `pes::Stream` (frame-level). `input()` / `output()` resolve URL strings to PES stream instances. `open_input()` / `open_output()` resolve to byte-level IOStream instances. All URLs use the `scheme://path` format — bare paths are rejected.
|
||||
Streams implement `FrameSource` (read) and/or `FrameSink` (write); direction is type-checked. `input()` / `output()` resolve URL strings to PES stream instances. All URLs use the `scheme://path` format — bare paths are rejected.
|
||||
|
||||
AACS decryption requires a KEYDB.cfg file. If available at `~/.config/aacs/KEYDB.cfg` or passed via `ScanOptions`, the library handles everything — handshake, key derivation, and per-sector decryption — without the application needing to know anything about encryption.
|
||||
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
Drive — open, identify, init, unlock, read (with recovery)
|
||||
Drive — open, identify, init, unlock, single-shot read
|
||||
├── ScsiTransport — SG_IO (Linux), IOKit (macOS), SPTI (Windows)
|
||||
├── DriveProfile — per-drive unlock parameters (bundled)
|
||||
└── PlatformDriver — MediaTek (supported), Renesas (planned)
|
||||
@@ -122,7 +142,8 @@ Disc — scan titles, streams, AACS/CSS state
|
||||
└── KEYDB — download + verify + save
|
||||
|
||||
Streams — unified PES pipeline
|
||||
├── pes::Stream — read()/write() PES frames
|
||||
├── FrameSource — read() PES frames (direction-typed)
|
||||
├── FrameSink — write() PES frames (direction-typed)
|
||||
├── DiscStream — sectors → decrypt → TS demux → PES
|
||||
├── IsoStream — ISO file → decrypt → TS demux → PES
|
||||
├── MkvStream — MKV mux/demux
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
# Troubleshooting Guide
|
||||
|
||||
Common problems and solutions for optical drive ripping with freemkv.
|
||||
|
||||
---
|
||||
|
||||
## 1. USB-SATA Bridge Issues
|
||||
|
||||
This is the single most common source of problems when ripping discs over USB.
|
||||
|
||||
### Symptoms
|
||||
|
||||
- The drive disappears mid-rip. The ripping tool reports the device is gone, and `ls /dev/sg*` no longer shows it.
|
||||
- The device re-enumerates under a different name: `sg4` becomes `sg5`, then `sg7`, then `sg11` after each USB port reset.
|
||||
- `dmesg` shows USB port resets: `usb X-Y: reset high-speed USB device`, `xhci_hcd 0000:00:14.0: Cannot enable. Maybe the USB cable is bad?`, or `usb-storage: device reset failed`.
|
||||
- The SCSI layer reports `host_status=7` (Linux USB transport error) in sense data.
|
||||
- The drive works fine for reading data discs or burning, but crashes when hitting damaged sectors during a rip.
|
||||
- After the crash, the drive is completely invisible until physically unplugged and reconnected.
|
||||
|
||||
### Root Cause
|
||||
|
||||
USB-SATA bridges translate between the USB Mass Storage protocol (BOT or UAS) and the drive's native SATA interface. When the optical drive encounters an unreadable sector, it returns a SCSI CHECK CONDITION with sense key 0x03 (MEDIUM ERROR). Some bridge chipsets -- particularly the Initio INIC-36xx family -- have buggy firmware that mishandles this error response.
|
||||
|
||||
Specific failure modes:
|
||||
|
||||
- **Incorrect residue reporting.** The bridge claims a different number of bytes transferred than what actually occurred. The Linux USB storage driver sees this discrepancy as a protocol violation and resets the port to recover. The `US_FL_IGNORE_RESIDUE` quirk exists specifically for this class of bug (see `drivers/usb/storage/transport.c` in the Linux kernel).
|
||||
- **Bridge firmware crash.** On some Initio bridges, a malformed SCSI error response from the drive causes the bridge MCU to hang entirely. The USB controller sees the device stop responding and initiates a port reset. The bridge recovers (it re-enumerates), but the rip is dead -- all state is lost.
|
||||
- **Sense data corruption.** The bridge forwards garbled or truncated sense data to the host, which the SCSI midlayer cannot parse, leading to a transport reset.
|
||||
|
||||
This is a hardware + firmware problem, not a software bug. The same drive connected via direct SATA does not exhibit these symptoms.
|
||||
|
||||
### Known Affected Bridges
|
||||
|
||||
| Chipset | USB IDs | Notes |
|
||||
|---------|---------|-------|
|
||||
| Initio INIC-3609 | `13fd:3609` | Very common in cheap SATA-to-USB enclosures. Highly problematic. |
|
||||
| Initio INIC-3619 | `13fd:3940` | Same firmware family as INIC-3609. |
|
||||
| Initio INIC-3069 | `13fd:0840` | Older variant, same residue bug. |
|
||||
| ASMedia ASM1051 | `174c:5106` | Early ASM SATA bridge. Residue issues on error paths. |
|
||||
| JMicron JMB36x | `152d:0561` | Some firmware versions. Not all JMicroon chips are affected. |
|
||||
|
||||
If your drive came in a pre-built external enclosure (Vantec, Sabrent, OWC, etc.), it almost certainly uses one of these bridge chips internally.
|
||||
|
||||
### The Fix: USB Storage Quirk
|
||||
|
||||
Apply the `US_FL_IGNORE_RESIDUE` kernel quirk for your bridge. This tells the Linux USB storage driver to ignore the residue field in SCSI response frames, preventing the port reset on mismatched byte counts.
|
||||
|
||||
**Step 1: Identify your bridge's vendor:product ID.**
|
||||
|
||||
```bash
|
||||
lsusb
|
||||
```
|
||||
|
||||
Look for your drive's entry. Example output:
|
||||
|
||||
```
|
||||
Bus 002 Device 005: ID 13fd:0840 Initio Corporation INIC-3609
|
||||
```
|
||||
|
||||
Here the vendor ID is `13fd` and the product ID is `0840`.
|
||||
|
||||
**Step 2: Apply the quirk at runtime.**
|
||||
|
||||
```bash
|
||||
echo "13fd:0840:i" > /sys/module/usb_storage/parameters/quirks
|
||||
```
|
||||
|
||||
Replace `13fd:0840` with your device's actual IDs. The `:i` flag means `US_FL_IGNORE_RESIDUE`.
|
||||
|
||||
You can combine multiple flags. Common additions:
|
||||
|
||||
- `:i` -- ignore residue (`US_FL_IGNORE_RESIDUE`)
|
||||
- `:u` -- force BOT mode instead of UAS, for bridges with UAS bugs
|
||||
|
||||
**Step 3: Reconnect the drive.** Unplug and replug the USB cable, or bind/unbind the device. The quirk is applied per-module-load, so existing sessions may need the drive reconnected.
|
||||
|
||||
### Making It Persistent
|
||||
|
||||
Add the quirk to your kernel boot parameters so it survives reboots.
|
||||
|
||||
Edit `/etc/default/grub` (GRUB) and add to `GRUB_CMDLINE_LINUX_DEFAULT`:
|
||||
|
||||
```
|
||||
GRUB_CMDLINE_LINUX_DEFAULT="quiet usb_storage.quirks=13fd:0840:i"
|
||||
```
|
||||
|
||||
Then rebuild the GRUB config:
|
||||
|
||||
```bash
|
||||
sudo update-grub
|
||||
```
|
||||
|
||||
For systemd-boot, add to your loader entry or `/etc/kernel/cmdline`:
|
||||
|
||||
```
|
||||
usb_storage.quirks=13fd:0840:i
|
||||
```
|
||||
|
||||
Multiple devices can be separated by commas:
|
||||
|
||||
```
|
||||
usb_storage.quirks=13fd:0840:i,174c:5106:u
|
||||
```
|
||||
|
||||
### Recommended Bridges
|
||||
|
||||
If you are buying a USB-SATA adapter or enclosure for optical drive use:
|
||||
|
||||
| Bridge | USB IDs | Notes |
|
||||
|--------|---------|-------|
|
||||
| ASMedia ASM1153 | `174c:1153` | Reliable. Widely available in SATA-USB 3.0 cables. |
|
||||
| JMicron JMS578 | `152d:0578` | Good firmware. Supports UASP. |
|
||||
| Icy Box IB-AC640-C3 | N/A | Uses a known-good bridge internally. Plug-and-play. |
|
||||
|
||||
Avoid any enclosure or adapter listing an Initio chipset.
|
||||
|
||||
### Best Solution: Direct SATA
|
||||
|
||||
Connect your optical drive directly to a motherboard SATA port. This eliminates the USB-SATA bridge entirely and is the most reliable configuration:
|
||||
|
||||
- No USB protocol overhead or translation errors.
|
||||
- No bridge firmware bugs.
|
||||
- No port resets or re-enumeration.
|
||||
- Full SATA error recovery handled natively by the kernel's libata driver.
|
||||
- Sustained read speeds are limited only by the drive, not the USB bus.
|
||||
|
||||
If your machine has a free SATA port, use it.
|
||||
|
||||
---
|
||||
|
||||
## 2. Damaged Disc Handling
|
||||
|
||||
### Symptoms
|
||||
|
||||
- SCSI MEDIUM ERROR (sense key 0x03) at specific LBAs. `dmesg` shows `sr X:0:0:0: [srY] Unrecoverable read error` or similar.
|
||||
- Read speed drops to near zero when approaching a damaged area.
|
||||
- The drive makes audible retrying noises (laser repositioning, spindle speed changes).
|
||||
- On USB-connected drives: the bridge crashes (see section 1 above) when the drive returns the error.
|
||||
|
||||
### How freemkv Handles This
|
||||
|
||||
freemkv uses a three-layer recovery model. See [`docs/rip-recovery.md`](docs/rip-recovery.md) for full details.
|
||||
|
||||
- **Pass 1 (Disc::copy):** Fast sweep with 64 KB reads. On failure, zero-fills the block and skips forward. Writes a ddrescue-format mapfile for later retry.
|
||||
- **Pass 2+ (Disc::patch):** Targeted re-reads of bad ranges with a long 30-second timeout per CDB. The drive firmware performs its own ECC and laser power retries within that window.
|
||||
- **In-stream (DiscStream):** Adaptive batch halving -- reduces request size on failure to isolate bad sectors within a larger block.
|
||||
|
||||
This means a disc with some bad sectors will still produce a usable ISO. The damaged areas are zero-filled in pass 1 and retried in subsequent passes. Structure-protected sectors (deliberate unreadable regions from copy protection) will never yield, which is expected.
|
||||
|
||||
### The Drive Taint Issue (LG BU40N)
|
||||
|
||||
Some drives, notably the LG BU40N, exhibit a "taint" behavior after encountering MEDIUM ERRORs:
|
||||
|
||||
1. The drive hits a damaged sector and returns a MEDIUM ERROR.
|
||||
2. From that point forward, **all subsequent reads fail** -- even reads to sectors that were previously successful.
|
||||
3. The only recovery is to physically unplug and reconnect the drive (or power-cycle it).
|
||||
|
||||
This is not a freemkv bug. It is a drive firmware behavior triggered by the interaction between the drive's internal error recovery and the USB-SATA bridge's handling of the error response. The drive firmware enters a degraded state that it does not recover from without a power cycle.
|
||||
|
||||
Workarounds:
|
||||
|
||||
- **Use a direct SATA connection.** This eliminates the bridge interaction that triggers the taint.
|
||||
- **Use a different bridge.** The ASM1153 and JMS578 are less likely to trigger this behavior.
|
||||
- **Accept the partial ISO.** freemkv's skip-forward recovery will zero-fill the unreadable blocks and continue. The resulting ISO may be playable with minor glitches in the affected areas.
|
||||
- **Physical replug between retry passes.** If running multi-pass patch, replug the drive between passes to clear the taint state.
|
||||
|
||||
freemkv deliberately does not attempt inline SCSI resets or eject cycles to recover from this state, because those operations were found to make the problem worse on affected hardware (see the design rationale in [`docs/rip-recovery.md`](docs/rip-recovery.md)).
|
||||
|
||||
---
|
||||
|
||||
## 3. Drive Not Detected
|
||||
|
||||
### Check Hardware Visibility
|
||||
|
||||
```bash
|
||||
lsusb
|
||||
```
|
||||
|
||||
Verify the drive appears in the USB device list. If it does not show up, the drive is not visible to the host at all -- check cables, power, and USB port.
|
||||
|
||||
```bash
|
||||
ls /dev/sg*
|
||||
```
|
||||
|
||||
On Linux, optical drives appear as `/dev/sg*` devices (the SCSI Generic interface). freemkv uses `/dev/sg*`, not `/dev/sr*`. If `lsusb` shows the device but no `/dev/sg*` entry exists, the `sg` kernel module may not be loaded:
|
||||
|
||||
```bash
|
||||
sudo modprobe sg
|
||||
```
|
||||
|
||||
### Check Kernel Messages
|
||||
|
||||
```bash
|
||||
dmesg | grep -i usb | tail -30
|
||||
dmesg | grep -i sg | tail -10
|
||||
```
|
||||
|
||||
Look for:
|
||||
- USB enumeration errors or failed port resets.
|
||||
- `sg_add` messages confirming the sg device was registered.
|
||||
- Permission denied or access errors.
|
||||
|
||||
### Permission Issues
|
||||
|
||||
On most Linux distributions, `/dev/sg*` devices are owned by `root:disk` or `root:cdrom` with restricted permissions. Running freemkv as an unprivileged user will fail with permission errors.
|
||||
|
||||
Options:
|
||||
|
||||
- Add your user to the appropriate group:
|
||||
|
||||
```bash
|
||||
sudo usermod -aG disk $USER
|
||||
```
|
||||
|
||||
Then log out and back in for the change to take effect. On some distributions the group is `cdrom` or `optical` instead of `disk`.
|
||||
|
||||
- Run with elevated privileges:
|
||||
|
||||
```bash
|
||||
sudo freemkv ...
|
||||
```
|
||||
|
||||
- Install a udev rule for persistent per-device permissions. Create `/etc/udev/rules.d/99-sg-optical.rules`:
|
||||
|
||||
```
|
||||
SUBSYSTEM=="scsi_generic", ATTRS{type}=="5", MODE="0666"
|
||||
```
|
||||
|
||||
Then reload udev rules:
|
||||
|
||||
```bash
|
||||
sudo udevadm control --reload-rules && sudo udevadm trigger
|
||||
```
|
||||
|
||||
### Spin-Up Delay
|
||||
|
||||
Optical drives take 30-60 seconds to spin up and become ready after hot-plug or disc insertion. During this window, SCSI commands may return NOT READY or timeout.
|
||||
|
||||
freemkv's `Drive::wait_ready()` handles this automatically by polling with TEST UNIT READY until the drive responds. If you are writing your own code using the library, always call `wait_ready()` before `init()`:
|
||||
|
||||
```rust
|
||||
let mut drive = Drive::open(Path::new("/dev/sg4"))?;
|
||||
drive.wait_ready()?; // blocks until disc is ready, up to 30s
|
||||
drive.init()?;
|
||||
```
|
||||
|
||||
If the drive was just plugged in, wait a full minute before concluding it is not detected.
|
||||
|
||||
---
|
||||
|
||||
## 4. How to Identify Your USB-SATA Bridge
|
||||
|
||||
If you are experiencing the issues described in section 1, you need to know which bridge chipset your adapter or enclosure uses.
|
||||
|
||||
### Step 1: Find the Device
|
||||
|
||||
```bash
|
||||
lsusb
|
||||
```
|
||||
|
||||
Look for entries matching your drive or enclosure. Bridges may appear under their own manufacturer name or as a generic SATA device. Common examples:
|
||||
|
||||
```
|
||||
Bus 002 Device 005: ID 13fd:0840 Initio Corporation
|
||||
Bus 002 Device 006: ID 174c:1153 ASMedia Technology Inc. ASM1153
|
||||
Bus 002 Device 007: ID 152d:0578 JMicron Technology Corp. JMS578
|
||||
```
|
||||
|
||||
### Step 2: Match the IDs
|
||||
|
||||
| Vendor | Product ID | Chipset | Status |
|
||||
|--------|-----------|---------|--------|
|
||||
| `13fd` | `3609` | Initio INIC-3609 | Affected. Apply quirk. |
|
||||
| `13fd` | `3940` | Initio INIC-3619 | Affected. Apply quirk. |
|
||||
| `13fd` | `0840` | Initio INIC-3069 | Affected. Apply quirk. |
|
||||
| `174c` | `5106` | ASMedia ASM1051 | Affected (early firmware). Apply quirk. |
|
||||
| `174c` | `1153` | ASMedia ASM1153 | Good. No quirk needed. |
|
||||
| `152d` | `0561` | JMicron JMB36x | Affected (some firmware). Apply quirk if issues occur. |
|
||||
| `152d` | `0578` | JMicron JMS578 | Good. No quirk needed. |
|
||||
|
||||
### Step 3: Check dmesg for the Bridge Name
|
||||
|
||||
```bash
|
||||
dmesg | grep -i "usb-storage\|uas\|initio\|asmedia\|jmicron"
|
||||
```
|
||||
|
||||
This often reveals the bridge chipset even when `lsusb` shows a generic name.
|
||||
|
||||
### Step 4: If the Enclosure Is Sealed
|
||||
|
||||
Many external drive enclosures (Vantec NexStar, Sabrent, OWC, etc.) do not advertise the bridge chipset on the packaging. In this case:
|
||||
|
||||
1. Check `lsusb` while the enclosure is connected.
|
||||
2. Search the vendor:product ID online -- there are community-maintained lists of which chipsets popular enclosures use.
|
||||
3. If you cannot determine the chipset and are experiencing bridge crashes, assume it is an Initio and apply the quirk with its IDs.
|
||||
4. The definitive test: connect the bare drive to a motherboard SATA port. If the problems disappear, the bridge was the cause.
|
||||
|
||||
---
|
||||
|
||||
## 5. General Debugging Checklist
|
||||
|
||||
When something goes wrong during a rip, gather this information before filing an issue:
|
||||
|
||||
1. **freemkv version:** `freemkv --version` or the crate version in `Cargo.toml`.
|
||||
2. **Drive model:** from the drive label, or from `freemkv info`.
|
||||
3. **Connection type:** USB (with bridge chipset if known) or direct SATA.
|
||||
4. **Operating system and kernel:** `uname -a`.
|
||||
5. **Kernel messages during the failure:** `dmesg | tail -50` immediately after the crash.
|
||||
6. **SCSI device:** which `/dev/sg*` the drive was on, and whether it changed after the failure.
|
||||
7. **The disc:** title, format (BD/DVD/UHD), condition.
|
||||
|
||||
Include all of the above in bug reports. SCSI transport errors that resolve with the `US_FL_IGNORE_RESIDUE` quirk or by switching to direct SATA are bridge firmware bugs, not freemkv bugs.
|
||||
@@ -8,13 +8,12 @@ fn main() {
|
||||
let device = std::env::args()
|
||||
.skip(1)
|
||||
.find(|a| !a.starts_with('-'))
|
||||
.unwrap_or_else(|| {
|
||||
let drives = libfreemkv::find_drives();
|
||||
if drives.is_empty() {
|
||||
.unwrap_or_else(|| match libfreemkv::find_drive() {
|
||||
Some(d) => d.device_path().to_string(),
|
||||
None => {
|
||||
eprintln!("No drives found");
|
||||
std::process::exit(1);
|
||||
}
|
||||
drives[0].device_path().to_string()
|
||||
});
|
||||
|
||||
let mut drive = Drive::open(Path::new(&device)).unwrap_or_else(|e| {
|
||||
|
||||
@@ -3,5 +3,34 @@ fn main() {
|
||||
if target == "macos" {
|
||||
println!("cargo:rustc-link-lib=framework=IOKit");
|
||||
println!("cargo:rustc-link-lib=framework=CoreFoundation");
|
||||
|
||||
let out_dir = std::env::var("OUT_DIR").unwrap();
|
||||
let obj = format!("{out_dir}/macos_shim.o");
|
||||
let lib = format!("{out_dir}/libmacos_scsi.a");
|
||||
|
||||
std::process::Command::new("cc")
|
||||
.args([
|
||||
"-c",
|
||||
"src/scsi/macos_shim.c",
|
||||
"-o",
|
||||
&obj,
|
||||
"-framework",
|
||||
"IOKit",
|
||||
"-framework",
|
||||
"CoreFoundation",
|
||||
"-Wall",
|
||||
"-O2",
|
||||
])
|
||||
.status()
|
||||
.expect("failed to compile macos_shim.c");
|
||||
|
||||
std::process::Command::new("ar")
|
||||
.args(["rcs", &lib, &obj])
|
||||
.status()
|
||||
.expect("failed to create static lib");
|
||||
|
||||
println!("cargo:rustc-link-search=native={out_dir}");
|
||||
println!("cargo:rustc-link-lib=static=macos_scsi");
|
||||
println!("cargo:rerun-if-changed=src/scsi/macos_shim.c");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ Technical documentation for [libfreemkv](https://github.com/freemkv/libfreemkv),
|
||||
|----------|---------------|
|
||||
| [Architecture](architecture.md) | Module map, design principles, error codes, platform support |
|
||||
| [Drive Access](drive-access.md) | Drive, SCSI transport, profiles, unlock, why raw mode is needed |
|
||||
| [Rip Recovery](rip-recovery.md) | Three-layer recovery model: Disc::patch, single-shot Drive::read, DiscStream batch halving |
|
||||
| [AACS Encryption](aacs.md) | Key resolution (4 paths), content decryption, bus encryption, SCSI handshake |
|
||||
| [UDF Filesystem](udf.md) | UDF 2.50 with metadata partitions, pointer chain, how files are read from disc |
|
||||
| [MPLS Playlists](mpls.md) | Playlist format, play items, STN stream table, coding types |
|
||||
|
||||
+72
-40
@@ -44,35 +44,30 @@ while let Ok(Some(frame)) = input.read() {
|
||||
output.finish()?;
|
||||
```
|
||||
|
||||
The `pes::Stream` trait:
|
||||
The `FrameSource` and `FrameSink` traits — direction is type-checked, so
|
||||
calling `read()` on a write-only sink (or `write()` on a read-only source)
|
||||
is a compile error rather than a runtime fault:
|
||||
|
||||
```rust
|
||||
pub trait Stream {
|
||||
fn read(&mut self) -> io::Result<Option<PesFrame>>;
|
||||
fn write(&mut self, frame: &PesFrame) -> io::Result<()>;
|
||||
fn finish(&mut self) -> io::Result<()>;
|
||||
pub trait FrameSource: Send {
|
||||
fn read(&mut self) -> Result<Option<PesFrame>, Error>;
|
||||
fn info(&self) -> &DiscTitle;
|
||||
fn codec_private(&self, track: usize) -> Option<Vec<u8>>;
|
||||
fn headers_ready(&self) -> bool;
|
||||
fn codec_private(&self, track: usize) -> Option<Vec<u8>> { None }
|
||||
fn headers_ready(&self) -> bool { true }
|
||||
}
|
||||
```
|
||||
|
||||
## IOStream (byte-level API)
|
||||
|
||||
For raw byte copies (disc→ISO, resume, benchmarks). Lower level than PES.
|
||||
|
||||
```rust
|
||||
let opts = InputOptions::default();
|
||||
let mut input = open_input("iso://Disc.iso", &opts)?;
|
||||
let mut output = open_output("mkv://Movie.mkv", input.info())?;
|
||||
io::copy(&mut *input, &mut *output)?;
|
||||
output.finish()?;
|
||||
pub trait FrameSink: Send {
|
||||
fn write(&mut self, frame: &PesFrame) -> Result<(), Error>;
|
||||
fn finish(self: Box<Self>) -> Result<(), Error>;
|
||||
fn info(&self) -> &DiscTitle;
|
||||
}
|
||||
```
|
||||
|
||||
## Streams
|
||||
|
||||
All streams implement `IOStream` (byte-level) and/or `pes::Stream` (frame-level).
|
||||
URL-based resolvers open any stream by string.
|
||||
All streams implement `FrameSource` (read) and/or `FrameSink` (write); the
|
||||
directional split prevents runtime "wrong-direction" errors. URL-based
|
||||
resolvers open any stream by string.
|
||||
|
||||
| Stream | Input | Output | URL | Transport |
|
||||
|--------|-------|--------|-----|-----------|
|
||||
@@ -87,21 +82,14 @@ URL-based resolvers open any stream by string.
|
||||
All URLs require a `scheme://path` format. Bare paths are rejected.
|
||||
|
||||
```rust
|
||||
// PES pipeline (frame-level)
|
||||
// PES pipeline (frame-level) — input() returns Box<dyn FrameSource>,
|
||||
// output() returns Box<dyn FrameSink>.
|
||||
let input = libfreemkv::input("disc:///dev/sg4", &opts)?; // DiscStream
|
||||
let input = libfreemkv::input("iso://Dune.iso", &opts)?; // IsoStream
|
||||
let output = libfreemkv::output("mkv://Dune.mkv", &title)?; // MkvOutputStream
|
||||
let output = libfreemkv::output("m2ts://Dune.m2ts", &title)?; // M2tsOutputStream
|
||||
let output = libfreemkv::output("network://10.0.0.1:9000", &title)?; // NetworkOutputStream
|
||||
let output = libfreemkv::output("null://", &title)?; // NullOutputStream
|
||||
|
||||
// IOStream (byte-level)
|
||||
let input = open_input("disc://", &opts)?; // DiscStream
|
||||
let input = open_input("iso://Dune.iso", &opts)?; // IsoStream
|
||||
let output = open_output("iso://Copy.iso", &meta)?; // IsoStream (write)
|
||||
let output = open_output("mkv://Dune.mkv", &meta)?; // MkvStream
|
||||
let output = open_output("m2ts://Dune.m2ts", &meta)?; // M2tsStream
|
||||
let output = open_output("null://", &meta)?; // NullStream
|
||||
```
|
||||
|
||||
### FMKV Metadata Header
|
||||
@@ -126,15 +114,45 @@ pub struct Event {
|
||||
}
|
||||
|
||||
pub enum EventKind {
|
||||
// Init / scan
|
||||
DriveOpened { device: String },
|
||||
DriveReady,
|
||||
InitComplete { success: bool },
|
||||
ProbeComplete { success: bool },
|
||||
ScanComplete { titles: usize },
|
||||
|
||||
// Read pipeline
|
||||
BytesRead { bytes: u64, total: u64 },
|
||||
ReadError { sector: u64, error: Error },
|
||||
Retry { attempt: u32 },
|
||||
SpeedChange { speed_kbs: u16 },
|
||||
ExtentStart { index: usize, start_sector: u64, sector_count: u64 },
|
||||
SectorSkipped { sector: u64 },
|
||||
BatchSizeChanged { new_size: u16, reason: BatchSizeReason },
|
||||
Complete { bytes: u64, errors: u32 },
|
||||
|
||||
// Kept for forward-compat; not emitted in 0.13.6+
|
||||
Retry { attempt: u32 },
|
||||
SectorRecovered { sector: u64 },
|
||||
}
|
||||
```
|
||||
|
||||
Emission notes:
|
||||
|
||||
- `BytesRead { bytes, total }` is emitted from `DiscStream::fill_extents`
|
||||
after each successful sector read. `bytes` is the cumulative running
|
||||
total; `total` is the precomputed extent sum (0 if unknown).
|
||||
- `SpeedChange` is emitted from the public `Drive::set_speed` API path.
|
||||
It is no longer emitted from a recovery hot loop (recovery loop removed
|
||||
in 0.13.6).
|
||||
- `BatchSizeChanged` fires from the `DiscStream` adaptive sizer on shrink
|
||||
(read failed at a larger size) and on probe-up (clean-read streak hit
|
||||
the threshold). Consumers use it to display a "recovering" state
|
||||
distinct from "ripping normally".
|
||||
- `Retry` and `SectorRecovered` are NOT emitted in 0.13.6+. They were
|
||||
tied to the inline `Drive::read` recovery phases that were removed; the
|
||||
variants are kept for forward compatibility so consumers' match arms
|
||||
don't need conditional compilation.
|
||||
|
||||
Events report what happened. App decides what to do. GUI shows a dialog. CLI
|
||||
prints a line. Server logs to file.
|
||||
|
||||
@@ -145,20 +163,34 @@ libfreemkv/src/
|
||||
├── lib.rs Public exports
|
||||
├── error.rs Error codes (no English)
|
||||
├── event.rs Event types for callbacks
|
||||
├── drive/ Drive (open, init, read with recovery)
|
||||
│ ├── mod.rs Drive struct, init, read, reset, eject
|
||||
├── halt.rs Halt cancellation token (Arc<AtomicBool> wrapper)
|
||||
├── io/ Pipeline + WritebackFile primitives
|
||||
│ ├── mod.rs Re-exports WritebackFile, Pipeline, Sink, Flow
|
||||
│ ├── pipeline.rs Generic Pipeline<I, R> + Sink trait
|
||||
│ ├── writeback_file.rs WritebackFile (was crate::io::Writer)
|
||||
│ └── writeback.rs sync_file_range pipeline
|
||||
├── drive/ Drive (open, init, single-shot read)
|
||||
│ ├── mod.rs Drive struct, init, read (single-shot), reset, eject
|
||||
│ ├── capture.rs Drive profile capture for contribution
|
||||
│ ├── linux.rs Linux drive discovery
|
||||
│ ├── macos.rs macOS drive discovery
|
||||
│ └── windows.rs Windows drive discovery
|
||||
├── disc/ Disc (scan, titles, AACS setup)
|
||||
├── disc/ Disc (scan, titles, AACS setup, sweep, patch)
|
||||
│ ├── mod.rs Disc struct, scan, titles, formats
|
||||
│ ├── sweep.rs Disc::sweep (Pass 1 forward sweep)
|
||||
│ ├── patch.rs Disc::patch (Pass N retry over mapfile)
|
||||
│ ├── mapfile.rs ddrescue-format mapfile
|
||||
│ └── read_error.rs ReadCtx / ReadAction state machine
|
||||
├── scsi/ SCSI transport (Linux SG_IO, macOS IOKit, Windows SPTI)
|
||||
├── platform/ Drive unlock (MT1959 A/B)
|
||||
├── aacs/ AACS decryption (handshake, keys, keydb, decrypt)
|
||||
├── css/ DVD CSS cipher
|
||||
├── decrypt.rs Unified decrypt dispatcher (AACS/CSS/None)
|
||||
├── pes.rs PES frame types, Stream trait
|
||||
├── sector.rs SectorReader trait
|
||||
├── pes.rs PES frame types, FrameSource / FrameSink traits
|
||||
├── sector/ Sector I/O (was sector.rs in 0.17)
|
||||
│ ├── mod.rs SectorSource, SectorSink traits
|
||||
│ ├── file.rs FileSectorSource, FileSectorSink (ISO-backed)
|
||||
│ └── decrypting.rs DecryptingSectorSource decorator
|
||||
├── udf.rs UDF 2.50 filesystem parser
|
||||
├── mpls.rs MPLS playlist parser
|
||||
├── clpi.rs CLPI clip info parser
|
||||
@@ -169,15 +201,15 @@ libfreemkv/src/
|
||||
├── profile.rs Bundled drive profiles
|
||||
├── speed.rs DriveSpeed enum
|
||||
├── mux/
|
||||
│ ├── mod.rs IOStream trait, public exports
|
||||
│ ├── resolve.rs URL parser + open_input/open_output + input/output
|
||||
│ ├── mod.rs Public mux exports
|
||||
│ ├── resolve.rs URL parser + input/output (Box<dyn FrameSource/Sink>)
|
||||
│ ├── meta.rs FMKV header format
|
||||
│ ├── disc.rs DiscStream (optical drive → PES)
|
||||
│ ├── iso.rs IsoStream (ISO image read/write)
|
||||
│ ├── iso.rs IsoStream (ISO image read)
|
||||
│ ├── isowriter.rs ISO image writer (UDF, AVDP, multi-extent)
|
||||
│ ├── mkvstream.rs MkvStream (bidirectional Matroska, IOStream)
|
||||
│ ├── mkvstream.rs MkvStream (bidirectional Matroska)
|
||||
│ ├── mkvout.rs MkvOutputStream (PES → MKV)
|
||||
│ ├── m2ts.rs M2tsStream (BD-TS, IOStream)
|
||||
│ ├── m2ts.rs M2tsStream (BD-TS)
|
||||
│ ├── pesout.rs PES output streams (M2ts, Network, Stdio, Null)
|
||||
│ ├── network.rs NetworkStream (TCP + FMKV header)
|
||||
│ ├── stdio.rs StdioStream (stdin/stdout pipe)
|
||||
|
||||
+15
-4
@@ -41,7 +41,7 @@ AACS keys are derived internally, and all SCSI communication is handled in-proce
|
||||
libfreemkv (lib.rs)
|
||||
│
|
||||
├── Drive Access
|
||||
│ ├── drive Drive — open, identify, init, unlock, read (with recovery)
|
||||
│ ├── drive Drive — open, identify, init, unlock, single-shot read
|
||||
│ ├── scsi ScsiTransport trait + platform backends (sg async, IOKit, SPTI)
|
||||
│ ├── platform/ Platform trait — per-chipset command handlers
|
||||
│ │ └── mt1959 MediaTek MT1959 driver (LG, ASUS, HP)
|
||||
@@ -65,8 +65,12 @@ libfreemkv (lib.rs)
|
||||
│
|
||||
├── Streaming
|
||||
│ ├── mux/ Stream implementations (Disc, ISO, MKV, M2TS, Network, Stdio, Null)
|
||||
│ ├── pes PES frame types, Stream trait (read/write frames)
|
||||
│ └── sector SectorReader trait — abstracts disc vs ISO vs file
|
||||
│ ├── pes PES frame types; FrameSource / FrameSink direction-typed traits
|
||||
│ └── sector/ SectorSource / SectorSink traits, FileSector{Source,Sink}, DecryptingSectorSource
|
||||
│
|
||||
├── I/O Primitives
|
||||
│ ├── halt Halt cancellation token (one Arc<AtomicBool>, cloneable)
|
||||
│ └── io/ Pipeline<I, R> + Sink trait + WritebackFile (bounded-cache writer)
|
||||
│
|
||||
├── Support
|
||||
│ ├── keydb KEYDB.cfg download, parse, verify, save
|
||||
@@ -93,10 +97,17 @@ Drive::open(Path::new("/dev/sg4"))
|
||||
After open:
|
||||
- `init()` -- unlock + firmware upload + speed calibration
|
||||
- `probe_disc()` -- probe disc surface for optimal speeds
|
||||
- `read(lba, count, buf)` -- single read method with built-in error recovery
|
||||
- `read(lba, count, buf, recovery)` -- single-shot read; `recovery` only selects the per-CDB timeout (1.5 s vs. 30 s)
|
||||
- `wait_ready()` -- wait for disc insertion
|
||||
- `eject()` -- eject tray
|
||||
|
||||
Recovery is layered above `Drive::read`, not inside it. Layer 1
|
||||
(`Disc::patch`) handles bad-range retry by replaying the ddrescue mapfile.
|
||||
Layer 3 (`DiscStream::fill_extents` adaptive batch sizer) handles in-loop
|
||||
request-size adaptation. Inline recovery (gentle retry → SCSI reset → retry)
|
||||
was removed in 0.13.6 — see [`rip-recovery.md`](rip-recovery.md) and
|
||||
the stop-wedge postmortem (2026-04-25).
|
||||
|
||||
---
|
||||
|
||||
## Disc Scanning Flow
|
||||
|
||||
+12
-6
@@ -77,7 +77,9 @@ Insert disc
|
||||
10. Stream content (mux/disc.rs → DiscStream)
|
||||
│ Read sectors → decrypt → TS demux → PES frames
|
||||
│ Or: read sectors → decrypt → raw bytes (for ISO output)
|
||||
│ Drive::read() handles error recovery (min speed → reset → retry)
|
||||
│ Drive::read() is single-shot. DiscStream::fill_extents adapts the
|
||||
│ batch size on failure (halve / probe-up). Bad-range retry is layer
|
||||
│ 1 above this — Disc::patch re-runs against the mapfile.
|
||||
│
|
||||
▼
|
||||
PES frames → output stream (MKV, M2TS, network, etc.)
|
||||
@@ -95,7 +97,9 @@ drive.probe_disc()?;
|
||||
// Scan disc (UDF + playlists + AACS — all automatic)
|
||||
let disc = Disc::scan(&mut drive, &ScanOptions::default())?;
|
||||
|
||||
// Stream pipeline — PES frames from any source to any output
|
||||
// Stream pipeline — PES frames from any source to any output.
|
||||
// 0.18: input() returns Box<dyn FrameSource>, output() returns Box<dyn FrameSink>;
|
||||
// direction is type-checked, so calling .write() on an input is a compile error.
|
||||
let opts = InputOptions::default();
|
||||
let mut input = libfreemkv::input("disc:///dev/sg4", &opts)?;
|
||||
let title = input.info().clone();
|
||||
@@ -110,7 +114,7 @@ output.finish()?;
|
||||
|
||||
| Module | Doc | Purpose |
|
||||
|--------|-----|---------|
|
||||
| drive/ | [drive-access.md](drive-access.md) | Open, identify, init, unlock, read (with recovery) |
|
||||
| drive/ | [drive-access.md](drive-access.md) | Open, identify, init, unlock, single-shot read |
|
||||
| scsi/ | [drive-access.md](drive-access.md) | Platform SCSI transport (Linux, macOS, Windows) |
|
||||
| udf.rs | [udf.md](udf.md) | UDF 2.50 filesystem |
|
||||
| mpls.rs | [mpls.md](mpls.md) | MPLS playlists + STN streams |
|
||||
@@ -119,11 +123,13 @@ output.finish()?;
|
||||
| aacs/ | [aacs.md](aacs.md) | Key resolution + content decrypt + bus handshake |
|
||||
| css/ | -- | DVD CSS cipher |
|
||||
| decrypt.rs | -- | Unified decrypt dispatcher (AACS/CSS/None) |
|
||||
| disc/ | -- | High-level scan + read API |
|
||||
| disc/ | [rip-recovery.md](rip-recovery.md) | Disc::scan + Disc::sweep + Disc::patch + mapfile |
|
||||
| labels/ | -- | BD-J stream labels (5 format parsers) |
|
||||
| mux/ | -- | Stream implementations (7 stream types) |
|
||||
| pes.rs | -- | PES frame types + Stream trait |
|
||||
| sector.rs | -- | SectorReader trait |
|
||||
| pes.rs | -- | PES frame types + FrameSource / FrameSink traits |
|
||||
| sector/ | -- | SectorSource / SectorSink + DecryptingSectorSource decorator |
|
||||
| io/ | -- | Pipeline<I, R> + Sink trait + WritebackFile |
|
||||
| halt.rs | -- | Halt cancellation token |
|
||||
| keydb.rs | -- | KEYDB download, parse, save |
|
||||
| error.rs | -- | Error codes (E1xxx-E8xxx) |
|
||||
| event.rs | -- | Drive event system |
|
||||
|
||||
+48
-16
@@ -26,8 +26,8 @@ platform driver. The drive is ready for `wait_ready()` and `init()`.
|
||||
| `wait_ready()` | Wait for disc insertion (30s timeout, TUR polling) |
|
||||
| `init()` | Firmware upload + unlock + speed calibration |
|
||||
| `probe_disc()` | Probe disc surface for optimal speeds |
|
||||
| `read(lba, count, buf)` | Read sectors with built-in error recovery |
|
||||
| `reset()` | Close/reopen device, TUR, escalate if needed |
|
||||
| `read(lba, count, buf, recovery)` | Read sectors. Single-shot — no inline retries or reset. |
|
||||
| `reset()` | Eject-cycle escape hatch. Caller-invoked only; not on the read path. |
|
||||
| `lock_tray()` | Prevent tray ejection during rip |
|
||||
| `unlock_tray()` | Allow tray ejection (also runs on Drop) |
|
||||
| `eject()` | Eject disc tray |
|
||||
@@ -44,22 +44,33 @@ platform driver. The drive is ready for `wait_ready()` and `init()`.
|
||||
3. Speed calibration after unlock
|
||||
4. Max 3 attempts before giving up
|
||||
|
||||
### read() with Recovery
|
||||
### read() — single-shot
|
||||
|
||||
`Drive::read(lba, count, buf, recovery)` is the single read method. The
|
||||
`recovery` parameter controls whether to attempt multi-phase recovery on
|
||||
failure or return immediately (used by DiscStream's binary search for
|
||||
single-sector probes).
|
||||
`Drive::read(lba, count, buf, recovery)` is the single read method. It issues
|
||||
exactly one READ(10) CDB and returns the result. The `recovery` parameter only
|
||||
selects the per-CDB timeout:
|
||||
|
||||
On error with `recovery = true`:
|
||||
| `recovery` | Timeout | Used by |
|
||||
|------------|----------|------------------------------------------|
|
||||
| `false` | 1.5 s | `Disc::sweep` fast skip-forward pass, `DiscStream::fill_extents` |
|
||||
| `true` | 30 s | `Disc::patch` retry pass over the mapfile |
|
||||
|
||||
1. **Phase 1 — gentle retry (5 attempts):** set min speed, sleep 30s, retry.
|
||||
Each retry has a hard wall-clock timeout via async SG_IO.
|
||||
2. **Phase 2 — fresh start:** close transport, reset device, reopen, reinit.
|
||||
3. **Phase 3 — gentle retry on fresh connection (5 attempts).**
|
||||
4. If all fail: return `Err(DiscRead)`. DiscStream handles it (binary search,
|
||||
skip, zero-fill).
|
||||
5. Stay at min speed for 500 MB after any recovery (recovery window).
|
||||
On any SCSI failure or timeout, `read` returns `Err(DiscRead)` immediately.
|
||||
There are no inline retries, no SCSI reset, no Phase 1/2/3 escalation.
|
||||
|
||||
Recovery is layered above `Drive::read`:
|
||||
|
||||
- **Layer 1 — `Disc::patch`** loops over the ddrescue mapfile and re-issues
|
||||
`read(.., recovery=true)` against each non-`+` range.
|
||||
- **Layer 3 — `DiscStream::fill_extents`** halves the request size on
|
||||
failure, retries at the same LBA, and probes back up on a clean-read
|
||||
streak.
|
||||
|
||||
Inline recovery (5× gentle retry → close + reset + reopen → 5× more) was
|
||||
removed in 0.13.6. See the stop-wedge postmortem (2026-04-25)
|
||||
for rationale: the inline reset wedged drive firmware on the LG BU40N (Initio
|
||||
USB-SATA bridge) without ever recovering a sector. See
|
||||
[`rip-recovery.md`](rip-recovery.md) for the full three-layer model.
|
||||
|
||||
---
|
||||
|
||||
@@ -88,7 +99,6 @@ descriptors or calls ioctls outside of a `ScsiTransport` implementation.
|
||||
|----------|---------------|--------|
|
||||
| Linux | `SgIoTransport` — async `write`/`poll`/`read` on `/dev/sg*` | `/dev/sg*` |
|
||||
| macOS | `MacScsiTransport` — IOKit SCSITask | IOKit service |
|
||||
| Windows | `WindowsScsiTransport` — SPTI | `\\.\CdRomN` |
|
||||
|
||||
The Linux backend uses the sg driver's asynchronous interface: `write()` submits
|
||||
the command, `poll()` waits with an enforceable wall-clock timeout, `read()`
|
||||
@@ -96,9 +106,31 @@ retrieves the result. If `poll()` times out, the fd is abandoned (closed in a
|
||||
background thread) and a fresh fd opened — the kernel's USB error recovery
|
||||
cannot block us. Opens with `O_RDWR | O_NONBLOCK`.
|
||||
|
||||
The macOS backend uses a C shim (`macos_shim.c`) for IOKit exclusive access.
|
||||
The shim handles:
|
||||
1. `shim_open_exclusive(bsd_name)` — unmounts the target device via `diskutil`,
|
||||
then walks the IOKit registry to find the `IOBDServices` matching the
|
||||
requested BSD name (IOBDServices → IOBDBlockStorageDriver → IOMedia → "BSD Name"),
|
||||
then creates MMCDeviceInterface → SCSITaskDeviceInterface → ObtainExclusiveAccess.
|
||||
2. `shim_list_drives()` — registry-based enumeration with zero SCSI, zero exclusive
|
||||
access, zero unmounts. Reads IOBDServices "Device Characteristics" for
|
||||
vendor/model/firmware and child IOMedia "BSD Name" for the device path.
|
||||
3. `shim_execute()` / `shim_close()` — raw CDB dispatch and cleanup.
|
||||
|
||||
On non-zero SCSI status, the transport parses sense key from the sense buffer
|
||||
and returns `Error::ScsiError`.
|
||||
|
||||
`SgIoTransport::reset` (Linux) does pure userspace state cleanup: an open +
|
||||
close pair to make the kernel cancel any SG_IO commands queued against a
|
||||
previous fd, a 2 s sleep to let the kernel finish that cancellation, then a
|
||||
fresh fd to send ALLOW MEDIUM REMOVAL to clear any stale tray lock. It does
|
||||
NOT issue `SG_SCSI_RESET` or escalate via STOP+START UNIT. Both were tried
|
||||
in 0.13.0–0.13.5 against the LG BU40N (Initio USB-SATA bridge); both failed
|
||||
to recover wedged drives and made the wedge worse. The macOS reset (which
|
||||
had been a no-op) was removed entirely in 0.13.6, and the top-level
|
||||
`scsi::reset()` / `reset_with_timeout()` / `reset_blocking()` wrappers were
|
||||
removed at the same time (no callers).
|
||||
|
||||
### CDB Builders
|
||||
|
||||
The `scsi` module provides platform-agnostic CDB constructors:
|
||||
|
||||
+121
-47
@@ -1,22 +1,37 @@
|
||||
# Rip recovery — multi-pass architecture
|
||||
# Rip recovery — three-layer architecture
|
||||
|
||||
`libfreemkv` supports a two-stage rip model for damaged or protection-bearing
|
||||
discs: a fast forward sweep that tolerates read failures, followed by targeted
|
||||
`libfreemkv` supports a multi-stage rip model for damaged or protection-bearing
|
||||
discs: a fast forward sweep that tolerates read failures, in-loop request-size
|
||||
adaptation that survives transient drive trouble without bailing, and targeted
|
||||
retry passes against a persistent bad-range map. The stream pipeline
|
||||
(`DiscStream` + `input`/`output`) operates against the resulting ISO image, so
|
||||
the mux stage never touches the drive.
|
||||
|
||||
Three primitives compose the flow:
|
||||
Recovery is layered cleanly. Each layer has one responsibility and does not
|
||||
reach into the others.
|
||||
|
||||
| Layer | Where it lives | What it does |
|
||||
|-------|---------------|--------------|
|
||||
| 1 — Bad-range retry | `Disc::patch` (one pass over the mapfile per call) | Re-reads non-`+` ranges with the long timeout. Idempotent; caller invokes N times. |
|
||||
| 2 — Single-shot primitive | `Drive::read` in `src/drive/mod.rs` | One CDB, one timeout, one result. No inline retries, no SCSI reset. |
|
||||
| 3 — In-loop request adaptation | `DiscStream::fill_extents` adaptive batch sizer | Halves the batch on failure, retries at the same LBA, walks back up on a clean-read streak. |
|
||||
|
||||
The library exposes flat verbs; the caller drives the multipass loop. Autorip
|
||||
runs `Disc::sweep` once, then loops `Disc::patch` until either the mapfile is
|
||||
clean or the configured retry budget is exhausted, then hands the ISO off to
|
||||
the mux pipeline. The `freemkv` CLI does the same shape with a
|
||||
terminal-output progress sink. Layer 3 runs inside any consumer of
|
||||
`DiscStream` (direct PES pipeline, ISO playback, etc.) without caller
|
||||
involvement.
|
||||
|
||||
Three primitives compose the disc-side flow:
|
||||
|
||||
| Primitive | What it does |
|
||||
|---------------------------|-----------------------------------------------------------------------|
|
||||
| `Disc::copy` | disc → ISO. Writes a sidecar `.mapfile`. Opt-in skip-forward on failure. |
|
||||
| `Disc::patch` | Re-reads bad ranges from the drive. Idempotent; call N times. |
|
||||
| `Disc::sweep` | disc → ISO, one forward pass. Writes a sidecar `.mapfile`. Opt-in skip-on-error. |
|
||||
| `Disc::patch` | Re-reads bad ranges from the drive. One pass per call; caller invokes N times. |
|
||||
| `DiscStream` (ISO source) | Reads sectors from the ISO, feeds decrypt → demux → codec → mux. |
|
||||
|
||||
The caller orchestrates. Autorip's `rip_disc` loops `copy` then N × `patch` per
|
||||
the `MAX_RETRIES` config, then hands the ISO off to the existing mux pipeline.
|
||||
|
||||
## Data model
|
||||
|
||||
### Mapfile
|
||||
@@ -26,7 +41,7 @@ plain text, greppable, tool-interoperable. Flushed to disk on every `record()`
|
||||
so a crashed rip loses at most one block.
|
||||
|
||||
```
|
||||
# Rescue Logfile. Created by libfreemkv v0.11.22
|
||||
# Rescue Logfile. Created by libfreemkv v0.13.6
|
||||
# Current pos / status / pass / pass_time
|
||||
0x000000000 ? 1 0
|
||||
# pos size status
|
||||
@@ -48,68 +63,127 @@ Status characters match ddrescue:
|
||||
|
||||
Position and size are hex byte offsets into the ISO.
|
||||
|
||||
### `CopyOptions` and `PatchOptions`
|
||||
### `SweepOptions` and `PatchOptions`
|
||||
|
||||
Defaults preserve pre-`0.11.21` behavior — full drive recovery on every read,
|
||||
abort on the first unreadable sector. Opt in to the recovery-friendly path:
|
||||
The library no longer dispatches between sweep and patch internally — the
|
||||
caller picks the verb explicitly per pass. The two option structs are flat
|
||||
and have no overlap:
|
||||
|
||||
```rust
|
||||
CopyOptions {
|
||||
skip_on_error: true, // zero-fill bad blocks, continue
|
||||
skip_forward: true, // exponential skip-forward after a failure
|
||||
resume: true, // pick up from an existing ISO + mapfile
|
||||
decrypt: false, // keep the ISO a raw disc image
|
||||
..Default::default()
|
||||
SweepOptions {
|
||||
decrypt: true,
|
||||
resume: false,
|
||||
batch_sectors: None,
|
||||
skip_on_error: true, // damage-jump + zero-fill on read failure
|
||||
progress: Some(&reporter),
|
||||
halt: Some(flag),
|
||||
}
|
||||
|
||||
PatchOptions {
|
||||
decrypt: true,
|
||||
block_sectors: None,
|
||||
full_recovery: true,
|
||||
reverse: true, // walk bad ranges high → low LBA
|
||||
wedged_threshold: 50,
|
||||
progress: Some(&reporter),
|
||||
halt: Some(flag),
|
||||
}
|
||||
```
|
||||
|
||||
Caller-orchestrated dispatch (the policy `Disc::copy` used to embed):
|
||||
|
||||
- No mapfile → `sweep` (fresh Pass 1).
|
||||
- Mapfile with `?` ranges → `sweep` with `resume: true`.
|
||||
- Mapfile covers full disc, only `*` / `/` / `-` ranges → `patch`.
|
||||
- Mapfile clean → done; no further pass needed.
|
||||
|
||||
Each consumer (autorip, `freemkv` CLI) implements the loop in roughly five
|
||||
lines of `Mapfile::stats()` checks.
|
||||
|
||||
## Algorithm
|
||||
|
||||
### Pass 1 — fast sweep
|
||||
### Pass 1 — fast sweep (`Disc::sweep`)
|
||||
|
||||
1. Read 64 KB (32 sectors, one BD ECC block) at the current LBA.
|
||||
2. On success: mark the range `+`, advance by one block.
|
||||
3. On failure (with `skip_on_error`): zero-fill the block in the ISO, mark
|
||||
it `*`, advance.
|
||||
4. If `skip_forward` is set: after a failure, jump ahead by an exponentially
|
||||
growing amount (256 KB initial, doubling on consecutive failures, capped at
|
||||
1% of disc). The skipped bytes are also marked `*` — `patch` will visit
|
||||
them later.
|
||||
5. Reset the skip size to 256 KB on the first success after a failure.
|
||||
1. Read one ECC block (32 sectors for UHD, 16 for BD/DVD) at the current LBA.
|
||||
2. On success: write data to ISO, mark `+`, advance.
|
||||
3. On failure (with `multipass`): zero-fill, mark `*`, advance.
|
||||
4. Track a sliding window of the last 16 ECC block results. When ≥12% are failures
|
||||
→ **damage-jump**: skip ahead by `256×batch×multiplier` sectors (8 MB base for
|
||||
UHD). Double the multiplier on each jump (8→16→32→64 MB...). Zero-fill the gap as `*`.
|
||||
5. On 16 consecutive good reads: reset jump multiplier to 1, restore max read speed.
|
||||
6. Speed control: damage zone entry → minimum speed, exit → maximum speed.
|
||||
7. Only transport failures (USB bridge crash) abort the pass.
|
||||
|
||||
Pass 1 completes when every byte has terminal status (`+`, `-`, or the caller
|
||||
bails via the halt flag).
|
||||
Pass 1 completes when every byte has been visited (either `+` or `*`).
|
||||
|
||||
### Pass 2+ — patch
|
||||
### Pass 2+ — patch (`Disc::patch`)
|
||||
|
||||
`Disc::patch` reads the mapfile and iterates every non-`+` range. For each:
|
||||
`Disc::patch` reads the mapfile and iterates every non-`+` range. Default: **reverse** mode
|
||||
(walks ranges from highest LBA to lowest, within each range from end to start).
|
||||
|
||||
1. Issue a drive read with full recovery enabled (SCSI-level retries,
|
||||
ECC recovery, the lot).
|
||||
2. On success: write the good bytes into the ISO at the exact byte offset,
|
||||
mark `+`.
|
||||
3. On failure: mark `-`.
|
||||
4. Update the mapfile after every block — crash-safe resume.
|
||||
1. Issue a single-sector read with 60 s timeout (`recovery=true`). Drive firmware
|
||||
does its own ECC recovery inside that window.
|
||||
2. On success: write the good bytes into the ISO, mark `+`.
|
||||
3. On failure with non-marginal SCSI sense: bail immediately (drive won't produce data).
|
||||
4. On failure with marginal sense: mark `-`, continue.
|
||||
5. Update the mapfile after every block — crash-safe resume.
|
||||
6. Wedged-drive exit: 50 consecutive failures with zero recovery → bail this pass.
|
||||
|
||||
Idempotent. Call `patch` N times for N retry attempts; typically the caller
|
||||
stops early if a pass recovers zero bytes (structure-protected sectors will
|
||||
never yield).
|
||||
### In-stream — adaptive batch halving (`DiscStream::fill_extents`)
|
||||
|
||||
When a consumer reads a `DiscStream` directly (no ISO intermediate),
|
||||
`fill_extents` runs an adaptive sizer in front of `Drive::read`:
|
||||
|
||||
1. Try the current preferred batch size (e.g. 32 sectors, one BD ECC block).
|
||||
2. On failure: halve the batch and retry at the same LBA. Emit
|
||||
`EventKind::BatchSizeChanged { reason: Shrunk }`.
|
||||
3. On a clean-read streak: probe back up toward the preferred size. Emit
|
||||
`EventKind::BatchSizeChanged { reason: Probed }`.
|
||||
4. If a single-sector read fails: skip (zero-fill, emit
|
||||
`EventKind::SectorSkipped`) when `skip_errors` is set, otherwise return
|
||||
`Err(DiscRead)`.
|
||||
|
||||
This is layer 3. It exists so a transient single-sector glitch in a 32-sector
|
||||
batch can be isolated and read individually without the caller needing to
|
||||
implement retry logic.
|
||||
|
||||
## Design choices
|
||||
|
||||
**`Drive::read` is single-shot.** No inline retry phases, no SCSI reset,
|
||||
no eject cycle. The `recovery` flag controls only the per-CDB timeout
|
||||
(1.5 s vs. 30 s); on any failure it returns `Err(DiscRead)` immediately.
|
||||
Inline recovery (5× gentle retry → close + SCSI reset + reopen → 5× more)
|
||||
was removed in 0.13.6. See the stop-wedge postmortem (2026-04-25) for rationale:
|
||||
the inline reset on the LG BU40N (Initio USB-SATA bridge)
|
||||
wedged drive firmware below the bridge without ever recovering a sector,
|
||||
and the gentle-retry phase produced long stretches of 0 KB/s with no
|
||||
recoveries to show for it. Recovery responsibility is now layered: layer 1
|
||||
handles ranges, layer 3 handles request size, neither touches the
|
||||
wedge-prone reset path.
|
||||
|
||||
**No `MODE SELECT` to disable drive retries.** Research showed neither ddrescue
|
||||
nor MakeMKV does this. Drive firmware has access to raw analog signal, laser
|
||||
power control, and drive-specific ECC tuning that userspace can't replicate —
|
||||
disabling it throws away recovery headroom on marginal sectors. We fail fast
|
||||
via short SG_IO timeouts instead, and we avoid per-sector probing on first
|
||||
contact by using large blocks + skip-forward.
|
||||
via short SG_IO timeouts in pass 1 and let the firmware work the long timeout
|
||||
in pass 2 / patch.
|
||||
|
||||
**No SCSI reset from any retry path.** `SgIoTransport::reset` (Linux) is
|
||||
trimmed to a kernel SG_IO state flush plus ALLOW MEDIUM REMOVAL — the
|
||||
`SG_SCSI_RESET` ioctl and STOP/START UNIT escalation were removed in 0.13.6.
|
||||
The macOS reset (which had been a no-op) was removed entirely. The top-level
|
||||
`scsi::reset()` / `reset_with_timeout()` / `reset_blocking()` wrappers were
|
||||
also removed (no callers). The remaining `Drive::reset()` is only invoked
|
||||
explicitly by callers that need an eject-cycle escape hatch — it is never
|
||||
reached from a read path.
|
||||
|
||||
**ISO intermediate, even for single-pass.** Pass 1 always writes an ISO. The
|
||||
mux stage reads the ISO via `IsoSectorReader`. For single-pass (no retries),
|
||||
this adds ~2-3 min (local disk mux) but gains resumability across crashes,
|
||||
re-muxability without re-ripping, and a persistent forensic artifact. Callers
|
||||
who need pure speed can bypass and use `DiscStream::new(Box::new(drive), …)`
|
||||
directly — the lib doesn't forbid it.
|
||||
directly — the lib doesn't forbid it, and layer 3 (adaptive batch halving)
|
||||
still applies there.
|
||||
|
||||
**Mapfile in ddrescue format.** Plain text so users can `less` it, `diff` it,
|
||||
or feed it to ddrescue's own tooling. Crash-safe (flush-per-record). Entries
|
||||
@@ -117,11 +191,11 @@ coalesce on adjacent same-status ranges so files stay small.
|
||||
|
||||
**Patches target `-`, `*`, `/`, and `?` alike.** The status state machine is
|
||||
ddrescue's but `patch` collapses the distinction — it just tries every
|
||||
non-finished range with full recovery. Future work can specialize (trim vs.
|
||||
non-finished range with the long timeout. Future work can specialize (trim vs.
|
||||
scrape vs. retry with direction reversal) if there's measured benefit.
|
||||
|
||||
## References
|
||||
|
||||
- [ddrescue manual, Algorithm chapter](https://www.gnu.org/software/ddrescue/manual/ddrescue_manual.html)
|
||||
- [ddrescue optical media notes](https://www.electric-spoon.com/doc/gddrescue/html/Optical-media.html)
|
||||
- Source: [`src/disc/mapfile.rs`](../src/disc/mapfile.rs), [`src/disc/mod.rs`](../src/disc/mod.rs) (`Disc::copy`, `Disc::patch`)
|
||||
- Source: [`src/disc/mapfile.rs`](../src/disc/mapfile.rs), [`src/disc/sweep.rs`](../src/disc/sweep.rs) (`Disc::sweep`), [`src/disc/patch.rs`](../src/disc/patch.rs) (`Disc::patch`), [`src/drive/mod.rs`](../src/drive/mod.rs) (`Drive::read`), [`src/mux/disc.rs`](../src/mux/disc.rs) (`DiscStream::fill_extents`).
|
||||
|
||||
+1
-1
@@ -126,7 +126,7 @@ USB optical drives have ~500ms round-trip latency per SCSI command. Since `read_
|
||||
|
||||
`Disc::scan()` wraps the drive in a `BufferedSectorReader` before reading. On a single-sector read, the buffer prefetches a batch of sectors (sized from the kernel's `max_hw_sectors_kb` for the device) and caches them. Subsequent reads to nearby LBAs return from cache with zero SCSI overhead. After parsing the UDF directory structure, the entire metadata partition is pre-read into the cache, so all ICB lookups during title scanning and encryption resolution are instant.
|
||||
|
||||
The buffer is transparent -- `read_filesystem()`, `read_file()`, and all downstream code still call `read_sectors(lba, 1, buf)` as before. The batching happens inside the `SectorReader` implementation.
|
||||
The buffer is transparent -- `read_filesystem()`, `read_file()`, and all downstream code still call `read_sectors(lba, 1, buf)` as before. The batching happens inside the `SectorSource` implementation.
|
||||
|
||||
### UDF Filename Encoding
|
||||
|
||||
|
||||
@@ -247,6 +247,7 @@ fn validate_processing_key(
|
||||
/// Find Verify Media Key Record (type 0x10) in MKB.
|
||||
fn mkb_find_mk_dv(mkb: &[u8]) -> Option<[u8; 16]> {
|
||||
let mut pos = 0;
|
||||
let mut type10_seen: Vec<(usize, usize)> = Vec::new();
|
||||
while pos + 4 <= mkb.len() {
|
||||
let rec_type = mkb[pos];
|
||||
let rec_len = u32::from_be_bytes([0, mkb[pos + 1], mkb[pos + 2], mkb[pos + 3]]) as usize;
|
||||
@@ -254,14 +255,32 @@ fn mkb_find_mk_dv(mkb: &[u8]) -> Option<[u8; 16]> {
|
||||
break;
|
||||
}
|
||||
|
||||
if rec_type == 0x10 {
|
||||
type10_seen.push((pos, rec_len));
|
||||
}
|
||||
|
||||
if rec_type == 0x10 && rec_len >= 20 {
|
||||
// mk_dv is at offset 4 (after record header)
|
||||
let mut dv = [0u8; 16];
|
||||
dv.copy_from_slice(&mkb[pos + 4..pos + 20]);
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "mkb_mk_dv_found",
|
||||
pos,
|
||||
rec_len,
|
||||
"mk_dv extracted from MKB"
|
||||
);
|
||||
return Some(dv);
|
||||
}
|
||||
pos += rec_len;
|
||||
}
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "mkb_mk_dv_not_found",
|
||||
type10_seen = ?type10_seen,
|
||||
scanned_bytes = pos,
|
||||
"no 0x10 record with rec_len>=20 found"
|
||||
);
|
||||
None
|
||||
}
|
||||
|
||||
@@ -628,32 +647,70 @@ pub fn resolve_keys(
|
||||
}
|
||||
};
|
||||
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "resolve_keys_start",
|
||||
aacs2,
|
||||
bus_encryption,
|
||||
disc_hash = %hash_hex,
|
||||
mkb_present = mkb_data.is_some(),
|
||||
"resolve_keys: starting"
|
||||
);
|
||||
|
||||
// Path 1: Look up VUK by disc hash in KEYDB
|
||||
if let Some(entry) = keydb.find_disc(&hash_hex) {
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path1_hit_entry", "disc hash found in keydb");
|
||||
if let Some(vuk) = entry.vuk {
|
||||
return Some(build(vuk, 1));
|
||||
}
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path1_no_vuk", "disc hash entry has no VUK");
|
||||
} else {
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path1_miss", "disc hash NOT in keydb");
|
||||
}
|
||||
|
||||
// Path 2: Find entry with matching VID → derive VUK from MK + VID
|
||||
let mut path2_mk_did_count = 0usize;
|
||||
for entry in keydb.disc_entries.values() {
|
||||
if let (Some(mk), Some(did)) = (entry.media_key, entry.disc_id) {
|
||||
path2_mk_did_count += 1;
|
||||
if did == *volume_id {
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path2_hit", "MK+VID entry matched volume_id");
|
||||
return Some(build(derive_vuk(&mk, volume_id), 2));
|
||||
}
|
||||
}
|
||||
}
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path2_miss", mk_did_entries = path2_mk_did_count, "no MK+VID entry matched volume_id");
|
||||
|
||||
// Path 3: MKB + processing keys → media key → VUK
|
||||
if let Some(mkb) = mkb_data {
|
||||
let mk_dv = mkb_find_mk_dv(mkb);
|
||||
let subdiff = mkb_find_subdiff_records(mkb);
|
||||
let cvalues = mkb_find_cvalues(mkb);
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "resolve_keys_mkb_records",
|
||||
mk_dv_found = mk_dv.is_some(),
|
||||
subdiff_found = subdiff.is_some(),
|
||||
subdiff_len = subdiff.as_ref().map(|s| s.len()).unwrap_or(0),
|
||||
cvalues_found = cvalues.is_some(),
|
||||
cvalues_len = cvalues.as_ref().map(|c| c.len()).unwrap_or(0),
|
||||
"MKB record scan results"
|
||||
);
|
||||
|
||||
if let Some(mk) = derive_media_key_from_pk(mkb, &keydb.processing_keys) {
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path3_hit", "media key derived from processing key");
|
||||
return Some(build(derive_vuk(&mk, volume_id), 3));
|
||||
}
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path3_miss", pk_count = keydb.processing_keys.len(), "PK derivation failed");
|
||||
|
||||
// Path 4: MKB + device keys → processing key → media key → VUK
|
||||
if let Some(mk) = derive_media_key_from_dk(mkb, &keydb.device_keys) {
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path4_hit", "media key derived from device key");
|
||||
return Some(build(derive_vuk(&mk, volume_id), 4));
|
||||
}
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_path4_miss", dk_count = keydb.device_keys.len(), "DK derivation failed");
|
||||
} else {
|
||||
tracing::warn!(target: "freemkv::disc", phase = "resolve_keys_no_mkb", "no MKB data available; paths 3/4 skipped");
|
||||
}
|
||||
|
||||
None
|
||||
|
||||
+158
-1
@@ -20,6 +20,35 @@ pub struct ClipInfo {
|
||||
pub ep_coarse: Vec<EpCoarse>,
|
||||
/// Fine EP entries for the primary video stream
|
||||
pub ep_fine: Vec<EpFine>,
|
||||
/// Per-stream metadata from the ProgramInfo section (BD spec).
|
||||
/// Cross-validates the MPLS STN view — see `labels/clpi.rs`.
|
||||
/// Empty when program_info is missing or malformed.
|
||||
pub streams: Vec<ClpiStream>,
|
||||
}
|
||||
|
||||
/// One stream descriptor from the CLPI ProgramInfo / stream_coding_info
|
||||
/// table. Mirrors the same fields the MPLS STN table carries — see
|
||||
/// `mpls::StreamEntry` for the playlist-side equivalent.
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct ClpiStream {
|
||||
/// PID of the stream in the MPEG-TS (matches MPLS).
|
||||
pub pid: u16,
|
||||
/// SCSI/BD coding type byte (0x80 LPCM, 0x83 TrueHD, 0x86 DTS-HD MA,
|
||||
/// 0x90 PG, etc.). See `labels::mpls_universal::coding_type_to_codec_hint`.
|
||||
pub coding_type: u8,
|
||||
/// ISO 639-2 3-char language code. Empty for video streams.
|
||||
pub language: String,
|
||||
/// Audio format byte (1=mono, 3=stereo, 6=5.1, 12=7.1).
|
||||
/// Zero for non-audio streams.
|
||||
pub audio_format: u8,
|
||||
/// Audio sample rate (1=48kHz, 4=96kHz, 5=192kHz). Zero for non-audio.
|
||||
pub audio_rate: u8,
|
||||
/// Video format byte (1=480i, 4=1080i, 5=720p, 6=1080p, 8=2160p).
|
||||
/// Zero for non-video.
|
||||
pub video_format: u8,
|
||||
/// Video rate (1=23.976, 2=24, 3=25, 4=29.97, 6=50, 7=59.94).
|
||||
pub video_rate: u8,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -128,7 +157,7 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
|
||||
|
||||
// Header offsets
|
||||
let _seq_info_start = u32::from_be_bytes([data[8], data[9], data[10], data[11]]) as usize;
|
||||
let _prog_info_start = u32::from_be_bytes([data[12], data[13], data[14], data[15]]) as usize;
|
||||
let prog_info_start = u32::from_be_bytes([data[12], data[13], data[14], data[15]]) as usize;
|
||||
let cpi_start = u32::from_be_bytes([data[16], data[17], data[18], data[19]]) as usize;
|
||||
|
||||
// ClipInfo section at offset 40
|
||||
@@ -139,6 +168,16 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
|
||||
0
|
||||
};
|
||||
|
||||
// Parse ProgramInfo (per-stream language + codec). Best-effort:
|
||||
// malformed program_info doesn't fail the parse, just gives an
|
||||
// empty streams list. EP map is unaffected — sector-range lookups
|
||||
// continue to work.
|
||||
let streams = if prog_info_start > 0 && prog_info_start + 6 < data.len() {
|
||||
parse_program_info(&data[prog_info_start..])
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
// Parse CPI / EP Map
|
||||
let (ep_coarse, ep_fine) = if cpi_start > 0 && cpi_start + 8 < data.len() {
|
||||
parse_cpi(&data[cpi_start..])?
|
||||
@@ -151,9 +190,127 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
|
||||
source_packet_count,
|
||||
ep_coarse,
|
||||
ep_fine,
|
||||
streams,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse the ProgramInfo section: per-stream (pid, coding_type,
|
||||
/// language, codec sub-fields). Layout per BD spec / libbluray
|
||||
/// clpi_parse.c:
|
||||
///
|
||||
/// ```text
|
||||
/// ProgramInfo:
|
||||
/// length: 4 bytes
|
||||
/// reserved: 1 byte
|
||||
/// num_programs: 1 byte
|
||||
/// for each program:
|
||||
/// spn_program_sequence_start: 4 bytes
|
||||
/// program_map_pid: 2 bytes
|
||||
/// num_streams: 1 byte
|
||||
/// num_groups: 1 byte
|
||||
/// for each stream:
|
||||
/// pid: 2 bytes
|
||||
/// stream_coding_info_length: 1 byte
|
||||
/// stream_coding_info: (varies by coding_type)
|
||||
/// coding_type: 1 byte
|
||||
/// per-type bytes (see match arms below)
|
||||
/// ```
|
||||
///
|
||||
/// Returns `Vec::new()` on any structural mismatch — we don't propagate
|
||||
/// errors because the EP map is the primary CLPI output, and a corrupt
|
||||
/// program_info shouldn't break sector-range lookups.
|
||||
fn parse_program_info(data: &[u8]) -> Vec<ClpiStream> {
|
||||
let mut out = Vec::new();
|
||||
if data.len() < 6 {
|
||||
return out;
|
||||
}
|
||||
// length: 4 bytes (skipped — we trust the section bounds in the
|
||||
// caller's slice and read the bytes that follow). Reserved 1 byte
|
||||
// at offset 4. num_programs at offset 5.
|
||||
let num_programs = data[5] as usize;
|
||||
let mut pos = 6usize;
|
||||
for _ in 0..num_programs {
|
||||
// Program header: 4 (spn) + 2 (pmt_pid) + 1 (num_streams) + 1 (num_groups) = 8 bytes
|
||||
if pos + 8 > data.len() {
|
||||
return out;
|
||||
}
|
||||
let num_streams = data[pos + 6] as usize;
|
||||
pos += 8;
|
||||
|
||||
for _ in 0..num_streams {
|
||||
// Stream header: 2 (pid) + 1 (sci_length) + sci bytes
|
||||
if pos + 3 > data.len() {
|
||||
return out;
|
||||
}
|
||||
let pid = u16::from_be_bytes([data[pos], data[pos + 1]]);
|
||||
let sci_len = data[pos + 2] as usize;
|
||||
let sci_end = pos + 3 + sci_len;
|
||||
if sci_end > data.len() || sci_len < 1 {
|
||||
return out;
|
||||
}
|
||||
let sci = &data[pos + 3..sci_end];
|
||||
let coding_type = sci[0];
|
||||
|
||||
let mut audio_format = 0u8;
|
||||
let mut audio_rate = 0u8;
|
||||
let mut video_format = 0u8;
|
||||
let mut video_rate = 0u8;
|
||||
let mut language = String::new();
|
||||
|
||||
match coding_type {
|
||||
// Video — MPEG-2 (0x02), H.264 (0x1B), HEVC (0x24)
|
||||
0x02 | 0x1B | 0x24 => {
|
||||
if sci.len() >= 2 {
|
||||
video_format = (sci[1] >> 4) & 0x0F;
|
||||
video_rate = sci[1] & 0x0F;
|
||||
}
|
||||
}
|
||||
// Primary audio — LPCM(0x80), AC-3(0x81), DTS(0x82),
|
||||
// TrueHD(0x83), AC-3+(0x84), DTS-HD(0x85), DTS-HD MA(0x86)
|
||||
0x80..=0x86 => {
|
||||
if sci.len() >= 2 {
|
||||
audio_format = (sci[1] >> 4) & 0x0F;
|
||||
audio_rate = sci[1] & 0x0F;
|
||||
}
|
||||
if sci.len() >= 5 {
|
||||
language = String::from_utf8_lossy(&sci[2..5]).to_string();
|
||||
}
|
||||
}
|
||||
// Secondary audio (0xA1 AC-3+, 0xA2 DTS-HD)
|
||||
0xA1 | 0xA2 => {
|
||||
if sci.len() >= 2 {
|
||||
audio_format = (sci[1] >> 4) & 0x0F;
|
||||
audio_rate = sci[1] & 0x0F;
|
||||
}
|
||||
if sci.len() >= 5 {
|
||||
language = String::from_utf8_lossy(&sci[2..5]).to_string();
|
||||
}
|
||||
}
|
||||
// PG (0x90), IG (0x91): coding_type + 3-byte language [+ char_code for PG]
|
||||
0x90 | 0x91 => {
|
||||
if sci.len() >= 4 {
|
||||
language = String::from_utf8_lossy(&sci[1..4]).to_string();
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
out.push(ClpiStream {
|
||||
pid,
|
||||
coding_type,
|
||||
language,
|
||||
audio_format,
|
||||
audio_rate,
|
||||
video_format,
|
||||
video_rate,
|
||||
});
|
||||
|
||||
pos = sci_end;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Parse the CPI section containing the EP map.
|
||||
fn parse_cpi(data: &[u8]) -> Result<(Vec<EpCoarse>, Vec<EpFine>)> {
|
||||
if data.len() < 8 {
|
||||
|
||||
+2
-2
@@ -16,7 +16,7 @@ pub mod lfsr;
|
||||
pub(crate) mod tables;
|
||||
|
||||
use crate::disc::Extent;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
|
||||
/// CSS decryption state for a DVD title.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -34,7 +34,7 @@ pub struct CssState {
|
||||
/// 0x80 (start of the encrypted region). This only happens when a new PES
|
||||
/// packet begins at exactly sector offset 128. We scan up to 50000
|
||||
/// scrambled sectors sequentially across all extents.
|
||||
pub fn crack_key(reader: &mut dyn SectorReader, extents: &[Extent]) -> Option<CssState> {
|
||||
pub fn crack_key(reader: &mut dyn SectorSource, extents: &[Extent]) -> Option<CssState> {
|
||||
let mut tried = 0u32;
|
||||
let max_tries = 50_000;
|
||||
|
||||
|
||||
+189
-3
@@ -3,9 +3,131 @@
|
||||
//! Decrypts sectors in-place using resolved keys from disc scanning.
|
||||
//! Handles AACS 1.0, AACS 2.0, and CSS transparently.
|
||||
//! The caller never sees encrypted data unless explicitly bypassed.
|
||||
//!
|
||||
//! ## Parallel AACS decrypt
|
||||
//!
|
||||
//! Each AACS aligned unit (6144 bytes) is decrypted INDEPENDENTLY of
|
||||
//! every other unit — per-unit key derivation from the unit_key plus
|
||||
//! the unit's own first-16-byte header. There is no cross-unit
|
||||
//! dependency, so a buffer of N units can be decrypted on N threads
|
||||
//! in parallel via a persistent rayon thread pool.
|
||||
//!
|
||||
//! Small buffers (< [`PARALLEL_MIN_UNITS`] units) fall through to the
|
||||
//! serial path to avoid pool dispatch overhead beating the per-unit
|
||||
//! AES work.
|
||||
//!
|
||||
//! ## Thread-count configuration — three layers
|
||||
//!
|
||||
//! Resolution order (highest wins):
|
||||
//! 1. The most recent [`set_decrypt_threads`] call with `n > 0`.
|
||||
//! Calling this *replaces* the live thread pool — useful for a
|
||||
//! settings-page slider in a long-running daemon.
|
||||
//! 2. `FREEMKV_THREADS` env var, if set and `> 0`. Single knob
|
||||
//! covering decrypt today, intended to also drive any future
|
||||
//! input-side / output-side worker pools.
|
||||
//! 3. Default: all available cores. Algorithm optimisation comes
|
||||
//! first — we measure single-thread performance to find serial
|
||||
//! bottlenecks before throwing parallelism at it — but once a
|
||||
//! pool is engaged we use the whole box. Hard cap at
|
||||
//! [`MAX_THREADS`] (rayon stack memory).
|
||||
|
||||
use crate::aacs;
|
||||
use crate::css;
|
||||
use rayon::prelude::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
/// Minimum units in a buffer before we pay the pool-dispatch cost of
|
||||
/// fanning out. Below this, serial is faster.
|
||||
const PARALLEL_MIN_UNITS: usize = 8;
|
||||
|
||||
/// Hard upper bound on configurable thread count. Anything larger is
|
||||
/// almost certainly a misconfiguration; rayon would happily allocate
|
||||
/// thousands of worker stacks otherwise.
|
||||
pub const MAX_THREADS: usize = 64;
|
||||
|
||||
/// Process-wide decrypt thread count override. `0` means "use env
|
||||
/// var, else default" — see [`decrypt_threads`] for the resolution
|
||||
/// order.
|
||||
static DECRYPT_THREADS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
/// Current rayon pool. `RwLock<Option<Arc<...>>>` so that
|
||||
/// [`set_decrypt_threads`] can swap the pool out without leaking the
|
||||
/// old one and without blocking ongoing decrypt work (in-flight calls
|
||||
/// hold an `Arc` clone via [`decrypt_pool`] and finish on the old
|
||||
/// pool; new calls pick up the new pool).
|
||||
static DECRYPT_POOL: RwLock<Option<Arc<rayon::ThreadPool>>> = RwLock::new(None);
|
||||
|
||||
/// Configure how many threads to use for AACS unit decryption. A value
|
||||
/// of `0` resets to the env / default resolution. `1` forces serial.
|
||||
/// `N > 1` builds a new rayon pool of size N and atomically replaces
|
||||
/// the live pool.
|
||||
///
|
||||
/// Thread-safe. Live decrypt calls keep their previously-acquired
|
||||
/// pool reference for the rest of the call — no mid-call pool
|
||||
/// switch. Subsequent calls see the new pool.
|
||||
///
|
||||
/// Pool construction is ~ms-scale; safe to call from a settings POST
|
||||
/// handler.
|
||||
pub fn set_decrypt_threads(n: usize) {
|
||||
let clamped = n.min(MAX_THREADS);
|
||||
DECRYPT_THREADS.store(clamped, Ordering::Relaxed);
|
||||
// Drop the existing pool. Next decrypt_pool() call rebuilds with
|
||||
// the new resolved thread count.
|
||||
if let Ok(mut guard) = DECRYPT_POOL.write() {
|
||||
*guard = None;
|
||||
}
|
||||
}
|
||||
|
||||
/// Get (or lazily build) the active rayon thread pool. Returns an
|
||||
/// `Arc` so in-flight work survives a concurrent
|
||||
/// [`set_decrypt_threads`] swap.
|
||||
fn decrypt_pool() -> Arc<rayon::ThreadPool> {
|
||||
// Fast path: pool already built.
|
||||
if let Ok(guard) = DECRYPT_POOL.read() {
|
||||
if let Some(pool) = guard.as_ref() {
|
||||
return Arc::clone(pool);
|
||||
}
|
||||
}
|
||||
// Slow path: build a new one under the write lock. Double-check
|
||||
// after acquiring in case another caller built it first.
|
||||
let mut guard = DECRYPT_POOL.write().expect("DECRYPT_POOL RwLock poisoned");
|
||||
if let Some(pool) = guard.as_ref() {
|
||||
return Arc::clone(pool);
|
||||
}
|
||||
let n = decrypt_threads();
|
||||
let pool = Arc::new(
|
||||
rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(n)
|
||||
.thread_name(|i| format!("freemkv-decrypt-{i}"))
|
||||
.build()
|
||||
.expect("rayon decrypt pool build failed"),
|
||||
);
|
||||
*guard = Some(Arc::clone(&pool));
|
||||
pool
|
||||
}
|
||||
|
||||
/// Current effective decrypt thread count. Resolution order:
|
||||
/// 1. Most recent [`set_decrypt_threads`] value (if > 0)
|
||||
/// 2. `FREEMKV_THREADS` env var (if set and > 0)
|
||||
/// 3. Default: all available cores, capped at [`MAX_THREADS`].
|
||||
pub fn decrypt_threads() -> usize {
|
||||
let explicit = DECRYPT_THREADS.load(Ordering::Relaxed);
|
||||
if explicit > 0 {
|
||||
return explicit;
|
||||
}
|
||||
let env = std::env::var("FREEMKV_THREADS")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<usize>().ok())
|
||||
.unwrap_or(0);
|
||||
if env > 0 {
|
||||
return env.min(MAX_THREADS);
|
||||
}
|
||||
let cores = std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(2);
|
||||
cores.clamp(1, MAX_THREADS)
|
||||
}
|
||||
|
||||
/// Resolved decryption state from disc scanning.
|
||||
/// Passed to `decrypt_sectors()` — the caller doesn't need to know
|
||||
@@ -57,13 +179,44 @@ pub fn decrypt_sectors(
|
||||
return Err(crate::error::Error::DecryptFailed);
|
||||
}
|
||||
};
|
||||
let rdk = read_data_key.as_ref();
|
||||
let rdk: Option<[u8; 16]> = *read_data_key;
|
||||
let unit_len = aacs::ALIGNED_UNIT_LEN;
|
||||
let nthreads = decrypt_threads();
|
||||
let chunks: Vec<&mut [u8]> = buf.chunks_mut(unit_len).collect();
|
||||
let nunits = chunks.len();
|
||||
|
||||
for chunk in buf.chunks_mut(unit_len) {
|
||||
// Per-unit decrypt closure. The is_unit_encrypted check is
|
||||
// a byte-0 heuristic; on a misfire we snapshot+restore via
|
||||
// the original bytes so non-m2ts (e.g. MPLS/CLPI nav files)
|
||||
// survive. See test `nav_file_unit_survives_decrypt_attempt`.
|
||||
let decrypt_one = |chunk: &mut [u8]| {
|
||||
if chunk.len() == unit_len && aacs::is_unit_encrypted(chunk) {
|
||||
aacs::decrypt_unit_full(chunk, &uk, rdk);
|
||||
let original: Vec<u8> = chunk.to_vec();
|
||||
if !aacs::decrypt_unit_full(chunk, &uk, rdk.as_ref()) {
|
||||
chunk.copy_from_slice(&original);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if nthreads <= 1 || nunits < PARALLEL_MIN_UNITS {
|
||||
// Serial path: avoids thread-pool overhead for tiny
|
||||
// buffers; also the only path when caller pinned
|
||||
// single-threaded via FREEMKV_THREADS=1.
|
||||
for chunk in chunks {
|
||||
decrypt_one(chunk);
|
||||
}
|
||||
} else {
|
||||
// Parallel path via rayon's persistent global pool.
|
||||
// The pool is built once on first use (lazy_static-style)
|
||||
// and reused across every decrypt_sectors call — no
|
||||
// per-call OS thread spawn, no thread-creation latency
|
||||
// amortised per batch. Each unit decrypts independently
|
||||
// (own key derivation), so par_iter is sound.
|
||||
decrypt_pool().install(|| {
|
||||
chunks.into_par_iter().for_each(|chunk| {
|
||||
decrypt_one(chunk);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
DecryptKeys::Css { title_key } => {
|
||||
@@ -74,3 +227,36 @@ pub fn decrypt_sectors(
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Regression for the 0.18.1 nav-file scramble bug. A non-m2ts unit whose
|
||||
/// first byte has the top 2 bits set (here: the ASCII letter 'M' that
|
||||
/// MPLS files start with, 0x4D = 0b01001101) trips `is_unit_encrypted`,
|
||||
/// gets AES-decrypted with the unit key, fails the TS-sync verification,
|
||||
/// and must be restored to its original bytes — not left scrambled.
|
||||
#[test]
|
||||
fn nav_file_unit_survives_decrypt_attempt() {
|
||||
let mut unit = vec![0u8; aacs::ALIGNED_UNIT_LEN];
|
||||
unit[0] = b'M';
|
||||
unit[1] = b'P';
|
||||
unit[2] = b'L';
|
||||
unit[3] = b'S';
|
||||
for (i, b) in unit.iter_mut().enumerate().skip(4) {
|
||||
*b = (i as u8).wrapping_mul(31);
|
||||
}
|
||||
let snapshot = unit.clone();
|
||||
|
||||
let keys = DecryptKeys::Aacs {
|
||||
unit_keys: vec![(0, [0xAB; 16])],
|
||||
read_data_key: None,
|
||||
};
|
||||
decrypt_sectors(&mut unit, &keys, 0).unwrap();
|
||||
assert_eq!(
|
||||
unit, snapshot,
|
||||
"non-m2ts unit must be restored after failed decrypt"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -3,13 +3,13 @@
|
||||
use super::*;
|
||||
use crate::clpi;
|
||||
use crate::mpls;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf;
|
||||
|
||||
impl Disc {
|
||||
/// Scan Blu-ray titles from MPLS playlists.
|
||||
pub(super) fn scan_bluray_titles(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
) -> Vec<DiscTitle> {
|
||||
let mut titles = Vec::new();
|
||||
@@ -31,7 +31,7 @@ impl Disc {
|
||||
}
|
||||
|
||||
pub(super) fn parse_playlist(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
filename: &str,
|
||||
data: &[u8],
|
||||
@@ -199,7 +199,7 @@ impl Disc {
|
||||
/// Prefers English, falls back to first available language.
|
||||
/// Returns None if META directory is empty or XML has no usable title.
|
||||
pub(super) fn read_meta_title(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
) -> Option<String> {
|
||||
let meta_dir = udf_fs.find_dir("/BDMV/META")?;
|
||||
|
||||
+2
-2
@@ -2,13 +2,13 @@
|
||||
|
||||
use super::*;
|
||||
use crate::ifo;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf;
|
||||
|
||||
impl Disc {
|
||||
/// Scan DVD titles from IFO files (VIDEO_TS.IFO + VTS_XX_0.IFO).
|
||||
pub(super) fn scan_dvd_titles(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
) -> Vec<DiscTitle> {
|
||||
let dvd_info = match ifo::parse_vmg(reader, udf_fs) {
|
||||
|
||||
+104
-8
@@ -2,7 +2,7 @@
|
||||
|
||||
use super::*;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf;
|
||||
|
||||
/// Result of SCSI AACS handshake (ECDH authentication).
|
||||
@@ -22,42 +22,103 @@ impl Disc {
|
||||
) -> Option<HandshakeResult> {
|
||||
use crate::aacs::{self, KeyDb};
|
||||
|
||||
let keydb_path = opts.resolve_keydb()?;
|
||||
let keydb = KeyDb::load(&keydb_path).ok()?;
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_entry",
|
||||
"do_handshake entered"
|
||||
);
|
||||
let keydb_path = match opts.resolve_keydb() {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_no_keydb",
|
||||
"no KEYDB found in search paths; handshake skipped"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let keydb = match KeyDb::load(&keydb_path) {
|
||||
Ok(db) => db,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_keydb_load_failed",
|
||||
io_error_kind = ?e.kind(),
|
||||
keydb = %keydb_path.display(),
|
||||
"KEYDB load failed; handshake skipped"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let host_cert_count = keydb.host_certs.len();
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_start",
|
||||
host_cert_count,
|
||||
keydb = %keydb_path.display(),
|
||||
"handshake starting"
|
||||
);
|
||||
|
||||
const MAX_CERT_ATTEMPTS: usize = 16;
|
||||
for hc in keydb.host_certs.iter().take(MAX_CERT_ATTEMPTS) {
|
||||
let mut last_err_code: Option<u16> = None;
|
||||
for (idx, hc) in keydb.host_certs.iter().take(MAX_CERT_ATTEMPTS).enumerate() {
|
||||
match aacs::handshake::aacs_authenticate(session, &hc.private_key, &hc.certificate) {
|
||||
Ok(mut auth) => {
|
||||
let volume_id = match aacs::handshake::read_volume_id(session, &mut auth) {
|
||||
Ok(vid) => vid,
|
||||
Err(_) => return None, // handshake succeeded but can't read VID
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_vid_read_failed",
|
||||
cert_index = idx,
|
||||
error_code = e.code(),
|
||||
"auth ok but volume ID read failed"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let read_data_key = aacs::handshake::read_data_keys(session, &mut auth)
|
||||
.ok()
|
||||
.map(|(rdk, _)| rdk);
|
||||
tracing::debug!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_ok",
|
||||
cert_index = idx,
|
||||
has_read_data_key = read_data_key.is_some(),
|
||||
);
|
||||
return Some(HandshakeResult {
|
||||
volume_id,
|
||||
read_data_key,
|
||||
});
|
||||
}
|
||||
Err(_) => {
|
||||
Err(e) => {
|
||||
last_err_code = Some(e.code());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_all_certs_failed",
|
||||
host_cert_count,
|
||||
tried = host_cert_count.min(MAX_CERT_ATTEMPTS),
|
||||
last_error_code = last_err_code,
|
||||
"all host certs in KEYDB rejected by drive"
|
||||
);
|
||||
// All host certs failed — return None, not a fake success
|
||||
None
|
||||
}
|
||||
|
||||
/// Resolve disc encryption — AACS 1.0, AACS 2.0, CSS, or none.
|
||||
///
|
||||
/// Reads AACS files from UDF (via SectorReader), resolves keys through
|
||||
/// Reads AACS files from UDF (via SectorSource), resolves keys through
|
||||
/// whatever path works: KEYDB VUK lookup, media key derivation, processing
|
||||
/// keys, device keys. Uses handshake result (volume ID, bus key) if available.
|
||||
pub(super) fn resolve_encryption(
|
||||
udf_fs: &udf::UdfFs,
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
keydb_path: &std::path::Path,
|
||||
handshake: Option<&HandshakeResult>,
|
||||
) -> Result<AacsState> {
|
||||
@@ -73,6 +134,19 @@ impl Disc {
|
||||
.or_else(|_| udf_fs.read_file(reader, "/AACS/DUPLICATE/Unit_Key_RO.inf"))
|
||||
.map_err(|_| Error::AacsNoKeys)?;
|
||||
|
||||
// Log the disc hash so we can confirm whether it's present in KEYDB
|
||||
// when key resolution fails. The disc hash is SHA-1 of the full
|
||||
// Unit_Key_RO.inf file bytes — same value KEYDB.cfg keys VUK entries by.
|
||||
let dh = crate::aacs::disc_hash(&uk_ro_data);
|
||||
let dh_hex = crate::aacs::disc_hash_hex(&dh);
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "scan_aacs_disc_hash",
|
||||
disc_hash = %dh_hex,
|
||||
uk_ro_len = uk_ro_data.len(),
|
||||
"disc hash computed (compare with keydb.cfg entries)"
|
||||
);
|
||||
|
||||
let cc_data = udf_fs
|
||||
.read_file(reader, "/AACS/Content000.cer")
|
||||
.or_else(|_| udf_fs.read_file(reader, "/AACS/Content001.cer"))
|
||||
@@ -84,6 +158,28 @@ impl Disc {
|
||||
.ok();
|
||||
let mkb_ver = mkb_data.as_deref().and_then(aacs::mkb_version);
|
||||
|
||||
let mkb_first_64_hex = mkb_data
|
||||
.as_deref()
|
||||
.map(|m| {
|
||||
m.iter()
|
||||
.take(64)
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect::<String>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "scan_aacs_mkb_info",
|
||||
mkb_present = mkb_data.is_some(),
|
||||
mkb_len = mkb_data.as_deref().map(|m| m.len()).unwrap_or(0),
|
||||
mkb_version = ?mkb_ver,
|
||||
mkb_first_64 = %mkb_first_64_hex,
|
||||
keydb_disc_count = keydb.disc_entries.len(),
|
||||
keydb_dk_count = keydb.device_keys.len(),
|
||||
keydb_pk_count = keydb.processing_keys.len(),
|
||||
"AACS resolution inputs"
|
||||
);
|
||||
|
||||
// Use handshake volume ID if available, otherwise zeros
|
||||
// (KEYDB VUK lookup by disc hash works without volume ID)
|
||||
let volume_id = handshake.map(|h| h.volume_id).unwrap_or([0u8; 16]);
|
||||
|
||||
+172
-15
@@ -17,11 +17,22 @@
|
||||
//!
|
||||
//! Status chars: `?` non-tried · `*` non-trimmed · `/` non-scraped · `-` unreadable · `+` finished.
|
||||
//!
|
||||
//! The mapfile is flushed to disk on every `record()` call so a crashed
|
||||
//! rip loses at most one block of recorded state.
|
||||
//! The mapfile is flushed to disk at most once per `FLUSH_INTERVAL`
|
||||
//! during `record()` calls, plus on explicit `flush()` and on `Drop`.
|
||||
//! This bounds atomic-rename RPC rate on networked staging (e.g. NFS)
|
||||
//! where per-record persists otherwise serialize the rip pipeline.
|
||||
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Minimum interval between mapfile persists. `record()` updates in-memory
|
||||
/// state every call but only writes to disk when this interval has elapsed
|
||||
/// since the last persist (or when `flush()` is called explicitly, or on
|
||||
/// `Drop`). Bounds RPC rate on NFS staging where atomic-rename per record
|
||||
/// otherwise dominates throughput. On crash the worst-case progress loss
|
||||
/// is one interval's worth of records.
|
||||
const FLUSH_INTERVAL: Duration = Duration::from_millis(1000);
|
||||
|
||||
/// Status of a byte range in the mapfile. ddrescue-compatible.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -69,21 +80,56 @@ pub struct MapEntry {
|
||||
}
|
||||
|
||||
/// Summary statistics over all entries.
|
||||
///
|
||||
/// `bytes_pending` aggregates `NonTried + NonTrimmed + NonScraped` for
|
||||
/// back-compat. `bytes_nontried` and `bytes_retryable` (= NonTrimmed +
|
||||
/// NonScraped) split that aggregate so UIs can distinguish *unread*
|
||||
/// territory (still ahead of Pass 1's read head) from *needs-retry*
|
||||
/// territory (Pass 1 already encountered, queued for Pass 2-N).
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct MapStats {
|
||||
pub bytes_total: u64,
|
||||
pub bytes_good: u64,
|
||||
pub bytes_unreadable: u64,
|
||||
pub bytes_pending: u64,
|
||||
/// Sectors Pass 1 hasn't reached yet (`NonTried`). Subset of
|
||||
/// `bytes_pending`.
|
||||
pub bytes_nontried: u64,
|
||||
/// Sectors flagged for Pass 2-N retry — `NonTrimmed` (multi-sector
|
||||
/// read failed; needs split) + `NonScraped` (small-block read
|
||||
/// partially recovered; remainder still pending). Subset of
|
||||
/// `bytes_pending`. This is the right signal for a "MAYBE / will
|
||||
/// retry" UI bucket; `bytes_pending` over-counts because it folds
|
||||
/// in `bytes_nontried`.
|
||||
pub bytes_retryable: u64,
|
||||
/// Number of unreadable ranges (for UI display). Computed from
|
||||
/// `ranges_with(&[Unreadable])`.
|
||||
pub num_bad_ranges: u32,
|
||||
/// Largest gap among unreadable ranges in milliseconds. Computed as
|
||||
/// largest range size / bytes_per_sec * 1000. Set by caller (autorip)
|
||||
/// since bytes_per_sec is application-specific.
|
||||
pub main_lost_ms: f64,
|
||||
}
|
||||
|
||||
/// Write-through mapfile. Every `record()` persists to disk immediately
|
||||
/// so a crash during rip loses at most one block.
|
||||
/// Time-batched mapfile. `record()` keeps in-memory state up-to-date on
|
||||
/// every call; persists to disk at most once per `FLUSH_INTERVAL`.
|
||||
/// Explicit `flush()` and `Drop` guarantee state is on disk after a sweep
|
||||
/// or patch finishes. On hard crash the worst-case loss is one flush
|
||||
/// interval of records — the file's payload bytes are unaffected.
|
||||
pub struct Mapfile {
|
||||
path: PathBuf,
|
||||
entries: Vec<MapEntry>,
|
||||
total_size: u64,
|
||||
version: String,
|
||||
/// Incrementally maintained stats — updated on every `record()` call
|
||||
/// so `stats()` is O(1) instead of O(n).
|
||||
stats: MapStats,
|
||||
/// True when in-memory state has changed but `write_to_disk` has not
|
||||
/// yet captured it.
|
||||
dirty: bool,
|
||||
/// Wall-clock timestamp of the last successful `write_to_disk` (or
|
||||
/// the moment the mapfile was constructed, whichever is later).
|
||||
last_flushed: Instant,
|
||||
}
|
||||
|
||||
impl Mapfile {
|
||||
@@ -91,7 +137,7 @@ impl Mapfile {
|
||||
/// Writes to disk immediately so a resume can pick up even if the caller
|
||||
/// never records anything.
|
||||
pub fn create(path: &Path, total_size: u64, version: &str) -> io::Result<Self> {
|
||||
let mf = Self {
|
||||
let mut mf = Self {
|
||||
path: path.to_path_buf(),
|
||||
entries: vec![MapEntry {
|
||||
pos: 0,
|
||||
@@ -100,8 +146,19 @@ impl Mapfile {
|
||||
}],
|
||||
total_size,
|
||||
version: version.to_string(),
|
||||
stats: MapStats {
|
||||
bytes_total: total_size,
|
||||
bytes_pending: total_size,
|
||||
bytes_nontried: total_size,
|
||||
..Default::default()
|
||||
},
|
||||
dirty: false,
|
||||
last_flushed: Instant::now(),
|
||||
};
|
||||
// Eager initial persist so a resume can pick this up even if
|
||||
// `record()` is never called.
|
||||
mf.write_to_disk()?;
|
||||
mf.last_flushed = Instant::now();
|
||||
Ok(mf)
|
||||
}
|
||||
|
||||
@@ -161,11 +218,15 @@ impl Mapfile {
|
||||
}
|
||||
entries.sort_by_key(|e| e.pos);
|
||||
let total_size = entries.last().map(|e| e.pos + e.size).unwrap_or(0);
|
||||
let stats = Self::compute_stats(&entries, total_size);
|
||||
Ok(Self {
|
||||
path: path.to_path_buf(),
|
||||
entries,
|
||||
total_size,
|
||||
version,
|
||||
stats,
|
||||
dirty: false,
|
||||
last_flushed: Instant::now(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -228,8 +289,30 @@ impl Mapfile {
|
||||
merged.push(e);
|
||||
}
|
||||
|
||||
// Recompute stats from merged entries. record() is already O(n) due to
|
||||
// drain-and-rebuild, so this is a constant-factor overhead. The critical
|
||||
// win is that stats() is now O(1) — called millions of times in the hot
|
||||
// path during sweep/patch, it just returns the cached value.
|
||||
self.stats = Self::compute_stats(&merged, self.total_size);
|
||||
self.entries = merged;
|
||||
self.write_to_disk()?;
|
||||
self.dirty = true;
|
||||
if self.last_flushed.elapsed() >= FLUSH_INTERVAL {
|
||||
self.write_to_disk()?;
|
||||
self.dirty = false;
|
||||
self.last_flushed = Instant::now();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist any pending in-memory changes to disk. No-op if clean.
|
||||
/// Callers (sweep/patch finalisation) invoke this after their last
|
||||
/// `record()` to guarantee state is durable before returning.
|
||||
pub fn flush(&mut self) -> io::Result<()> {
|
||||
if self.dirty {
|
||||
self.write_to_disk()?;
|
||||
self.dirty = false;
|
||||
self.last_flushed = Instant::now();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -267,16 +350,25 @@ impl Mapfile {
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> MapStats {
|
||||
self.stats
|
||||
}
|
||||
|
||||
fn compute_stats(entries: &[MapEntry], total_size: u64) -> MapStats {
|
||||
let mut s = MapStats {
|
||||
bytes_total: self.total_size,
|
||||
bytes_total: total_size,
|
||||
..Default::default()
|
||||
};
|
||||
for e in &self.entries {
|
||||
for e in entries {
|
||||
match e.status {
|
||||
SectorStatus::Finished => s.bytes_good += e.size,
|
||||
SectorStatus::Unreadable => s.bytes_unreadable += e.size,
|
||||
SectorStatus::NonTried | SectorStatus::NonTrimmed | SectorStatus::NonScraped => {
|
||||
s.bytes_pending += e.size
|
||||
SectorStatus::NonTried => {
|
||||
s.bytes_pending += e.size;
|
||||
s.bytes_nontried += e.size;
|
||||
}
|
||||
SectorStatus::NonTrimmed | SectorStatus::NonScraped => {
|
||||
s.bytes_pending += e.size;
|
||||
s.bytes_retryable += e.size;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -295,11 +387,7 @@ impl Mapfile {
|
||||
{
|
||||
let file = std::fs::File::create(&tmp)?;
|
||||
let mut w = std::io::BufWriter::new(file);
|
||||
writeln!(
|
||||
w,
|
||||
"# Rescue Logfile. Created by libfreemkv v{}",
|
||||
self.version
|
||||
)?;
|
||||
writeln!(w, "# Rescue Logfile. Created by {}", self.version)?;
|
||||
writeln!(w, "# Current pos / status / pass / pass_time")?;
|
||||
writeln!(w, "0x000000000 ? 1 0")?;
|
||||
writeln!(w, "# pos size status")?;
|
||||
@@ -319,6 +407,16 @@ impl Mapfile {
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Mapfile {
|
||||
/// Best-effort flush on drop so a sweep / patch that returns early
|
||||
/// (or unwinds) doesn't lose its in-memory state. Errors here are
|
||||
/// swallowed because Drop has no way to surface them; explicit
|
||||
/// `flush()` on the success path gives callers proper error handling.
|
||||
fn drop(&mut self) {
|
||||
let _ = self.flush();
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_hex(s: &str) -> io::Result<u64> {
|
||||
let s = s.strip_prefix("0x").unwrap_or(s);
|
||||
u64::from_str_radix(s, 16).map_err(|_| {
|
||||
@@ -419,6 +517,8 @@ mod tests {
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
mf.record(100, 200, SectorStatus::Finished).unwrap();
|
||||
mf.record(500, 100, SectorStatus::Unreadable).unwrap();
|
||||
// record() batches; explicit flush before reading back from disk.
|
||||
mf.flush().unwrap();
|
||||
let loaded = Mapfile::load(&p).unwrap();
|
||||
assert_eq!(loaded.entries(), mf.entries());
|
||||
let _ = std::fs::remove_file(&p);
|
||||
@@ -450,4 +550,61 @@ mod tests {
|
||||
assert_eq!(bad, vec![(100, 50), (300, 50)]);
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_consistent_after_overlapping_records() {
|
||||
let p = tmpfile("stats_consistent_after_overlapping");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
// Record some finished, some unreadable, some nontrimmed
|
||||
mf.record(0, 300, SectorStatus::Finished).unwrap();
|
||||
mf.record(300, 200, SectorStatus::NonTrimmed).unwrap();
|
||||
mf.record(500, 100, SectorStatus::Unreadable).unwrap();
|
||||
mf.record(600, 400, SectorStatus::Finished).unwrap();
|
||||
|
||||
// Final entries: [0..300 Finished, 300..500 NonTrimmed, 500..600 Unreadable, 600..1000 Finished]
|
||||
let s = mf.stats();
|
||||
assert_eq!(s.bytes_good, 700); // 300 + 400
|
||||
assert_eq!(s.bytes_unreadable, 100); // 100
|
||||
assert_eq!(s.bytes_pending, 200); // NonTrimmed only (NonTried=0)
|
||||
assert_eq!(s.bytes_nontried, 0);
|
||||
assert_eq!(s.bytes_retryable, 200); // NonTrimmed
|
||||
assert_eq!(s.bytes_total, 1000);
|
||||
|
||||
// Overwrite a NonTrimmed range with Finished
|
||||
mf.record(300, 100, SectorStatus::Finished).unwrap();
|
||||
// Entries: [0..400 Finished, 400..500 NonTrimmed, 500..600 Unreadable, 600..1000 Finished]
|
||||
let s2 = mf.stats();
|
||||
assert_eq!(s2.bytes_good, 800); // 400 + 400
|
||||
assert_eq!(s2.bytes_unreadable, 100);
|
||||
assert_eq!(s2.bytes_pending, 100); // NonTrimmed only
|
||||
assert_eq!(s2.bytes_retryable, 100);
|
||||
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_consistent_after_split_record() {
|
||||
let p = tmpfile("stats_consistent_after_split");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
// Mark middle as NonTrimmed
|
||||
mf.record(200, 400, SectorStatus::NonTrimmed).unwrap();
|
||||
// Entries: [0..200 NonTried, 200..600 NonTrimmed, 600..1000 NonTried]
|
||||
let s = mf.stats();
|
||||
assert_eq!(s.bytes_pending, 1000); // NonTried(600) + NonTrimmed(400)
|
||||
assert_eq!(s.bytes_retryable, 400); // NonTrimmed only
|
||||
assert_eq!(s.bytes_nontried, 600); // 200 + 400
|
||||
|
||||
// Overwrite the NonTrimmed with Finished (splitting the remaining NonTried)
|
||||
mf.record(200, 400, SectorStatus::Finished).unwrap();
|
||||
// Entries: [0..200 NonTried, 200..600 Finished, 600..1000 NonTried]
|
||||
let s2 = mf.stats();
|
||||
assert_eq!(s2.bytes_good, 400);
|
||||
assert_eq!(s2.bytes_pending, 600); // NonTried(200 + 400)
|
||||
assert_eq!(s2.bytes_nontried, 600);
|
||||
assert_eq!(s2.bytes_retryable, 0);
|
||||
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
}
|
||||
|
||||
+1265
-275
File diff suppressed because it is too large
Load Diff
+1787
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,239 @@
|
||||
//! `Disc::sweep`'s consumer-side `Sink<WorkItem>`.
|
||||
//!
|
||||
//! Background: the original sweep loop runs strictly serialised —
|
||||
//! SCSI read → decrypt → seek + write → mapfile.record → next iter.
|
||||
//! On a healthy disc the SCSI read costs ~5-12 ms per 64 KB batch and
|
||||
//! the post-read work (decrypt 1-3 ms + file write + mapfile fsync
|
||||
//! 5-15 ms) adds another batch's worth of latency. The drive idles
|
||||
//! during the post-read work; throughput tops out at the *sum* of
|
||||
//! both costs.
|
||||
//!
|
||||
//! 0.17.11 introduced a bespoke producer/consumer split (the now-
|
||||
//! removed `disc/sweep_pipeline.rs`) to overlap the two stages. 0.18
|
||||
//! collapses that split — together with the analogous splits patch
|
||||
//! and mux need — onto the generic [`crate::io::Pipeline`] +
|
||||
//! [`crate::io::Sink`] primitive. This module is the sweep-specific
|
||||
//! `Sink` impl; the producer-side state machine (read_error context,
|
||||
//! decrypt, set_speed, halt) stays in `Disc::sweep` in `disc/mod.rs`.
|
||||
//!
|
||||
//! Correctness invariants preserved (same as 0.17.11):
|
||||
//! - Mapfile is single-writer (consumer-only). No locking.
|
||||
//! - All `read_error::ReadCtx` state stays on the producer thread.
|
||||
//! - `set_speed` calls happen on the producer thread (same thread that
|
||||
//! owns the `SectorSource`). No new SCSI concurrency.
|
||||
//! - Per-iteration ordering of file-write → mapfile-record is kept
|
||||
//! intact in the consumer (write before record), so the on-disk
|
||||
//! invariant "mapfile only marks Finished what the file has
|
||||
//! received" survives a crash mid-pass.
|
||||
//! - The BU40N+Initio bridge wedge concern is unchanged: only one
|
||||
//! SCSI command in flight at a time, error-path timing identical,
|
||||
//! no new retry logic.
|
||||
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
use std::sync::mpsc::{Receiver, SyncSender, sync_channel};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::io::{Flow, Sink};
|
||||
|
||||
use super::mapfile::{MapStats, Mapfile, SectorStatus};
|
||||
|
||||
/// Reusable zero buffer for SkipFill / GapFill / BisectBad. 64 KB
|
||||
/// matches the existing zero_gap chunk size used by the pre-split
|
||||
/// sweep loop.
|
||||
const ZERO_CHUNK: usize = 65 * 1024;
|
||||
|
||||
/// Producer → Consumer messages. The consumer applies these in FIFO
|
||||
/// order; ordering of file writes and mapfile records across items is
|
||||
/// preserved.
|
||||
pub(super) enum WorkItem {
|
||||
/// Successful batch read. Producer has already decrypted `buf` if
|
||||
/// `opts.decrypt` was set. Consumer writes `buf` at `pos` and
|
||||
/// records the range as `Finished`.
|
||||
Good { pos: u64, buf: Vec<u8> },
|
||||
|
||||
/// Bisect inner-loop good single sector (already decrypted by the
|
||||
/// producer). 2048 bytes.
|
||||
BisectGood { pos: u64, buf: Box<[u8; 2048]> },
|
||||
|
||||
/// Bisect inner-loop bad single sector. Consumer writes 2048
|
||||
/// zeros at `pos` and records the sector as `NonTrimmed`.
|
||||
BisectBad { pos: u64 },
|
||||
|
||||
/// Whole-batch zero-fill (failed batch on `SkipBlock`, or the
|
||||
/// failed batch portion of `JumpAhead`). Consumer streams zeros
|
||||
/// across `[pos, pos+len)` and records the range as `NonTrimmed`.
|
||||
SkipFill { pos: u64, len: u64 },
|
||||
|
||||
/// Gap fill following a `JumpAhead`. Same effect as `SkipFill`;
|
||||
/// distinguished only so future logging / instrumentation can
|
||||
/// tell them apart without parsing a flag.
|
||||
GapFill { pos: u64, len: u64 },
|
||||
|
||||
/// Producer wants the latest mapfile stats for the progress
|
||||
/// callback. Consumer responds on `prog_tx` with a fresh
|
||||
/// [`ProgressSnapshot`]. Best-effort: if the producer hasn't
|
||||
/// drained the previous snapshot, the new one is silently
|
||||
/// dropped — the producer's local cache stays current enough.
|
||||
StatsRequest,
|
||||
}
|
||||
|
||||
/// Snapshot the consumer sends back to the producer for the progress
|
||||
/// callback.
|
||||
pub(super) struct ProgressSnapshot {
|
||||
pub stats: MapStats,
|
||||
pub bad_ranges: Vec<(u64, u64)>,
|
||||
}
|
||||
|
||||
/// Final summary returned by the consumer thread on shutdown — what
|
||||
/// `SweepSink::close` produces, surfaced to the producer via
|
||||
/// `Pipeline::finish`.
|
||||
pub(super) struct ConsumerSummary {
|
||||
pub stats: MapStats,
|
||||
}
|
||||
|
||||
/// Drain any pending progress snapshots from the consumer. Returns
|
||||
/// the most recent one, if any. The producer caches it and uses it
|
||||
/// for subsequent progress callbacks until a fresh one arrives.
|
||||
pub(super) fn try_recv_progress(rx: &Receiver<ProgressSnapshot>) -> Option<ProgressSnapshot> {
|
||||
let mut latest = None;
|
||||
while let Ok(snap) = rx.try_recv() {
|
||||
latest = Some(snap);
|
||||
}
|
||||
latest
|
||||
}
|
||||
|
||||
/// `Sink<WorkItem>` for sweep. Owns the writeback file + mapfile +
|
||||
/// progress back-channel. `apply` carries the file-write +
|
||||
/// mapfile.record per item; `close` drains the writeback pipeline,
|
||||
/// fsyncs the ISO, and flushes the mapfile.
|
||||
pub(super) struct SweepSink {
|
||||
file: crate::io::WritebackFile,
|
||||
map: Mapfile,
|
||||
/// `sync_all`-on-failure-is-an-error iff the output is a regular
|
||||
/// file. `/dev/null` and pipes always fail `sync_all`; that's not
|
||||
/// a real error.
|
||||
is_regular: bool,
|
||||
/// Back-channel for `StatsRequest` responses. The producer caches
|
||||
/// the latest snapshot and uses it for the progress callback;
|
||||
/// dropped sends on a full channel are by design.
|
||||
prog_tx: SyncSender<ProgressSnapshot>,
|
||||
/// Reusable zero buffer for SkipFill / GapFill / BisectBad. Held
|
||||
/// in the sink so each apply call doesn't reallocate.
|
||||
zero: Box<[u8; ZERO_CHUNK]>,
|
||||
}
|
||||
|
||||
impl SweepSink {
|
||||
/// Construct a new `SweepSink` plus the matching progress
|
||||
/// receiver. Channel depth on the back-channel is `1` — the
|
||||
/// producer's cache is the source of truth between snapshots.
|
||||
pub(super) fn new(
|
||||
file: crate::io::WritebackFile,
|
||||
map: Mapfile,
|
||||
is_regular: bool,
|
||||
) -> (Self, Receiver<ProgressSnapshot>) {
|
||||
let (prog_tx, prog_rx) = sync_channel::<ProgressSnapshot>(1);
|
||||
let sink = SweepSink {
|
||||
file,
|
||||
map,
|
||||
is_regular,
|
||||
prog_tx,
|
||||
zero: Box::new([0u8; ZERO_CHUNK]),
|
||||
};
|
||||
(sink, prog_rx)
|
||||
}
|
||||
}
|
||||
|
||||
impl Sink<WorkItem> for SweepSink {
|
||||
type Output = ConsumerSummary;
|
||||
|
||||
fn apply(&mut self, item: WorkItem) -> Result<Flow, Error> {
|
||||
match item {
|
||||
WorkItem::Good { pos, buf } => {
|
||||
// Decrypt is on the producer; consumer assumes plaintext.
|
||||
let len = buf.len() as u64;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&buf)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, len, SectorStatus::Finished)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::BisectGood { pos, buf } => {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&buf[..])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, 2048, SectorStatus::Finished)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::BisectBad { pos } => {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&self.zero[..2048])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, 2048, SectorStatus::NonTrimmed)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::SkipFill { pos, len } | WorkItem::GapFill { pos, len } => {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
// Subsequent writes are sequential; `WritebackFile`'s
|
||||
// seek-elision keeps them on the writeback pipeline path.
|
||||
let mut filled = 0u64;
|
||||
while filled < len {
|
||||
let chunk = (len - filled).min(self.zero.len() as u64) as usize;
|
||||
self.file
|
||||
.write_all(&self.zero[..chunk])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
filled += chunk as u64;
|
||||
}
|
||||
self.map
|
||||
.record(pos, len, SectorStatus::NonTrimmed)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::StatsRequest => {
|
||||
let stats = self.map.stats();
|
||||
let bad_ranges = self.map.ranges_with(&[
|
||||
SectorStatus::NonTrimmed,
|
||||
SectorStatus::Unreadable,
|
||||
SectorStatus::NonScraped,
|
||||
SectorStatus::NonTried,
|
||||
]);
|
||||
// Best-effort: drop on backpressure; producer's cache
|
||||
// stays current enough.
|
||||
let _ = self
|
||||
.prog_tx
|
||||
.try_send(ProgressSnapshot { stats, bad_ranges });
|
||||
}
|
||||
}
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(mut self) -> Result<Self::Output, Error> {
|
||||
// Drain the writeback pipeline + fsync the ISO, then persist
|
||||
// any pending mapfile state. Same finalisation order as the
|
||||
// pre-Pipeline consumer loop.
|
||||
if let Err(e) = self.file.sync_all() {
|
||||
if self.is_regular {
|
||||
return Err(Error::IoError { source: e });
|
||||
}
|
||||
// Non-regular outputs (/dev/null, pipes) always fail
|
||||
// sync_all; that's not a real error.
|
||||
}
|
||||
self.map.flush().map_err(|e| Error::IoError { source: e })?;
|
||||
|
||||
Ok(ConsumerSummary {
|
||||
stats: self.map.stats(),
|
||||
})
|
||||
}
|
||||
}
|
||||
+10
-11
@@ -1,26 +1,25 @@
|
||||
//! macOS drive discovery and device resolution.
|
||||
//!
|
||||
//! `find_drives` uses IOKit registry enumeration (via `scsi::list_drives`)
|
||||
//! to discover optical drives without exclusive access or unmounts. Only
|
||||
//! the returned paths are then opened for INQUIRY to build full `DriveId`.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::identity::DriveId;
|
||||
|
||||
pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
let mut drives = Vec::new();
|
||||
for i in 0..16 {
|
||||
let path = format!("/dev/disk{}", i);
|
||||
if !std::path::Path::new(&path).exists() {
|
||||
continue;
|
||||
}
|
||||
match crate::scsi::open(std::path::Path::new(&path)) {
|
||||
let discovered = crate::scsi::list_drives();
|
||||
for info in discovered {
|
||||
let path = std::path::Path::new(&info.path);
|
||||
match crate::scsi::open(path) {
|
||||
Ok(mut transport) => {
|
||||
if let Ok(id) = DriveId::from_drive(transport.as_mut()) {
|
||||
if !id.raw_inquiry.is_empty() && (id.raw_inquiry[0] & 0x1F) == 0x05 {
|
||||
drives.push((path, id));
|
||||
}
|
||||
drives.push((info.path.clone(), id));
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
// Device exists but can't be opened (likely mounted).
|
||||
// Use `diskutil unmountDisk /dev/diskN` to unmount before accessing.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+214
-234
@@ -1,19 +1,26 @@
|
||||
//! Drive session — open, identify, and read from optical drives.
|
||||
//!
|
||||
//! Three-step open:
|
||||
//! 1. `open()` — open device, identify drive. Always OEM.
|
||||
//! 2. `wait_ready()` — wait for disc to spin up. Call before reading.
|
||||
//! 3. `init()` — activate custom firmware. Removes riplock.
|
||||
//! 4. `probe_disc()` — probe disc surface. Drive learns optimal speeds.
|
||||
|
||||
pub(crate) fn extract_scsi_context(e: &Error) -> (u8, Option<crate::scsi::ScsiSense>) {
|
||||
match e {
|
||||
Error::ScsiError { status, sense, .. } => (*status, *sense),
|
||||
Error::DiscRead { status, sense, .. } => (status.unwrap_or(0), *sense),
|
||||
_ => (0, None),
|
||||
}
|
||||
}
|
||||
|
||||
pub mod capture;
|
||||
|
||||
// Per-platform discovery helpers (the `pub(crate)` `find_drives` /
|
||||
// equivalents). Crate-public so `scsi/{linux,macos,windows}.rs` can
|
||||
// reuse the existing enumeration logic when shaping `DriveInfo`.
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
pub(crate) mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
pub(crate) mod macos;
|
||||
#[cfg(windows)]
|
||||
mod windows;
|
||||
pub(crate) mod windows;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::event::{Event, EventKind};
|
||||
@@ -22,7 +29,7 @@ use crate::platform::PlatformDriver;
|
||||
use crate::platform::mt1959::Mt1959;
|
||||
use crate::profile::{self, DriveProfile};
|
||||
use crate::scsi::ScsiTransport;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
@@ -50,9 +57,6 @@ const SCSI_GET_EVENT_STATUS: u8 = 0x4A;
|
||||
const SCSI_MODE_SENSE: u8 = 0x5A;
|
||||
const SCSI_REPORT_KEY: u8 = 0xA4;
|
||||
|
||||
/// Recovery state after a read error — stay at min speed for N bytes.
|
||||
const RECOVERY_WINDOW: u64 = 500 * 1024 * 1024; // 500 MB
|
||||
|
||||
/// Optical disc drive session -- open, identify, unlock, and read.
|
||||
pub struct Drive {
|
||||
scsi: Box<dyn ScsiTransport>,
|
||||
@@ -61,12 +65,20 @@ pub struct Drive {
|
||||
pub platform: Option<profile::Platform>,
|
||||
pub drive_id: DriveId,
|
||||
device_path: String,
|
||||
/// Bytes remaining in the min-speed recovery window.
|
||||
recovery_bytes_remaining: u64,
|
||||
/// Halt flag — when set, Drive::read() bails at the next check point.
|
||||
halt: Arc<AtomicBool>,
|
||||
/// Event handler — fires during read recovery.
|
||||
/// Event handler — fires for read errors and library-level state changes.
|
||||
event_fn: Option<Box<dyn Fn(Event) + Send>>,
|
||||
/// Linux only: raw fd for the corresponding block device (`/dev/sr*`)
|
||||
/// used as a recovery fallback when SCSI READ via `/dev/sg*` returns
|
||||
/// an error. The kernel `sr_mod` driver auto-retries failed reads
|
||||
/// (~5× per command) — historically the reason `dd if=/dev/sr0`
|
||||
/// recovers ~50% of bad sectors that single-shot `SG_IO` READ
|
||||
/// misses on the same drive. `None` when the block device couldn't
|
||||
/// be resolved or opened (no fallback in that case; SCSI read
|
||||
/// errors propagate as before).
|
||||
#[cfg(target_os = "linux")]
|
||||
block_dev_fd: Option<std::os::unix::io::RawFd>,
|
||||
}
|
||||
|
||||
impl Drive {
|
||||
@@ -85,6 +97,9 @@ impl Drive {
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let block_dev_fd = open_block_device_for_sg(device);
|
||||
|
||||
Ok(Drive {
|
||||
scsi: transport,
|
||||
driver,
|
||||
@@ -92,9 +107,10 @@ impl Drive {
|
||||
profile,
|
||||
drive_id,
|
||||
device_path: device.to_string_lossy().to_string(),
|
||||
recovery_bytes_remaining: 0,
|
||||
halt: Arc::new(AtomicBool::new(false)),
|
||||
event_fn: None,
|
||||
#[cfg(target_os = "linux")]
|
||||
block_dev_fd,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -118,6 +134,9 @@ impl Drive {
|
||||
self.event_fn = Some(Box::new(f));
|
||||
}
|
||||
|
||||
#[allow(dead_code)] // public on_event registration kept; Drive currently
|
||||
// has no internal emission sites after the 0.13.6 recovery strip.
|
||||
// DiscStream is the BytesRead source. Plan to drop on_event in 0.14.
|
||||
fn emit(&self, kind: EventKind) {
|
||||
if let Some(ref f) = self.event_fn {
|
||||
f(Event { kind });
|
||||
@@ -128,13 +147,6 @@ impl Drive {
|
||||
self.halt.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Halt-aware sleep. Returns `Err(Halted)` if the flag fires before or
|
||||
/// during the wait. The only sleep the drive's recovery path is allowed
|
||||
/// to use — makes it impossible to accidentally block through a Stop.
|
||||
fn checked_sleep(&self, total: std::time::Duration) -> Result<()> {
|
||||
sleep_until_halted(&self.halt, total)
|
||||
}
|
||||
|
||||
/// Halt-aware SCSI execute. Returns `Err(Halted)` if the flag is set
|
||||
/// before the command dispatches or by the time it completes. The only
|
||||
/// path to talk to the drive in the recovery hot loop; keeps Drive::read
|
||||
@@ -167,11 +179,6 @@ impl Drive {
|
||||
self.unlock_tray();
|
||||
}
|
||||
|
||||
// NOTE: Debug aid — remove after fd issue is resolved
|
||||
pub fn device_path_owned(&self) -> String {
|
||||
self.device_path.clone()
|
||||
}
|
||||
|
||||
/// Whether this drive has a known profile (unlock parameters available).
|
||||
pub fn has_profile(&self) -> bool {
|
||||
self.profile.is_some()
|
||||
@@ -184,31 +191,16 @@ impl Drive {
|
||||
|
||||
pub fn wait_ready(&mut self) -> Result<()> {
|
||||
let tur = [SCSI_TEST_UNIT_READY, 0x00, 0x00, 0x00, 0x00, 0x00];
|
||||
let mut tried_reset = false;
|
||||
|
||||
for _ in 0..60 {
|
||||
let mut buf = [0u8; 0];
|
||||
match self.scsi.as_mut().execute(
|
||||
&tur,
|
||||
crate::scsi::DataDirection::None,
|
||||
&mut buf,
|
||||
5_000,
|
||||
) {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(Error::ScsiError { sense_key: 5, .. }) if !tried_reset => {
|
||||
// Illegal Request on TUR — drive may be stuck from a previous session.
|
||||
// Try reset() which attempts multiple recovery approaches.
|
||||
tried_reset = true;
|
||||
if self.reset().is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
// If reset failed but disc is present, proceed anyway —
|
||||
// the scan path will handle errors individually.
|
||||
if self.drive_status() == DriveStatus::DiscPresent {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err(_) => {}
|
||||
if self
|
||||
.scsi
|
||||
.as_mut()
|
||||
.execute(&tur, crate::scsi::DataDirection::None, &mut buf, 5_000)
|
||||
.is_ok()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
}
|
||||
@@ -263,94 +255,18 @@ impl Drive {
|
||||
5_000,
|
||||
) {
|
||||
Ok(_) => DriveStatus::DiscPresent,
|
||||
Err(Error::ScsiError { sense_key: 2, .. }) => DriveStatus::NotReady,
|
||||
Err(Error::ScsiError { sense_key: 6, .. }) => DriveStatus::NotReady, // UNIT ATTENTION
|
||||
Err(ref e)
|
||||
if e.scsi_sense()
|
||||
.is_some_and(|s| s.is_not_ready() || s.is_unit_attention()) =>
|
||||
{
|
||||
DriveStatus::NotReady
|
||||
}
|
||||
_ => DriveStatus::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Attempt to reset the drive to a clean state.
|
||||
///
|
||||
/// Escalates through increasingly aggressive recovery:
|
||||
/// 1. Unlock tray + stop/start — handles normal stuck states
|
||||
/// 2. Eject cycle — clears LibreDrive firmware stuck state (proven on BU40N)
|
||||
/// 3. Re-init — firmware re-upload if profile available
|
||||
///
|
||||
/// Note: step 2 physically ejects the tray. On slimline drives the user
|
||||
/// must push it back in manually. Returns Ok(()) if TUR succeeds after
|
||||
/// any step, even if the drive reports "tray open" (that's a valid state).
|
||||
pub fn reset(&mut self) -> Result<()> {
|
||||
let mut buf = [0u8; 0];
|
||||
let tur = [SCSI_TEST_UNIT_READY, 0x00, 0x00, 0x00, 0x00, 0x00];
|
||||
|
||||
// 1. Unlock + stop/start
|
||||
self.unlock_tray();
|
||||
let stop = [SCSI_START_STOP_UNIT, 0x00, 0x00, 0x00, 0x00, 0x00];
|
||||
let _ =
|
||||
self.scsi
|
||||
.as_mut()
|
||||
.execute(&stop, crate::scsi::DataDirection::None, &mut buf, 5_000);
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
let start = [SCSI_START_STOP_UNIT, 0x00, 0x00, 0x00, 0x01, 0x00];
|
||||
let _ =
|
||||
self.scsi
|
||||
.as_mut()
|
||||
.execute(&start, crate::scsi::DataDirection::None, &mut buf, 5_000);
|
||||
std::thread::sleep(std::time::Duration::from_millis(2000));
|
||||
|
||||
if self
|
||||
.scsi
|
||||
.as_mut()
|
||||
.execute(&tur, crate::scsi::DataDirection::None, &mut buf, 5_000)
|
||||
.is_ok()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// 2. Eject cycle — clears MT1959 LibreDrive stuck state.
|
||||
// After eject, TUR returning "Not Ready — tray open" (sense key 2)
|
||||
// counts as success: the drive is functional, just needs disc reinserted.
|
||||
self.unlock_tray();
|
||||
let eject = [SCSI_START_STOP_UNIT, 0x00, 0x00, 0x00, 0x02, 0x00];
|
||||
let _ =
|
||||
self.scsi
|
||||
.as_mut()
|
||||
.execute(&eject, crate::scsi::DataDirection::None, &mut buf, 30_000);
|
||||
std::thread::sleep(std::time::Duration::from_millis(2000));
|
||||
|
||||
match self
|
||||
.scsi
|
||||
.as_mut()
|
||||
.execute(&tur, crate::scsi::DataDirection::None, &mut buf, 5_000)
|
||||
{
|
||||
Ok(_) => return Ok(()),
|
||||
Err(Error::ScsiError { sense_key: 2, .. }) => return Ok(()), // tray open = valid
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// 3. If still stuck and we have a profile, try re-init
|
||||
if self.driver.is_some() {
|
||||
self.init()?;
|
||||
std::thread::sleep(std::time::Duration::from_millis(1000));
|
||||
match self.scsi.as_mut().execute(
|
||||
&tur,
|
||||
crate::scsi::DataDirection::None,
|
||||
&mut buf,
|
||||
5_000,
|
||||
) {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(Error::ScsiError { sense_key: 2, .. }) => return Ok(()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::DeviceResetFailed {
|
||||
path: self.device_path.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn platform_name(&self) -> &str {
|
||||
match self.platform {
|
||||
Some(ref p) => p.name(),
|
||||
@@ -516,29 +432,41 @@ impl Drive {
|
||||
}
|
||||
}
|
||||
|
||||
/// Read sectors from the disc with automatic error recovery.
|
||||
/// Read sectors from the disc. Single-shot — no inline retries, no
|
||||
/// SCSI reset.
|
||||
///
|
||||
/// On failure: drops to min speed, waits with escalating patience
|
||||
/// (5s, 10s, 15s, 30s, 60s), resets drive between attempts.
|
||||
/// After recovery, stays at min speed for 500 MB before ramping up.
|
||||
/// `recovery=true` uses [`crate::scsi::READ_RECOVERY_TIMEOUT_MS`] (60 s,
|
||||
/// matches sg_dd) for the `Disc::patch` pass; `recovery=false` uses
|
||||
/// [`crate::scsi::READ_TIMEOUT_MS`] (30 s, matches the kernel's
|
||||
/// `/sys/block/sr*/device/timeout` default) for `Disc::copy`'s fast
|
||||
/// skip-forward sweep. Both budgets are generous enough that the drive
|
||||
/// can finish ECC recovery on a marginal sector — pre-0.13.21 this was
|
||||
/// 1.5 s on the fast path which forced the kernel mid-layer to time
|
||||
/// out and escalate while we waited anyway. On any failure returns
|
||||
/// `Err(DiscRead)` immediately; orchestration (`Disc::patch` multi-pass,
|
||||
/// `DiscStream` adaptive batch halving) handles retry policy.
|
||||
///
|
||||
/// Returns Err only after all attempts exhausted — user should clean
|
||||
/// the disc and resume.
|
||||
/// Inline retry phases (5× gentle + reset+reopen + 5× more) were
|
||||
/// removed in 0.13.6. Per
|
||||
/// the stop-wedge postmortem (2026-04-25),
|
||||
/// the inline reset on the LG BU40N (Initio bridge) wedged drive
|
||||
/// firmware without ever recovering a sector. The remaining recovery
|
||||
/// layers (Disc::patch multi-pass, DiscStream batch halving) do not
|
||||
/// touch the wedge-prone reset path.
|
||||
pub fn read(&mut self, lba: u32, count: u16, buf: &mut [u8], recovery: bool) -> Result<usize> {
|
||||
// Non-recovery mode is the Disc::copy fast pass — intent is "fail
|
||||
// fast, let skip_forward advance past bad regions." A 5s budget let
|
||||
// the drive grind L-EC on structure-protected / marginal sectors for
|
||||
// nearly the full interval per 64 KB block, pinning throughput at
|
||||
// ~13 KB/s on difficult discs. 1500ms kills slow reads early so
|
||||
// skip_forward can double its stride; recoverable sectors are picked
|
||||
// up on Disc::patch where recovery=true and the timeout is 30s.
|
||||
let timeout_ms = if !recovery {
|
||||
1_500
|
||||
} else if self.recovery_bytes_remaining > 0 {
|
||||
30_000
|
||||
let timeout_ms = if recovery {
|
||||
crate::scsi::READ_RECOVERY_TIMEOUT_MS
|
||||
} else {
|
||||
10_000
|
||||
crate::scsi::READ_TIMEOUT_MS
|
||||
};
|
||||
tracing::debug!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
recovery,
|
||||
timeout_ms,
|
||||
"Drive::read enter"
|
||||
);
|
||||
let cdb = [
|
||||
crate::scsi::SCSI_READ_10,
|
||||
0x00,
|
||||
@@ -552,82 +480,79 @@ impl Drive {
|
||||
0x00,
|
||||
];
|
||||
|
||||
// Normal read. Fast path: drive returns data, we return it.
|
||||
// checked_exec short-circuits to Err(Halted) if Stop was requested;
|
||||
// read() never touches the halt flag directly.
|
||||
match self.checked_exec(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
buf,
|
||||
timeout_ms,
|
||||
) {
|
||||
Ok(result) => {
|
||||
if self.recovery_bytes_remaining > 0 {
|
||||
let bytes_read = count as u64 * 2048;
|
||||
self.recovery_bytes_remaining =
|
||||
self.recovery_bytes_remaining.saturating_sub(bytes_read);
|
||||
if self.recovery_bytes_remaining == 0 {
|
||||
self.emit(EventKind::SpeedChange { speed_kbs: 0xFFFF });
|
||||
self.set_speed(0xFFFF);
|
||||
Ok(result) => Ok(result.bytes_transferred),
|
||||
Err(Error::Halted) => Err(Error::Halted),
|
||||
Err(e) => {
|
||||
let (status, sense) = extract_scsi_context(&e);
|
||||
tracing::warn!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
inner_error = %e,
|
||||
scsi_status = status,
|
||||
"Drive::read checked_exec failed"
|
||||
);
|
||||
|
||||
// /dev/sr0 pread fallback (Linux only). The kernel
|
||||
// sr_mod driver auto-retries failed reads (~5× per
|
||||
// command). Empirically (BU40N + Dune Part 2 UHD,
|
||||
// 2026-05-08) dd via /dev/sr0 recovers ~50% of bad
|
||||
// sectors that a single-shot SG_IO READ misses.
|
||||
#[cfg(target_os = "linux")]
|
||||
if recovery {
|
||||
if let Some(fd) = self.block_dev_fd {
|
||||
let len = count as usize * 2048;
|
||||
if buf.len() >= len {
|
||||
let offset = lba as i64 * 2048;
|
||||
// Drop kernel cache for this region so we get
|
||||
// a fresh device read, not stale page-cache
|
||||
// data from a prior successful neighbour read.
|
||||
let _ = unsafe {
|
||||
libc::posix_fadvise(
|
||||
fd,
|
||||
offset,
|
||||
len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
)
|
||||
};
|
||||
let n = unsafe {
|
||||
libc::pread(fd, buf.as_mut_ptr() as *mut libc::c_void, len, offset)
|
||||
};
|
||||
if n == len as isize {
|
||||
tracing::info!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
bytes = len,
|
||||
"Drive::read recovered via /dev/sr0 pread fallback"
|
||||
);
|
||||
return Ok(len);
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
pread_ret = n as i64,
|
||||
errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0),
|
||||
"/dev/sr0 pread fallback also failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return Ok(result.bytes_transferred);
|
||||
}
|
||||
Err(Error::Halted) => return Err(Error::Halted),
|
||||
Err(_) => { /* fall through to recovery */ }
|
||||
}
|
||||
|
||||
if !recovery {
|
||||
return Err(Error::DiscRead { sector: lba as u64 });
|
||||
}
|
||||
|
||||
// Enter recovery
|
||||
self.emit(EventKind::ReadError {
|
||||
sector: lba as u64,
|
||||
error: Error::DiscRead { sector: lba as u64 },
|
||||
});
|
||||
self.emit(EventKind::SpeedChange { speed_kbs: 0 });
|
||||
self.set_speed(0);
|
||||
|
||||
// Phase 1: gentle — sleep 30 s, retry. 5 times.
|
||||
for attempt in 1..=5u32 {
|
||||
self.emit(EventKind::Retry { attempt });
|
||||
self.checked_sleep(std::time::Duration::from_secs(30))?;
|
||||
if let Ok(result) =
|
||||
self.checked_exec(&cdb, crate::scsi::DataDirection::FromDevice, buf, 30_000)
|
||||
{
|
||||
self.emit(EventKind::SectorRecovered { sector: lba as u64 });
|
||||
self.recovery_bytes_remaining = RECOVERY_WINDOW;
|
||||
return Ok(result.bytes_transferred);
|
||||
Err(Error::DiscRead {
|
||||
sector: lba as u64,
|
||||
status: Some(status),
|
||||
sense,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: fresh start — close, reset, open, init.
|
||||
let device = std::path::PathBuf::from(&self.device_path);
|
||||
self.checked_sleep(std::time::Duration::from_secs(5))?;
|
||||
let _ = crate::scsi::reset(&device);
|
||||
self.checked_sleep(std::time::Duration::from_secs(5))?;
|
||||
self.scsi = crate::scsi::open(&device)?;
|
||||
let _ = self.init();
|
||||
let _ = self.wait_ready();
|
||||
self.set_speed(0);
|
||||
|
||||
// Phase 3: gentle again on fresh connection — sleep 30 s, retry. 5 times.
|
||||
for attempt in 6..=10u32 {
|
||||
self.emit(EventKind::Retry { attempt });
|
||||
self.checked_sleep(std::time::Duration::from_secs(30))?;
|
||||
if let Ok(result) =
|
||||
self.checked_exec(&cdb, crate::scsi::DataDirection::FromDevice, buf, 30_000)
|
||||
{
|
||||
self.emit(EventKind::SectorRecovered { sector: lba as u64 });
|
||||
self.recovery_bytes_remaining = RECOVERY_WINDOW;
|
||||
return Ok(result.bytes_transferred);
|
||||
}
|
||||
}
|
||||
|
||||
// Both phases failed.
|
||||
self.recovery_bytes_remaining = RECOVERY_WINDOW;
|
||||
Err(Error::DiscRead { sector: lba as u64 })
|
||||
}
|
||||
|
||||
/// Read the disc capacity in sectors (2048 bytes each).
|
||||
@@ -724,10 +649,65 @@ impl Drop for Drive {
|
||||
fn drop(&mut self) {
|
||||
self.cleanup();
|
||||
// SgIoTransport::drop() runs next, calling libc::close(fd)
|
||||
#[cfg(target_os = "linux")]
|
||||
if let Some(fd) = self.block_dev_fd.take() {
|
||||
unsafe { libc::close(fd) };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for Drive {
|
||||
/// Resolve a `/dev/sg*` path to the corresponding `/dev/sr*` block
|
||||
/// device by walking sysfs, then open it for read (no `O_DIRECT` —
|
||||
/// `posix_fadvise(POSIX_FADV_DONTNEED)` flushes the cache before each
|
||||
/// pread, which avoids buffer-alignment requirements while still
|
||||
/// forcing fresh device reads).
|
||||
///
|
||||
/// Returns `None` on any error (sysfs not present, no matching block
|
||||
/// device, open failed). Callers treat that as "no fallback available"
|
||||
/// and propagate the original SCSI READ error.
|
||||
#[cfg(target_os = "linux")]
|
||||
fn open_block_device_for_sg(sg_path: &Path) -> Option<std::os::unix::io::RawFd> {
|
||||
let basename = sg_path.file_name()?.to_str()?;
|
||||
if !basename.starts_with("sg") {
|
||||
return None;
|
||||
}
|
||||
let sysfs_dir = format!("/sys/class/scsi_generic/{}/device/block", basename);
|
||||
let entries = std::fs::read_dir(&sysfs_dir).ok()?;
|
||||
let block_name = entries
|
||||
.flatten()
|
||||
.find_map(|e| e.file_name().into_string().ok())?;
|
||||
let block_path = format!("/dev/{}", block_name);
|
||||
|
||||
let mut bytes = block_path.as_bytes().to_vec();
|
||||
bytes.push(0);
|
||||
let fd = unsafe {
|
||||
libc::open(
|
||||
bytes.as_ptr() as *const libc::c_char,
|
||||
libc::O_RDONLY | libc::O_CLOEXEC,
|
||||
)
|
||||
};
|
||||
if fd < 0 {
|
||||
tracing::debug!(
|
||||
target: "freemkv::drive",
|
||||
sg = basename,
|
||||
block_path,
|
||||
errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0),
|
||||
"Failed to open block device for fallback; sr0 fallback disabled"
|
||||
);
|
||||
None
|
||||
} else {
|
||||
tracing::info!(
|
||||
target: "freemkv::drive",
|
||||
sg = basename,
|
||||
block_path,
|
||||
fd,
|
||||
"Opened /dev/sr* as recovery fallback for failed SCSI reads"
|
||||
);
|
||||
Some(fd)
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for Drive {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
@@ -737,13 +717,28 @@ impl SectorReader for Drive {
|
||||
) -> Result<usize> {
|
||||
self.read(lba, count, buf, recovery)
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
Drive::set_speed(self, kbs);
|
||||
}
|
||||
}
|
||||
|
||||
/// Halt-aware sleep primitive. Returns `Err(Halted)` if the flag is set
|
||||
/// before or during the wait. Extracted as a free function so it can be
|
||||
/// unit-tested without constructing a full `Drive`.
|
||||
/// Find the first optical drive on this system and open it.
|
||||
///
|
||||
/// Wakes within ~100 ms of a halt, regardless of the requested duration.
|
||||
/// For just listing drives without opening (e.g. UI sidebar), use
|
||||
/// `scsi::list_drives()` — that returns `DriveInfo` (path + identity)
|
||||
/// without the cost of running every drive's profile + identity probe.
|
||||
pub fn find_drive() -> Option<Drive> {
|
||||
discover_drives()
|
||||
.into_iter()
|
||||
.find_map(|(path, _)| Drive::open(std::path::Path::new(&path)).ok())
|
||||
}
|
||||
|
||||
/// Halt-aware sleep primitive — wakes within ~100 ms of `halt` flipping
|
||||
/// to true. Kept for the unit tests that cover the slicing behaviour;
|
||||
/// production code paths no longer sleep on the recovery hot path
|
||||
/// (recovery loop removed in 0.13.6).
|
||||
#[cfg(test)]
|
||||
fn sleep_until_halted(halt: &AtomicBool, total: std::time::Duration) -> Result<()> {
|
||||
const SLICE: std::time::Duration = std::time::Duration::from_millis(100);
|
||||
let deadline = std::time::Instant::now() + total;
|
||||
@@ -760,21 +755,6 @@ fn sleep_until_halted(halt: &AtomicBool, total: std::time::Duration) -> Result<(
|
||||
}
|
||||
}
|
||||
|
||||
/// Find all optical drives connected to this system.
|
||||
/// Returns opened Drive objects ready for use.
|
||||
pub fn find_drives() -> Vec<Drive> {
|
||||
discover_drives()
|
||||
.into_iter()
|
||||
.filter_map(|(path, _)| Drive::open(std::path::Path::new(&path)).ok())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Find the first optical drive.
|
||||
/// Returns an opened Drive ready for use.
|
||||
pub fn find_drive() -> Option<Drive> {
|
||||
find_drives().into_iter().next()
|
||||
}
|
||||
|
||||
/// Internal: discover drive paths + IDs without opening full Drive objects.
|
||||
fn discover_drives() -> Vec<(String, DriveId)> {
|
||||
#[cfg(target_os = "linux")]
|
||||
|
||||
+126
-9
@@ -155,10 +155,24 @@ pub enum Error {
|
||||
},
|
||||
|
||||
// SCSI (4xxx)
|
||||
/// SCSI command failed.
|
||||
///
|
||||
/// `opcode` is the failing CDB byte 0. `status` is the raw SCSI
|
||||
/// status byte: `0x02` = CHECK CONDITION (drive replied with sense
|
||||
/// data), `0xFF` = libfreemkv-synthesised sentinel meaning "no SCSI
|
||||
/// status delivered" (kernel timeout, USB bridge wedge, IOKit
|
||||
/// service failure). `sense` carries the drive's SPC-4 sense triple
|
||||
/// when the drive replied; `None` for transport-layer failures.
|
||||
///
|
||||
/// Recommended dispatch (callers shouldn't pattern-match raw
|
||||
/// fields):
|
||||
/// - [`Error::is_scsi_transport_failure`] — bail; bridge/transport wedge
|
||||
/// - [`Error::is_marginal_read`] — drive said this read was marginal; smaller block may recover
|
||||
/// - [`Error::scsi_sense`] — borrow the sense triple for finer routing ([`ScsiSense::is_medium_error`] etc.)
|
||||
ScsiError {
|
||||
opcode: u8,
|
||||
status: u8,
|
||||
sense_key: u8,
|
||||
sense: Option<crate::scsi::ScsiSense>,
|
||||
},
|
||||
|
||||
// I/O (5xxx)
|
||||
@@ -169,6 +183,8 @@ pub enum Error {
|
||||
// Disc format (6xxx)
|
||||
DiscRead {
|
||||
sector: u64,
|
||||
status: Option<u8>,
|
||||
sense: Option<crate::scsi::ScsiSense>,
|
||||
},
|
||||
/// Drive was halted by caller.
|
||||
Halted,
|
||||
@@ -363,19 +379,46 @@ impl std::fmt::Display for Error {
|
||||
Error::ScsiError {
|
||||
opcode,
|
||||
status,
|
||||
sense_key,
|
||||
} => {
|
||||
write!(
|
||||
sense,
|
||||
} => match sense {
|
||||
Some(s) => write!(
|
||||
f,
|
||||
"E{}: 0x{:02x}/0x{:02x}/0x{:02x}",
|
||||
"E{}: 0x{:02x}/0x{:02x}/0x{:02x}/0x{:02x}/0x{:02x}",
|
||||
self.code(),
|
||||
opcode,
|
||||
status,
|
||||
sense_key
|
||||
)
|
||||
}
|
||||
s.sense_key,
|
||||
s.asc,
|
||||
s.ascq,
|
||||
),
|
||||
None => write!(f, "E{}: 0x{:02x}/0x{:02x}", self.code(), opcode, status,),
|
||||
},
|
||||
Error::IoError { source } => write!(f, "E{}: {}", self.code(), source),
|
||||
Error::DiscRead { sector } => write!(f, "E{}: {}", self.code(), sector),
|
||||
Error::DiscRead {
|
||||
sector,
|
||||
status,
|
||||
sense,
|
||||
} => match (status, sense) {
|
||||
(Some(st), Some(s)) => write!(
|
||||
f,
|
||||
"E{}: {} 0x{:02x}/0x{:02x}/0x{:02x}",
|
||||
self.code(),
|
||||
sector,
|
||||
st,
|
||||
s.sense_key,
|
||||
s.asc,
|
||||
),
|
||||
(Some(st), None) => write!(f, "E{}: {} 0x{:02x}", self.code(), sector, st,),
|
||||
(None, Some(s)) => write!(
|
||||
f,
|
||||
"E{}: {} 0x{:02x}/0x{:02x}",
|
||||
self.code(),
|
||||
sector,
|
||||
s.sense_key,
|
||||
s.asc,
|
||||
),
|
||||
(None, None) => write!(f, "E{}: {}", self.code(), sector),
|
||||
},
|
||||
Error::Halted => write!(f, "E{}", self.code()),
|
||||
Error::UdfNotFound { path } => write!(f, "E{}: {}", self.code(), path),
|
||||
Error::DiscTitleRange { index, count } => {
|
||||
@@ -438,6 +481,80 @@ impl From<Error> for std::io::Error {
|
||||
/// Convenience alias for `Result<T, Error>`.
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
impl Error {
|
||||
/// Borrow the drive-returned SPC-4 sense triple if this error is a
|
||||
/// [`Error::ScsiError`] carrying sense data. `None` for any other
|
||||
/// variant **and** for `ScsiError`s that represent a transport-layer
|
||||
/// failure (where the device never delivered a SCSI status reply, so
|
||||
/// no sense data exists).
|
||||
pub fn scsi_sense(&self) -> Option<&crate::scsi::ScsiSense> {
|
||||
match self {
|
||||
Error::ScsiError { sense: Some(s), .. } => Some(s),
|
||||
Error::DiscRead { sense: Some(s), .. } => Some(s),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// True if this is a [`Error::ScsiError`] representing a transport-layer
|
||||
/// failure — kernel timeout, USB bridge wedge, IOKit service error.
|
||||
/// The device never delivered a SCSI status reply, so there is no
|
||||
/// sense data to inspect; retrying typically requires physical
|
||||
/// intervention (replug).
|
||||
pub fn is_scsi_transport_failure(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Error::ScsiError {
|
||||
status: crate::scsi::SCSI_STATUS_TRANSPORT_FAILURE,
|
||||
..
|
||||
}
|
||||
) || matches!(
|
||||
self,
|
||||
Error::DiscRead {
|
||||
status: Some(crate::scsi::SCSI_STATUS_TRANSPORT_FAILURE),
|
||||
..
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
/// True if this error indicates bridge degradation — the SCSI status
|
||||
/// is neither GOOD (0x00), CHECK CONDITION (0x02), nor transport failure
|
||||
/// (0xFF). Observed on the Initio INIC-1618L USB bridge preceding a full
|
||||
/// crash: the bridge firmware returns non-standard status bytes (e.g.
|
||||
/// 0x04, 0x05) with empty sense data. The caller should cool down
|
||||
/// (10 s pause) and retry rather than hammering the bridge.
|
||||
pub fn is_bridge_degradation(&self) -> bool {
|
||||
let status = match self {
|
||||
Error::ScsiError { status, .. } => *status,
|
||||
Error::DiscRead { status, .. } => status.unwrap_or(0),
|
||||
_ => return false,
|
||||
};
|
||||
status != crate::scsi::SCSI_STATUS_GOOD
|
||||
&& status != crate::scsi::SCSI_STATUS_CHECK_CONDITION
|
||||
&& status != crate::scsi::SCSI_STATUS_TRANSPORT_FAILURE
|
||||
}
|
||||
|
||||
/// True if the underlying SCSI failure is a *marginal read* — the
|
||||
/// drive returned an error category in which smaller-granularity
|
||||
/// retries can sometimes recover the data:
|
||||
///
|
||||
/// - MEDIUM ERROR (sense key 3) — canonical bad-sector signal
|
||||
/// - ABORTED COMMAND (sense key B) — transient; retry usually works
|
||||
/// - RECOVERED ERROR (sense key 1) / NO SENSE (sense key 0) — not
|
||||
/// classified as fatal; treat as recoverable
|
||||
///
|
||||
/// Returns `false` for transport failures (no sense data delivered),
|
||||
/// HARDWARE ERROR, DATA PROTECT, UNIT ATTENTION, ILLEGAL
|
||||
/// REQUEST, BLANK CHECK, kernel `IoError`, and any non-SCSI variant.
|
||||
/// Caller-agnostic predicate — describes a property of the *error*,
|
||||
/// not what one specific call site should do with it. Used by
|
||||
/// `Disc::copy`'s hysteresis dispatch.
|
||||
pub fn is_marginal_read(&self) -> bool {
|
||||
self.scsi_sense()
|
||||
.map(crate::scsi::ScsiSense::is_marginal)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
//! Smoke tests for the error code → variant mapping. Each new variant
|
||||
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
//! One-bit cooperative cancellation flag.
|
||||
//!
|
||||
//! `Halt` is a clonable token wrapping `Arc<AtomicBool>`. Pass clones into
|
||||
//! every long-running loop; the loop polls `is_cancelled()` and bails out
|
||||
//! cleanly. Calling `cancel()` from any clone flips the shared flag, and
|
||||
//! every other clone observes it on its next poll.
|
||||
//!
|
||||
//! Why: `Ordering::Relaxed` is sufficient on both load and store because
|
||||
//! this flag is purely advisory — no other memory operations piggyback on
|
||||
//! it for happens-before ordering. Callers that need to publish data
|
||||
//! across threads do so via channels or other synchronization, not via
|
||||
//! this bit.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
/// Clonable, infallible cooperative-cancellation token.
|
||||
///
|
||||
/// Clones share the same underlying flag. `cancel()` is one-way; there is
|
||||
/// no `reset()` by design — construct a fresh `Halt` for a fresh
|
||||
/// operation.
|
||||
///
|
||||
/// Construct with [`Halt::new`] (or [`Halt::default`]). The `Default`
|
||||
/// impl forwards to `new()` — both produce a fresh, uncancelled token.
|
||||
/// The pair exists because clippy's `new_without_default` lint requires
|
||||
/// `Default` whenever a public `new()` is present, even when the two
|
||||
/// would do exactly the same thing.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Halt(Arc<AtomicBool>);
|
||||
|
||||
impl Halt {
|
||||
/// Construct a fresh, uncancelled token.
|
||||
pub fn new() -> Self {
|
||||
Self(Arc::new(AtomicBool::new(false)))
|
||||
}
|
||||
|
||||
/// Wrap an existing `Arc<AtomicBool>` as a `Halt`. Useful as a
|
||||
/// bridge during the 0.18 deprecation window: callers that already
|
||||
/// hold an `Arc<AtomicBool>` (e.g. `Drive::halt_flag()`, the
|
||||
/// deprecated `DiscStream::set_halt`) can adopt the new token API
|
||||
/// without changing the underlying flag.
|
||||
///
|
||||
/// Cancelling either side flips the same bit — the wrapping `Halt`
|
||||
/// and the original `Arc` are two views over one shared flag.
|
||||
pub fn from_arc(flag: Arc<AtomicBool>) -> Self {
|
||||
Self(flag)
|
||||
}
|
||||
|
||||
/// Borrow the underlying `Arc<AtomicBool>`. Used at boundaries with
|
||||
/// pre-`Halt` APIs that still take an `Arc<AtomicBool>` directly
|
||||
/// (`CopyOptions::halt`, the deprecated `DiscStream::set_halt`).
|
||||
/// Round 3 deletes those boundaries and this accessor with them.
|
||||
pub fn as_arc(&self) -> &Arc<AtomicBool> {
|
||||
&self.0
|
||||
}
|
||||
|
||||
/// Flip the shared flag to cancelled. Idempotent.
|
||||
pub fn cancel(&self) {
|
||||
self.0.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Read the shared flag.
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.0.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Halt {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared poll interval for halt-aware loops.
|
||||
///
|
||||
/// `bounded_syscall` checks the cancellation flag and the deadline
|
||||
/// every [`POLL_INTERVAL`] while blocked on a worker; the same
|
||||
/// cadence governs `Pipeline::send_with_halt`'s `try_send` retry.
|
||||
/// 250 ms is the sweet spot between responsiveness (operator presses
|
||||
/// Stop, sees it take effect within ~quarter-second) and waste
|
||||
/// (atomic load + clock read is cheap but not free at thousands of
|
||||
/// hertz).
|
||||
///
|
||||
/// Centralised here so the half-dozen halt-polling loops across `io`
|
||||
/// can't drift apart silently.
|
||||
pub const POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(250);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fresh_is_not_cancelled() {
|
||||
let h = Halt::new();
|
||||
assert!(!h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_flips_state() {
|
||||
let h = Halt::new();
|
||||
assert!(!h.is_cancelled());
|
||||
h.cancel();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_is_idempotent() {
|
||||
let h = Halt::new();
|
||||
h.cancel();
|
||||
h.cancel();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state() {
|
||||
let original = Halt::new();
|
||||
let cloned = original.clone();
|
||||
assert!(!original.is_cancelled());
|
||||
assert!(!cloned.is_cancelled());
|
||||
|
||||
// Cancel via the clone; the original observes it.
|
||||
cloned.cancel();
|
||||
assert!(original.is_cancelled());
|
||||
assert!(cloned.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state_reverse_direction() {
|
||||
let original = Halt::new();
|
||||
let cloned = original.clone();
|
||||
|
||||
// Cancel via the original; the clone observes it.
|
||||
original.cancel();
|
||||
assert!(cloned.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state_across_threads() {
|
||||
let h = Halt::new();
|
||||
let h2 = h.clone();
|
||||
let handle = std::thread::spawn(move || {
|
||||
h2.cancel();
|
||||
});
|
||||
handle.join().unwrap();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_arc_shares_state() {
|
||||
// The 0.18 deprecation-window bridge: a Halt built from an
|
||||
// existing Arc<AtomicBool> must be a *view* over the same bit,
|
||||
// not a fresh copy. Cancelling either side flips both.
|
||||
let arc = Arc::new(AtomicBool::new(false));
|
||||
let halt = Halt::from_arc(arc.clone());
|
||||
assert!(!halt.is_cancelled());
|
||||
assert!(!arc.load(Ordering::Relaxed));
|
||||
|
||||
// Cancel via the wrapping Halt; the original Arc observes it.
|
||||
halt.cancel();
|
||||
assert!(arc.load(Ordering::Relaxed));
|
||||
|
||||
// Conversely: flip the Arc directly; the Halt view observes it.
|
||||
let arc2 = Arc::new(AtomicBool::new(false));
|
||||
let halt2 = Halt::from_arc(arc2.clone());
|
||||
arc2.store(true, Ordering::Relaxed);
|
||||
assert!(halt2.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn as_arc_returns_backing_flag() {
|
||||
// `as_arc()` must hand back the *same* Arc, not a clone of a
|
||||
// different bit. Verified by writing through the borrowed Arc
|
||||
// and observing through the Halt.
|
||||
let halt = Halt::new();
|
||||
let arc = halt.as_arc().clone();
|
||||
assert!(!halt.is_cancelled());
|
||||
arc.store(true, Ordering::Relaxed);
|
||||
assert!(halt.is_cancelled());
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -9,7 +9,7 @@
|
||||
|
||||
use crate::disc::{Codec, Resolution};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
// ── Public types ────────────────────────────────────────────────────────────
|
||||
@@ -184,7 +184,7 @@ fn bcd_byte(b: u8) -> u32 {
|
||||
///
|
||||
/// Reads the VMG (Video Manager) to discover title sets, then reads each
|
||||
/// VTS IFO to extract PGC chains, cell addresses, and stream attributes.
|
||||
pub fn parse_vmg(reader: &mut dyn SectorReader, udf: &UdfFs) -> Result<DvdInfo> {
|
||||
pub fn parse_vmg(reader: &mut dyn SectorSource, udf: &UdfFs) -> Result<DvdInfo> {
|
||||
let vmg_data = udf.read_file(reader, "/VIDEO_TS/VIDEO_TS.IFO")?;
|
||||
|
||||
// Validate VMG magic
|
||||
@@ -263,7 +263,7 @@ pub fn parse_vmg(reader: &mut dyn SectorReader, udf: &UdfFs) -> Result<DvdInfo>
|
||||
///
|
||||
/// `titles_info` is a list of (chapter_count, vts_title_number) from TT_SRPT.
|
||||
fn parse_vts(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf: &UdfFs,
|
||||
vts_number: u8,
|
||||
titles_info: &[(u16, u8)],
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
//! Bounded-syscall primitive: run a (potentially-blocking) operation
|
||||
//! on a worker thread, with a hard wall-clock deadline and an optional
|
||||
//! cooperative [`Halt`] poll. The calling thread is never trapped
|
||||
//! inside a kernel call.
|
||||
//!
|
||||
//! ## Why this exists
|
||||
//!
|
||||
//! [`crate::halt::Halt`] is cooperative: callers poll
|
||||
//! `is_cancelled()`. It cannot reach inside a syscall the kernel
|
||||
//! currently owns the thread for — `libc::sync_file_range`,
|
||||
//! `libc::fsync`, `File::write` on NFS, and so on. `/api/stop` from
|
||||
//! autorip therefore can't unstick a thread sitting in such a syscall.
|
||||
//!
|
||||
//! [`bounded_syscall`] is the escape hatch: it runs `op` on a fresh
|
||||
//! worker thread, then `recv_timeout`s on a rendezvous channel for the
|
||||
//! result. The wait is broken into ~250 ms slices so the calling
|
||||
//! thread can poll the supplied [`Halt`] in between. If the deadline
|
||||
//! elapses or the halt fires, the worker is intentionally leaked — the
|
||||
//! syscall will unwind whenever the kernel decides, or at process
|
||||
//! exit, but the caller is free to fall back to a degraded code path
|
||||
//! (skip the sync, log loudly, etc.).
|
||||
//!
|
||||
//! ## Trade-offs
|
||||
//!
|
||||
//! - **Thread per call.** Cheap (`std::thread::spawn` is < 100 µs on
|
||||
//! Linux/macOS), but not free. Use on coarse-grained finalisation
|
||||
//! syscalls (`sync_all`, `sync_file_range(WAIT_AFTER)`), not on hot
|
||||
//! inner-loop writes.
|
||||
//! - **Leak on timeout.** A wedged syscall keeps a kernel slot and a
|
||||
//! user-space thread around for the rest of the process's life.
|
||||
//! Bounded by the number of independent rip/mux sessions, which is
|
||||
//! one per disc. The alternative — trapping the caller forever —
|
||||
//! defeats the entire purpose of `/api/stop`.
|
||||
//! - **Halt granularity ~250 ms.** Halt observation is not instant;
|
||||
//! it's the worst-case latency of the `recv_timeout` slice. Good
|
||||
//! enough for human-driven stop requests; not suitable for hard
|
||||
//! real-time deadlines.
|
||||
//!
|
||||
//! ## Single source of truth
|
||||
//!
|
||||
//! Do NOT inline this pattern. Every blocking-syscall wrapper in the
|
||||
//! rip + mux pipeline calls this helper, so changes (e.g. swapping the
|
||||
//! channel impl, adjusting the poll slice, adding metrics) land in one
|
||||
//! place.
|
||||
//!
|
||||
//! ## Platform
|
||||
//!
|
||||
//! Pure `std::thread` + `std::sync::mpsc`. No `cfg(target_os)` needed
|
||||
//! here — the helper itself is platform-agnostic. Callers that wrap
|
||||
//! Linux-only syscalls (`sync_file_range`) still need their own
|
||||
//! `#[cfg(target_os = "linux")]` gates; this helper does not.
|
||||
|
||||
use std::sync::mpsc::{RecvTimeoutError, sync_channel};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::halt::{Halt, POLL_INTERVAL};
|
||||
|
||||
/// Failure outcome from a bounded syscall wrapper.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum BoundedError {
|
||||
/// The user-visible halt token fired during the wait. The worker
|
||||
/// thread is intentionally leaked — the caller should fall back to
|
||||
/// a degraded code path rather than waiting on the syscall to
|
||||
/// return.
|
||||
Halted,
|
||||
/// The deadline elapsed before the syscall returned. Same leak
|
||||
/// semantics as `Halted`.
|
||||
Timeout,
|
||||
/// The worker thread panicked, or its sender disconnected before
|
||||
/// sending a result. Treat as a benign no-op (callers usually
|
||||
/// log and continue) rather than a hard error — by definition no
|
||||
/// syscall observably ran to completion in this case.
|
||||
WorkerLost,
|
||||
}
|
||||
|
||||
/// Run a (potentially-blocking) operation on a worker thread with a
|
||||
/// deadline and an optional cooperative halt-token poll. Returns the
|
||||
/// operation's result if it completes within `timeout`; otherwise one
|
||||
/// of [`BoundedError::Halted`] / [`BoundedError::Timeout`] /
|
||||
/// [`BoundedError::WorkerLost`].
|
||||
///
|
||||
/// On `Halted` / `Timeout` the worker thread is intentionally leaked:
|
||||
/// the syscall will unwind whenever the kernel decides, or when the
|
||||
/// process exits. The calling thread is never trapped inside a kernel
|
||||
/// call.
|
||||
///
|
||||
/// `halt` is polled at [`POLL_INTERVAL`] granularity. Pass `None` for
|
||||
/// callers that don't (yet) have a halt token plumbed through —
|
||||
/// behaviour degrades to deadline-only, matching the 0.20.5
|
||||
/// `wait_after_with_timeout` shape this helper generalises.
|
||||
///
|
||||
/// `op` returns `R: Send + 'static`. The closure must own everything
|
||||
/// it touches because it may outlive this call (timeout / halt cases).
|
||||
pub(crate) fn bounded_syscall<F, R>(
|
||||
halt: Option<&Halt>,
|
||||
timeout: Duration,
|
||||
op: F,
|
||||
) -> Result<R, BoundedError>
|
||||
where
|
||||
F: FnOnce() -> R + Send + 'static,
|
||||
R: Send + 'static,
|
||||
{
|
||||
// Rendezvous channel: the worker sends exactly one value (the
|
||||
// op's return) and then exits. Capacity-0 means the send blocks
|
||||
// until we receive — fine on the happy path; on the timeout /
|
||||
// halt path the receiver is dropped and the worker's send
|
||||
// returns Err, which the worker ignores.
|
||||
let (tx, rx) = sync_channel::<R>(0);
|
||||
let _ = thread::Builder::new()
|
||||
.name("freemkv-bounded-syscall".into())
|
||||
.spawn(move || {
|
||||
// Ignore the send error: if we time out (or get halted)
|
||||
// before the worker finishes, the receiver is dropped
|
||||
// and `tx.send` returns Err. Either way, the worker has
|
||||
// nothing more to do.
|
||||
let _ = tx.send(op());
|
||||
});
|
||||
|
||||
let deadline = Instant::now() + timeout;
|
||||
loop {
|
||||
let now = Instant::now();
|
||||
let remaining = deadline.saturating_duration_since(now);
|
||||
let slice = remaining.min(POLL_INTERVAL);
|
||||
match rx.recv_timeout(slice) {
|
||||
Ok(v) => return Ok(v),
|
||||
Err(RecvTimeoutError::Timeout) => {
|
||||
if let Some(h) = halt {
|
||||
if h.is_cancelled() {
|
||||
return Err(BoundedError::Halted);
|
||||
}
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(BoundedError::Timeout);
|
||||
}
|
||||
// Otherwise: another slice.
|
||||
}
|
||||
Err(RecvTimeoutError::Disconnected) => {
|
||||
// Worker thread spawn failed, or it panicked before
|
||||
// sending. Caller treats this as "no syscall ran" —
|
||||
// typically a no-op + log.
|
||||
return Err(BoundedError::WorkerLost);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
#[test]
|
||||
fn op_completes_quickly() {
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), || 42u32);
|
||||
assert!(matches!(r, Ok(42)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn op_exceeds_timeout() {
|
||||
// Op sleeps longer than the deadline → Timeout.
|
||||
let r = bounded_syscall(None, Duration::from_millis(300), || {
|
||||
thread::sleep(Duration::from_secs(2));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Timeout)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_fires_during_wait() {
|
||||
let halt = Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
// Flip the halt from a side thread after ~300 ms — long
|
||||
// enough that the receive loop has rolled at least one
|
||||
// 250 ms slice and is sitting in `recv_timeout` again when
|
||||
// the bit flips.
|
||||
thread::spawn(move || {
|
||||
thread::sleep(Duration::from_millis(300));
|
||||
halt2.cancel();
|
||||
});
|
||||
let r = bounded_syscall(Some(&halt), Duration::from_secs(5), || {
|
||||
thread::sleep(Duration::from_secs(5));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Halted)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_panics() {
|
||||
// Worker panics → sender drops without sending → recv sees
|
||||
// Disconnected → WorkerLost. We use an explicit panic in the
|
||||
// op closure rather than `panic!()` from inside the channel
|
||||
// machinery; the spawned thread's panic is contained (no
|
||||
// process abort) because we don't `.join()` it.
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), || -> u32 {
|
||||
panic!("intentional test panic");
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::WorkerLost)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_already_set_before_call_still_returns_halted() {
|
||||
// Halt observed on the very first poll slice. The op blocks
|
||||
// forever; we must not wait the full timeout to notice the
|
||||
// halt is already set.
|
||||
let halt = Halt::new();
|
||||
halt.cancel();
|
||||
let started = Instant::now();
|
||||
let r = bounded_syscall(Some(&halt), Duration::from_secs(10), || {
|
||||
thread::sleep(Duration::from_secs(10));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Halted)));
|
||||
// Should bail out within ~1 s; allow 2 s of slack for slow
|
||||
// CI hosts.
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(2),
|
||||
"halt-already-set took {:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ok_path_takes_no_halt_token() {
|
||||
// Sanity: the `None` halt path is the documented zero-config
|
||||
// form (matches the 0.20.5 `wait_after_with_timeout`
|
||||
// behaviour). Op returns immediately; we must observe Ok.
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let f2 = flag.clone();
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), move || {
|
||||
f2.store(true, Ordering::Relaxed);
|
||||
"ok"
|
||||
});
|
||||
assert!(matches!(r, Ok("ok")));
|
||||
assert!(flag.load(Ordering::Relaxed));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
//! Byte-sized bounded producer/consumer channel.
|
||||
//!
|
||||
//! Wraps `std::sync::mpsc::sync_channel` with a byte-accounting
|
||||
//! `Mutex<usize> + Condvar` cap. Sender blocks (cooperatively) when
|
||||
//! `used_bytes + item.byte_size() > capacity_bytes`. Receiver
|
||||
//! decrements `used_bytes` when it takes the item.
|
||||
//!
|
||||
//! Why: the existing producer→consumer channel between `DiscStream`
|
||||
//! (PES producer) and `MuxSink` (PES consumer) is bounded by frame
|
||||
//! count. Frame sizes vary 100× between metadata and keyframes, so a
|
||||
//! count-based cap either starves on small frames or buffers far too
|
||||
//! much memory on big ones. Byte-sized accounting sizes the buffer for
|
||||
//! the worst-case input stall (NFS read p99 ≈ 1–2 s × ~15 MB/s peak
|
||||
//! compressed bitrate ≈ ~30 MB) directly.
|
||||
//!
|
||||
//! The underlying mpsc channel is created with a very large slot count
|
||||
//! so the byte cap (not the slot count) is the real backpressure. Slot
|
||||
//! count is only there to give the kernel a small chunk to wake on.
|
||||
//!
|
||||
//! See `(internal)/memory/project_buffering_architecture.md` §
|
||||
//! Pipeline channel — sizing.
|
||||
|
||||
use std::sync::mpsc::{Receiver as MpscReceiver, RecvError, SendError, SyncSender, sync_channel};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
|
||||
/// Default byte cap for the muxer's input channel. Sized to hide a
|
||||
/// worst-case ~2 s NFS read refill at UHD peak compressed bitrate
|
||||
/// (~15 MB/s); 64 MiB gives headroom. Tweakable; not magic.
|
||||
pub const BYTE_CHANNEL_DEFAULT_CAPACITY: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// Slot capacity of the inner `sync_channel`. Large so the byte cap is
|
||||
/// the real backpressure mechanism — the mpsc slot count only exists
|
||||
/// to give the kernel a chunk to wake on. PES frames are typically
|
||||
/// ~700 B each, so 64 MiB ≈ 90 k frames; 200 k is comfortable headroom.
|
||||
const INNER_SLOT_CAPACITY: usize = 200_000;
|
||||
|
||||
/// Anything whose in-memory cost can be accounted by a single
|
||||
/// `usize`. Implement on the item type sent through [`Sender`].
|
||||
pub trait HasByteSize {
|
||||
/// Bytes this item contributes to the channel's used budget.
|
||||
/// Must be > 0 to make progress (a 0-byte item would never
|
||||
/// block the sender no matter the cap; see send_blocks_at_capacity
|
||||
/// test).
|
||||
fn byte_size(&self) -> usize;
|
||||
}
|
||||
|
||||
impl HasByteSize for crate::pes::PesFrame {
|
||||
fn byte_size(&self) -> usize {
|
||||
// Frame data + the fixed header overhead the serializer
|
||||
// writes (track + pts + keyframe + len). The `Vec<u8>` heap
|
||||
// allocation also has alloc-header overhead but that's
|
||||
// <0.1 % at typical frame sizes — folding it in would just
|
||||
// add noise to the budget.
|
||||
self.data.len() + 14
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared book-keeping between [`Sender`] and [`Receiver`]. Wrapped in
|
||||
/// an `Arc` because both halves hold it independently.
|
||||
struct Accounting {
|
||||
used: Mutex<usize>,
|
||||
cv: Condvar,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
/// Send half of the byte-bounded channel.
|
||||
///
|
||||
/// `send` blocks (on a `Condvar`) when adding the item would push
|
||||
/// `used_bytes` past `capacity_bytes`. Unblocks when the receiver
|
||||
/// `recv`s items out and notifies. Returns `Err(item)` if the
|
||||
/// receiver has been dropped — mirrors `mpsc::SyncSender::send`.
|
||||
pub struct Sender<T: HasByteSize> {
|
||||
tx: SyncSender<T>,
|
||||
acct: Arc<Accounting>,
|
||||
}
|
||||
|
||||
impl<T: HasByteSize> Clone for Sender<T> {
|
||||
fn clone(&self) -> Self {
|
||||
Sender {
|
||||
tx: self.tx.clone(),
|
||||
acct: self.acct.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: HasByteSize> Sender<T> {
|
||||
/// Push one item. Blocks until adding it would not exceed the
|
||||
/// capacity, then sends through the inner mpsc channel.
|
||||
pub fn send(&self, item: T) -> Result<(), SendError<T>> {
|
||||
let sz = item.byte_size();
|
||||
// Reserve capacity first. The reservation is observable to
|
||||
// other senders via `used`; only after we win the slot do we
|
||||
// hand the item to the inner mpsc channel. That ordering means
|
||||
// `used` is always a conservative upper bound on what's in the
|
||||
// mpsc queue + about-to-be-sent.
|
||||
{
|
||||
let mut used = self.acct.used.lock().expect("byte_channel poisoned");
|
||||
// An item bigger than the whole capacity will never fit; let
|
||||
// it through anyway as a one-shot reservation, otherwise the
|
||||
// sender deadlocks forever waiting for `used == 0` AND
|
||||
// nothing in flight. The receiver will drain it on the
|
||||
// other side. Same behaviour as `std::sync::mpsc` for
|
||||
// arbitrarily large messages.
|
||||
while *used + sz > self.acct.capacity && *used > 0 {
|
||||
used = self.acct.cv.wait(used).expect("byte_channel cv poisoned");
|
||||
}
|
||||
*used += sz;
|
||||
}
|
||||
match self.tx.send(item) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(SendError(returned)) => {
|
||||
// Receiver dropped — refund the reservation so a later
|
||||
// sender on a clone doesn't observe phantom used bytes
|
||||
// (the receiver is gone so nobody will decrement).
|
||||
let mut used = self.acct.used.lock().expect("byte_channel poisoned");
|
||||
*used = used.saturating_sub(sz);
|
||||
self.acct.cv.notify_all();
|
||||
Err(SendError(returned))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Receive half of the byte-bounded channel.
|
||||
///
|
||||
/// `recv` blocks on the inner mpsc until an item is available, then
|
||||
/// decrements the byte-accounting and wakes any sender waiting on
|
||||
/// capacity.
|
||||
pub struct Receiver<T: HasByteSize> {
|
||||
rx: MpscReceiver<T>,
|
||||
acct: Arc<Accounting>,
|
||||
}
|
||||
|
||||
impl<T: HasByteSize> Receiver<T> {
|
||||
/// Take the next item. Returns `Err(RecvError)` when all senders
|
||||
/// have been dropped and the channel is empty.
|
||||
pub fn recv(&self) -> Result<T, RecvError> {
|
||||
let item = self.rx.recv()?;
|
||||
let sz = item.byte_size();
|
||||
let mut used = self.acct.used.lock().expect("byte_channel poisoned");
|
||||
*used = used.saturating_sub(sz);
|
||||
// Notify all so multi-sender setups wake every blocked sender,
|
||||
// not just one. Wasted wakeups are cheap; missed wakeups would
|
||||
// be a deadlock.
|
||||
self.acct.cv.notify_all();
|
||||
Ok(item)
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a byte-bounded channel with the given capacity in bytes.
|
||||
/// Returns a `(Sender, Receiver)` pair; clone the `Sender` for
|
||||
/// multi-producer setups.
|
||||
pub fn channel<T: HasByteSize>(capacity_bytes: usize) -> (Sender<T>, Receiver<T>) {
|
||||
let (tx, rx) = sync_channel::<T>(INNER_SLOT_CAPACITY);
|
||||
let acct = Arc::new(Accounting {
|
||||
used: Mutex::new(0),
|
||||
cv: Condvar::new(),
|
||||
capacity: capacity_bytes,
|
||||
});
|
||||
(
|
||||
Sender {
|
||||
tx,
|
||||
acct: acct.clone(),
|
||||
},
|
||||
Receiver { rx, acct },
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Test payload — its `byte_size` returns whatever we passed at
|
||||
/// construction so capacity math is exact and predictable.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
struct Item {
|
||||
sz: usize,
|
||||
tag: u32,
|
||||
}
|
||||
|
||||
impl HasByteSize for Item {
|
||||
fn byte_size(&self) -> usize {
|
||||
self.sz
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_recv_round_trip() {
|
||||
let (tx, rx) = channel::<Item>(1024);
|
||||
for i in 0..5 {
|
||||
tx.send(Item { sz: 100, tag: i }).unwrap();
|
||||
}
|
||||
for i in 0..5 {
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got, Item { sz: 100, tag: i });
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn byte_accounting_decrements_on_recv() {
|
||||
// Internal book-keeping check via observable side-effect: after
|
||||
// sending K items totalling N bytes and receiving them all, a
|
||||
// subsequent send of an N-byte item must NOT block (no items
|
||||
// in flight, all capacity refunded).
|
||||
let (tx, rx) = channel::<Item>(1024);
|
||||
for _ in 0..4 {
|
||||
tx.send(Item { sz: 256, tag: 0 }).unwrap();
|
||||
}
|
||||
for _ in 0..4 {
|
||||
rx.recv().unwrap();
|
||||
}
|
||||
// Cap is now fully available again. Send a 1024-byte item; the
|
||||
// `used > 0` guard means it goes through alone (no wait).
|
||||
let start = Instant::now();
|
||||
tx.send(Item { sz: 1024, tag: 99 }).unwrap();
|
||||
assert!(start.elapsed() < Duration::from_millis(100));
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got.tag, 99);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_blocks_at_capacity_unblocks_on_recv() {
|
||||
// Cap = 200 bytes, item = 100 bytes. First two sends fit
|
||||
// exactly; the third must block until a recv frees capacity.
|
||||
let (tx, rx) = channel::<Item>(200);
|
||||
tx.send(Item { sz: 100, tag: 0 }).unwrap();
|
||||
tx.send(Item { sz: 100, tag: 1 }).unwrap();
|
||||
|
||||
let tx2 = tx.clone();
|
||||
let sent_at = Arc::new(Mutex::new(None::<Instant>));
|
||||
let sent_at2 = sent_at.clone();
|
||||
let h = thread::spawn(move || {
|
||||
tx2.send(Item { sz: 100, tag: 2 }).unwrap();
|
||||
*sent_at2.lock().unwrap() = Some(Instant::now());
|
||||
});
|
||||
|
||||
// Give the sender thread a head start; it should be parked in
|
||||
// `cv.wait` because used (200) + 100 > capacity (200).
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
assert!(
|
||||
sent_at.lock().unwrap().is_none(),
|
||||
"third send should be blocked at capacity"
|
||||
);
|
||||
|
||||
// Drain one. Sender wakes and completes.
|
||||
let recv_at = Instant::now();
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got.tag, 0);
|
||||
h.join().unwrap();
|
||||
|
||||
let sent_when = sent_at.lock().unwrap().unwrap();
|
||||
assert!(
|
||||
sent_when >= recv_at,
|
||||
"sender must complete AFTER receiver freed capacity"
|
||||
);
|
||||
|
||||
// Drain the remaining two.
|
||||
assert_eq!(rx.recv().unwrap().tag, 1);
|
||||
assert_eq!(rx.recv().unwrap().tag, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_larger_than_capacity_still_goes_through() {
|
||||
// Pathological case: a single item bigger than the capacity.
|
||||
// The guard `*used > 0` lets it through when the channel is
|
||||
// empty (otherwise the sender deadlocks forever). Matches
|
||||
// `mpsc::SyncSender` semantics for oversize messages.
|
||||
let (tx, rx) = channel::<Item>(100);
|
||||
tx.send(Item { sz: 1000, tag: 7 }).unwrap();
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got, Item { sz: 1000, tag: 7 });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_send_recv_stress() {
|
||||
// 4 sender threads × 1k items each, 1 receiver. Verify byte
|
||||
// accounting stays sane (channel never deadlocks, every item
|
||||
// arrives exactly once) under contention.
|
||||
const SENDERS: u32 = 4;
|
||||
const PER_SENDER: u32 = 1000;
|
||||
const TOTAL: u32 = SENDERS * PER_SENDER;
|
||||
|
||||
let (tx, rx) = channel::<Item>(8 * 1024);
|
||||
let sent = Arc::new(AtomicUsize::new(0));
|
||||
let mut handles = Vec::new();
|
||||
for s in 0..SENDERS {
|
||||
let tx = tx.clone();
|
||||
let sent = sent.clone();
|
||||
handles.push(thread::spawn(move || {
|
||||
for i in 0..PER_SENDER {
|
||||
// Vary item size so accounting actually has to
|
||||
// multiplex differently-sized blockers. 1B → 256B.
|
||||
let sz = 1 + ((i as usize) % 256);
|
||||
tx.send(Item {
|
||||
sz,
|
||||
tag: s * PER_SENDER + i,
|
||||
})
|
||||
.unwrap();
|
||||
sent.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}));
|
||||
}
|
||||
// Drop our local sender so the receiver can eventually see
|
||||
// RecvError once all sender clones are done. Cloning the
|
||||
// sender into each producer means each clone Drop'd separately.
|
||||
drop(tx);
|
||||
|
||||
let mut received = 0u32;
|
||||
while let Ok(_item) = rx.recv() {
|
||||
received += 1;
|
||||
}
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
assert_eq!(received, TOTAL);
|
||||
assert_eq!(sent.load(Ordering::SeqCst) as u32, TOTAL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_after_recv_dropped_returns_err() {
|
||||
let (tx, rx) = channel::<Item>(1024);
|
||||
drop(rx);
|
||||
let r = tx.send(Item { sz: 10, tag: 0 });
|
||||
assert!(r.is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
//! `BytePrefetcher` — `std::io::Read` analogue of
|
||||
//! [`crate::sector::PrefetchedSectorSource`].
|
||||
//!
|
||||
//! Spawns a producer thread that fills a bounded pool of `Vec<u8>`
|
||||
//! chunks from the underlying reader and ships them through a
|
||||
//! channel; the consumer pulls filled chunks, uses them, and sends
|
||||
//! the empty `Vec<u8>` back through a recycle channel so the
|
||||
//! producer can re-fill in place. Result: zero allocations and zero
|
||||
//! cross-thread frees in the steady-state hot loop.
|
||||
//!
|
||||
//! This is the byte-stream half of the freemkv mux highway —
|
||||
//! `BytePrefetcher` feeds [`crate::mux::demux_thread::DemuxThread`]
|
||||
//! for `m2ts://`, `network://`, `stdio://`, and any other stream
|
||||
//! whose source is an `io::Read` rather than a `SectorSource`.
|
||||
|
||||
use crate::halt::Halt;
|
||||
use crossbeam_channel::{Receiver, Sender, bounded};
|
||||
use std::io::Read;
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// Items flowing through the forward channel.
|
||||
pub type Batch = std::io::Result<Vec<u8>>;
|
||||
|
||||
/// Forward channel depth — how many filled buffers the producer can
|
||||
/// stay ahead by. Two is enough to absorb a moderate consumer stall
|
||||
/// without piling up bytes.
|
||||
const FORWARD_DEPTH: usize = 2;
|
||||
|
||||
/// Recycle channel depth = forward + 1 so the producer always has at
|
||||
/// least one buffer to fill while the consumer holds one.
|
||||
const RECYCLE_DEPTH: usize = FORWARD_DEPTH + 1;
|
||||
|
||||
/// Default chunk size — 16 MiB matches the ISO-mux sector batch and
|
||||
/// is large enough that per-chunk overhead is amortised; small
|
||||
/// enough that the in-flight memory footprint stays bounded.
|
||||
pub const DEFAULT_CHUNK_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// Returned from [`BytePrefetcher::into_channels`]. Owns the
|
||||
/// producer-thread join handle so dropping the shell joins the
|
||||
/// producer.
|
||||
pub struct PrefetchShell {
|
||||
producer: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl Drop for PrefetchShell {
|
||||
fn drop(&mut self) {
|
||||
if let Some(h) = self.producer.take() {
|
||||
let _ = h.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawned byte prefetcher. Drop joins the producer thread.
|
||||
pub struct BytePrefetcher {
|
||||
rx: Receiver<Batch>,
|
||||
recycle_tx: Sender<Vec<u8>>,
|
||||
producer: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl BytePrefetcher {
|
||||
/// Spawn the producer thread. `reader` must be `Send` because it
|
||||
/// moves into the thread. `chunk_bytes` is the size of each
|
||||
/// recycled buffer; pick the natural batch size of the
|
||||
/// downstream demuxer (16 MiB for the BD-TS mux pipeline).
|
||||
pub fn new<R: Read + Send + 'static>(
|
||||
mut reader: R,
|
||||
chunk_bytes: usize,
|
||||
halt: Option<Halt>,
|
||||
) -> Self {
|
||||
let (tx, rx) = bounded::<Batch>(FORWARD_DEPTH);
|
||||
let (recycle_tx, recycle_rx) = bounded::<Vec<u8>>(RECYCLE_DEPTH);
|
||||
|
||||
// Seed the recycle pool. Without these the first
|
||||
// `recycle_rx.recv()` would block forever (no consumer has
|
||||
// returned a buffer yet).
|
||||
for _ in 0..RECYCLE_DEPTH {
|
||||
let _ = recycle_tx.send(vec![0u8; chunk_bytes]);
|
||||
}
|
||||
|
||||
let producer = std::thread::Builder::new()
|
||||
.name("freemkv-byte-prefetch".into())
|
||||
.spawn(move || {
|
||||
loop {
|
||||
if halt.as_ref().map(|h| h.is_cancelled()).unwrap_or(false) {
|
||||
return;
|
||||
}
|
||||
let mut buf = match recycle_rx.recv() {
|
||||
Ok(b) => b,
|
||||
Err(_) => return, // consumer dropped both channels
|
||||
};
|
||||
// Re-expose the full extent (previous iteration
|
||||
// may have truncated after a short read).
|
||||
if buf.len() < chunk_bytes {
|
||||
buf.resize(chunk_bytes, 0);
|
||||
} else {
|
||||
// SAFETY: capacity is at least chunk_bytes
|
||||
// after construction.
|
||||
unsafe { buf.set_len(chunk_bytes) };
|
||||
}
|
||||
// Read up to one full chunk. Short reads are
|
||||
// valid and common — pipe `truncate` so the
|
||||
// consumer sees only the bytes that arrived.
|
||||
let n = match reader.read(&mut buf[..]) {
|
||||
Ok(0) => return, // EOF — drop tx, consumer sees RecvError
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
let _ = tx.send(Err(e));
|
||||
return;
|
||||
}
|
||||
};
|
||||
buf.truncate(n);
|
||||
if tx.send(Ok(buf)).is_err() {
|
||||
return; // consumer dropped
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("freemkv-byte-prefetch thread spawn failed");
|
||||
|
||||
Self {
|
||||
rx,
|
||||
recycle_tx,
|
||||
producer: Some(producer),
|
||||
}
|
||||
}
|
||||
|
||||
/// Peel off the channels for zero-copy pipeline consumption. The
|
||||
/// caller (typically [`crate::mux::demux_thread::DemuxThread`])
|
||||
/// drains `rx`, runs the demuxer in place on each filled buffer,
|
||||
/// and recycles back through `recycle_tx`.
|
||||
pub fn into_channels(self) -> (Receiver<Batch>, Sender<Vec<u8>>, PrefetchShell) {
|
||||
let mut me = self;
|
||||
let producer = me.producer.take();
|
||||
let rx = me.rx.clone();
|
||||
let recycle = me.recycle_tx.clone();
|
||||
std::mem::forget(me);
|
||||
(rx, recycle, PrefetchShell { producer })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BytePrefetcher {
|
||||
fn drop(&mut self) {
|
||||
if let Some(h) = self.producer.take() {
|
||||
let _ = h.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
//! Linux read-side platform hooks: sequential-access hint at open +
|
||||
//! periodic page-cache eviction during streaming reads.
|
||||
//!
|
||||
//! ## Why both
|
||||
//!
|
||||
//! `POSIX_FADV_SEQUENTIAL` at open widens the kernel's readahead window
|
||||
//! so each pread aggregates into fewer NFS round-trips. `DONTNEED` on
|
||||
//! the consumed window (called periodically by the caller) drops the
|
||||
//! already-read pages from the page cache so an 85 GB streaming ISO
|
||||
//! read doesn't fill memory and starve concurrent writes (the MKV
|
||||
//! output during mux). Together they mirror the write-side
|
||||
//! WritebackPipeline's policy.
|
||||
//!
|
||||
//! ## History
|
||||
//!
|
||||
//! Pre-Phase-1 (0.20.7 baseline) had both. Phase 1's introduction of
|
||||
//! `FileSectorSource` silently dropped the read-side DONTNEED, and
|
||||
//! 0.21.2's revert of `SEQUENTIAL` (mistakenly attributing a regression
|
||||
//! to it) removed the hint. Net effect: 85 GB of ISO reads pinned in
|
||||
//! the page cache + no readahead widening → mux throughput collapse
|
||||
//! from 18 MB/s historical to 2.7-8 MB/s on 0.21.x. Restored in 0.21.6.
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
pub(super) fn hint_sequential(file: &File, _len_bytes: u64) {
|
||||
// Best-effort: return value ignored. A fadvise failure has no
|
||||
// user-observable consequence.
|
||||
unsafe {
|
||||
libc::posix_fadvise(file.as_raw_fd(), 0, 0, libc::POSIX_FADV_SEQUENTIAL);
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop pages in the half-open byte range `[start, start+len)` from
|
||||
/// the page cache. Called periodically by `read_sectors` to bound the
|
||||
/// read-side page cache pressure.
|
||||
pub(super) fn drop_window(file: &File, start: u64, len: u64) {
|
||||
unsafe {
|
||||
libc::posix_fadvise(
|
||||
file.as_raw_fd(),
|
||||
start as i64,
|
||||
len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Async-prefetch `len` bytes at `offset` into the page cache. The
|
||||
/// kernel `readahead(2)` syscall queues the I/O and returns
|
||||
/// immediately — it does NOT wait for completion. Called right after
|
||||
/// each consumed read so the next batch's I/O overlaps with the
|
||||
/// caller's processing of the current batch (decrypt + demux + mux).
|
||||
///
|
||||
/// Without this hint, with a synchronous demux consumer running at
|
||||
/// ~50 MB/s and a single-spindle disk capable of ~150 MB/s, the disk
|
||||
/// sits idle ~70% of each iteration because kernel readahead alone
|
||||
/// (capped at `/sys/block/<dev>/queue/read_ahead_kb`, default 128 KB)
|
||||
/// can only pre-stage a tiny slice of the next batch. An explicit
|
||||
/// `readahead()` of the same size as the current batch tells the
|
||||
/// kernel to queue the full next-batch read now.
|
||||
pub(super) fn prefetch(file: &File, offset: u64, len: u64) {
|
||||
unsafe {
|
||||
libc::readahead(file.as_raw_fd(), offset as i64, len as usize);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
//! macOS: hint the kernel to prefetch a generous chunk. macOS has no
|
||||
//! direct `POSIX_FADV_SEQUENTIAL` equivalent; the idiomatic hint is
|
||||
//! `fcntl(F_RDADVISE, &radvisory)` describing the byte range you
|
||||
//! intend to read soon. We point it at the whole file (clamped to a
|
||||
//! ceiling so a multi-TB ISO doesn't ask the kernel to prefetch
|
||||
//! everything at once).
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
/// `F_RDADVISE` opcode — not in libc's named constants on all SDKs.
|
||||
const F_RDADVISE: libc::c_int = 44;
|
||||
|
||||
/// Cap on the byte length we pass to `F_RDADVISE`. Asking for a
|
||||
/// multi-GB readahead window is counterproductive — the OS doesn't
|
||||
/// have that much cache to throw at one fd. 64 MiB is generous for
|
||||
/// our use case (sweep, mux) and matches the byte-channel cap so the
|
||||
/// kernel's prefetch ≥ our app-level pipeline depth.
|
||||
const RDADVISE_MAX_BYTES: i64 = 64 * 1024 * 1024;
|
||||
|
||||
/// `radvisory` per `<sys/fcntl.h>`. repr(C) layout is stable.
|
||||
#[repr(C)]
|
||||
struct RadAdvisory {
|
||||
ra_offset: libc::off_t,
|
||||
ra_count: libc::c_int,
|
||||
}
|
||||
|
||||
pub(super) fn hint_sequential(file: &File, len_bytes: u64) {
|
||||
let bytes = (len_bytes as i64).min(RDADVISE_MAX_BYTES);
|
||||
let mut ra = RadAdvisory {
|
||||
ra_offset: 0,
|
||||
ra_count: bytes as libc::c_int,
|
||||
};
|
||||
// Best-effort.
|
||||
unsafe {
|
||||
libc::fcntl(file.as_raw_fd(), F_RDADVISE, &mut ra);
|
||||
}
|
||||
}
|
||||
|
||||
/// macOS has no direct `POSIX_FADV_DONTNEED` equivalent for a byte
|
||||
/// range. `fcntl(F_NOCACHE)` would disable caching globally on the fd
|
||||
/// (too coarse — we want the unread region to still benefit). Best
|
||||
/// approximation: no-op. macOS's unified buffer cache is generally
|
||||
/// less prone to the pin-everything pathology that triggers the
|
||||
/// regression on Linux NFS clients.
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
|
||||
/// Async-prefetch the byte range `[offset, offset+len)`. macOS uses
|
||||
/// the same `fcntl(F_RDADVISE, &radvisory)` primitive as the open-
|
||||
/// time sequential hint, just targeted at a moving window instead of
|
||||
/// the whole file. The kernel queues I/O for the requested range and
|
||||
/// returns immediately.
|
||||
pub(super) fn prefetch(file: &File, offset: u64, len: u64) {
|
||||
let bytes = (len as i64).min(RDADVISE_MAX_BYTES);
|
||||
let mut ra = RadAdvisory {
|
||||
ra_offset: offset as libc::off_t,
|
||||
ra_count: bytes as libc::c_int,
|
||||
};
|
||||
unsafe {
|
||||
libc::fcntl(file.as_raw_fd(), F_RDADVISE, &mut ra);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
//! [`FileSectorSource`] — read 2048-byte sectors from an ISO file on
|
||||
//! disk via direct `seek + read_exact` (`pread`-equivalent) calls,
|
||||
//! letting the kernel's own readahead policy manage prefetch.
|
||||
//!
|
||||
//! ## Why no app-level buffer
|
||||
//!
|
||||
//! Pre-0.21.3 this source held a 32 MiB (later 4 MiB) read-ahead
|
||||
//! buffer to amortise per-sector NFS round-trips. Empirically that
|
||||
//! buffer hurt: 32 MiB refills bursted the NFS TCP connection hard
|
||||
//! enough to starve the concurrent writer, and even a 4 MiB window
|
||||
//! gave the kernel less freedom to pipeline reads with writes. Direct
|
||||
//! pread per call lets Linux's readahead widen as it detects the
|
||||
//! sequential pattern, and naturally interleaves with writeback.
|
||||
//!
|
||||
//! ## DONTNEED on the consumed window
|
||||
//!
|
||||
//! Without page-cache eviction an 85 GB streaming ISO read pins the
|
||||
//! entire file in memory, starves the concurrent writer, and collapses
|
||||
//! mux throughput (observed: 2.7 MB/s mux on 0.21.5 vs. 70 MB/s
|
||||
//! isolated NFS reads). Every [`READ_DROP_CHUNK_BYTES`] of consumed
|
||||
//! bytes we call `posix_fadvise(DONTNEED)` over that window, mirroring
|
||||
//! the write-side [`crate::io::writeback::WritebackPipeline`] policy.
|
||||
//!
|
||||
//! ## Platform open hint
|
||||
//!
|
||||
//! On `open()` each platform issues its "sequential access expected"
|
||||
//! hint so OS-level readahead widens. The hint and the DONTNEED call
|
||||
//! live in per-OS sibling modules ([`linux::hint_sequential`] et al.)
|
||||
//! — no inline `#[cfg]` in this file.
|
||||
//!
|
||||
//! ## Read-ahead prefetch
|
||||
//!
|
||||
//! After every consumed read we issue an OS-level prefetch hint for
|
||||
//! the next equivalent-sized window (`platform::prefetch`). The
|
||||
//! kernel queues that I/O asynchronously and returns immediately, so
|
||||
//! the next batch's read overlaps with the caller's processing of
|
||||
//! the current batch (decrypt + demux + mux). Without this the disk
|
||||
//! sits idle ~70% of each iteration because kernel SEQUENTIAL
|
||||
//! readahead alone (capped at `read_ahead_kb`, default 128 KB) is
|
||||
//! far smaller than our 16 MiB app-level batch.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
mod other;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux as platform;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos as platform;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
use other as platform;
|
||||
#[cfg(target_os = "windows")]
|
||||
use windows as platform;
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::Path;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorSource;
|
||||
|
||||
const SECTOR_SIZE: usize = 2048;
|
||||
|
||||
/// Bytes-read threshold per `posix_fadvise(DONTNEED)` drop on the
|
||||
/// read side. Mirrors `WRITEBACK_CHUNK_BYTES` so the read-side page
|
||||
/// cache stays bounded the same way the write side does.
|
||||
///
|
||||
/// 32 MiB is the empirically tuned value on the rip1 test bed (single
|
||||
/// 7200rpm HDD via SATA): smaller windows (8 / 16 MiB) shorten the
|
||||
/// kernel-readahead overlap and slow the producer; larger windows
|
||||
/// (64 / 128 MiB) let the page cache pin enough of the ISO to
|
||||
/// pressure concurrent writes. Override via `FREEMKV_READ_DROP_CHUNK_MIB`.
|
||||
const READ_DROP_CHUNK_BYTES_DEFAULT: u64 = 32 * 1024 * 1024;
|
||||
|
||||
fn read_drop_chunk_bytes() -> u64 {
|
||||
std::env::var("FREEMKV_READ_DROP_CHUNK_MIB")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.filter(|&n| n > 0)
|
||||
.map(|n| n * 1024 * 1024)
|
||||
.unwrap_or(READ_DROP_CHUNK_BYTES_DEFAULT)
|
||||
}
|
||||
|
||||
/// SectorSource backed by a file (ISO image). Every `read_sectors`
|
||||
/// call is a direct `seek + read_exact` against the underlying file
|
||||
/// — kernel readahead handles prefetch, and every
|
||||
/// [`READ_DROP_CHUNK_BYTES_DEFAULT`] bytes of consumed data the
|
||||
/// platform's `DONTNEED` hook drops the consumed window from the
|
||||
/// page cache to bound memory pressure.
|
||||
pub struct FileSectorSource {
|
||||
file: File,
|
||||
/// Total file size in sectors. Constant after construction;
|
||||
/// surfaced via [`SectorSource::capacity_sectors`].
|
||||
capacity: u32,
|
||||
/// Bytes read since the last DONTNEED drop. Drives the per-
|
||||
/// [`read_drop_chunk_bytes`] page-cache eviction in read_sectors.
|
||||
bytes_read_since_drop: u64,
|
||||
/// File offset at which the current drop window starts. The next
|
||||
/// DONTNEED drops from `drop_window_start` for
|
||||
/// `bytes_read_since_drop` bytes.
|
||||
drop_window_start: u64,
|
||||
/// Cached drop chunk size (resolved from env once at open).
|
||||
drop_chunk_bytes: u64,
|
||||
}
|
||||
|
||||
impl FileSectorSource {
|
||||
/// Open an existing ISO file for reading. Capacity is derived
|
||||
/// from `metadata().len() / 2048`. Returns
|
||||
/// [`Error::IsoTooLarge`] if the file would exceed the 32-bit
|
||||
/// LBA address space (~8 TB).
|
||||
///
|
||||
/// Issues the platform's "sequential access expected" hint on the
|
||||
/// fd (Linux `posix_fadvise(SEQUENTIAL)`, macOS `fcntl(F_RDADVISE)`,
|
||||
/// Windows TODO stub) so the kernel's readahead widens.
|
||||
pub fn open(path: &Path) -> std::io::Result<Self> {
|
||||
let file = File::open(path)?;
|
||||
let len = file.metadata()?.len();
|
||||
let sectors = len / SECTOR_SIZE as u64;
|
||||
if sectors > u32::MAX as u64 {
|
||||
return Err(Error::IsoTooLarge {
|
||||
path: path.to_string_lossy().into_owned(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let capacity = sectors as u32;
|
||||
|
||||
// Best-effort sequential hint. Ignored on platforms without
|
||||
// an equivalent primitive (or where the API exists but the
|
||||
// FS doesn't honour it).
|
||||
platform::hint_sequential(&file, len);
|
||||
|
||||
Ok(Self {
|
||||
file,
|
||||
capacity,
|
||||
bytes_read_since_drop: 0,
|
||||
drop_window_start: 0,
|
||||
drop_chunk_bytes: read_drop_chunk_bytes(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for FileSectorSource {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
out: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
let count = count as u32;
|
||||
let bytes = count as usize * SECTOR_SIZE;
|
||||
debug_assert!(
|
||||
out.len() >= bytes,
|
||||
"FileSectorSource::read_sectors: out len {} < requested {}",
|
||||
out.len(),
|
||||
bytes
|
||||
);
|
||||
if count == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let offset = lba as u64 * SECTOR_SIZE as u64;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.read_exact(&mut out[..bytes])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
|
||||
// Queue the next batch's read with the kernel before the
|
||||
// caller starts processing what we just returned. readahead()
|
||||
// is non-blocking — it queues I/O and returns, so the kernel
|
||||
// pulls those pages into cache while the consumer (decrypt +
|
||||
// demux + mux) runs. Next read_sectors call hits a warm cache.
|
||||
platform::prefetch(&self.file, offset + bytes as u64, bytes as u64);
|
||||
|
||||
// Periodic page-cache eviction on the read side. Without
|
||||
// this, an 85 GB streaming ISO read pins the entire file in
|
||||
// the kernel page cache, which starves concurrent writes and
|
||||
// collapses mux throughput. Mirrors the write-side
|
||||
// WritebackPipeline's DONTNEED policy.
|
||||
self.bytes_read_since_drop += bytes as u64;
|
||||
if self.bytes_read_since_drop >= self.drop_chunk_bytes {
|
||||
let drop_start = self.drop_window_start;
|
||||
let drop_len = self.bytes_read_since_drop;
|
||||
platform::drop_window(&self.file, drop_start, drop_len);
|
||||
self.drop_window_start = drop_start + drop_len;
|
||||
self.bytes_read_since_drop = 0;
|
||||
}
|
||||
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
use tempfile::tempdir;
|
||||
|
||||
/// Build a deterministic ISO of `sectors` sectors where sector `n`
|
||||
/// is filled with the byte pattern `((n & 0xff) as u8)`. Lets us
|
||||
/// verify any sector by content alone.
|
||||
fn make_iso(path: &std::path::Path, sectors: u32) {
|
||||
let mut f = std::fs::File::create(path).unwrap();
|
||||
let mut chunk = vec![0u8; SECTOR_SIZE];
|
||||
for n in 0..sectors {
|
||||
let b = (n & 0xff) as u8;
|
||||
chunk.iter_mut().for_each(|c| *c = b);
|
||||
f.write_all(&chunk).unwrap();
|
||||
}
|
||||
f.flush().unwrap();
|
||||
}
|
||||
|
||||
/// Sectors used by spanning-boundary tests. Pick something that
|
||||
/// exercises multi-megabyte reads without making test ISOs huge.
|
||||
/// 8192 sectors = 16 MiB — large enough to cross any readahead
|
||||
/// chunk size we set the kernel hint to.
|
||||
const TEST_SPAN_SECTORS: u32 = 8192;
|
||||
|
||||
#[test]
|
||||
fn sequential_reads_match_file() {
|
||||
let total = TEST_SPAN_SECTORS * 2 + 17;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("seq.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), total);
|
||||
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
for lba in 0..total {
|
||||
src.read_sectors(lba, 1, &mut got, false).unwrap();
|
||||
let expected = (lba & 0xff) as u8;
|
||||
assert!(
|
||||
got.iter().all(|b| *b == expected),
|
||||
"sector {lba} content mismatch: expected 0x{expected:02x}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_sector_read_across_chunk_boundary() {
|
||||
let total = TEST_SPAN_SECTORS * 2;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("span.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
|
||||
let span_lba = TEST_SPAN_SECTORS - 2;
|
||||
let mut buf4 = vec![0u8; SECTOR_SIZE * 4];
|
||||
src.read_sectors(span_lba, 4, &mut buf4, false).unwrap();
|
||||
for i in 0..4 {
|
||||
let lba = span_lba + i as u32;
|
||||
let expected = (lba & 0xff) as u8;
|
||||
for b in &buf4[i * SECTOR_SIZE..(i + 1) * SECTOR_SIZE] {
|
||||
assert_eq!(*b, expected, "byte mismatch at sub-sector {i}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backward_seek_reads_correct_bytes() {
|
||||
// Read forward then jump back: the SectorSource contract is
|
||||
// byte-correctness regardless of access pattern.
|
||||
let total = TEST_SPAN_SECTORS * 2 + 5;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("back.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
|
||||
src.read_sectors(TEST_SPAN_SECTORS + 1, 1, &mut got, false)
|
||||
.unwrap();
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
assert!(got.iter().all(|b| *b == 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_at_eof_returns_correct_bytes() {
|
||||
// File smaller than the readahead chunk — reads near EOF must
|
||||
// still return correct bytes.
|
||||
let total: u32 = 100;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("small.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), total);
|
||||
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
src.read_sectors(total - 1, 1, &mut got, false).unwrap();
|
||||
let expected = ((total - 1) & 0xff) as u8;
|
||||
assert!(got.iter().all(|b| *b == expected));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_single_read() {
|
||||
// A multi-MB single read must work — the implementation has
|
||||
// no app-level chunking, so this just exercises the direct
|
||||
// pread path on a larger request.
|
||||
let total = TEST_SPAN_SECTORS + 100;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("big.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
let req = (TEST_SPAN_SECTORS + 1) as u16;
|
||||
let req_bytes = req as usize * SECTOR_SIZE;
|
||||
let mut big = vec![0u8; req_bytes];
|
||||
src.read_sectors(0, req, &mut big, false).unwrap();
|
||||
assert!(big[..SECTOR_SIZE].iter().all(|b| *b == 0));
|
||||
let last_lba = req as u32 - 1;
|
||||
let exp = (last_lba & 0xff) as u8;
|
||||
let last_off = (req as usize - 1) * SECTOR_SIZE;
|
||||
assert!(
|
||||
big[last_off..last_off + SECTOR_SIZE]
|
||||
.iter()
|
||||
.all(|b| *b == exp)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_chunk_size_env_override() {
|
||||
// Explicit 8 MiB via env var.
|
||||
// SAFETY: tests in this crate are single-threaded per the
|
||||
// default cargo test harness, but std::env::set_var is
|
||||
// declared `unsafe` since Rust 2024 (it can race with other
|
||||
// threads / TLS). For a test that runs in-process before any
|
||||
// FileSectorSource construction this is safe in practice.
|
||||
unsafe {
|
||||
std::env::set_var("FREEMKV_READ_DROP_CHUNK_MIB", "8");
|
||||
}
|
||||
assert_eq!(read_drop_chunk_bytes(), 8 * 1024 * 1024);
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("FREEMKV_READ_DROP_CHUNK_MIB");
|
||||
}
|
||||
assert_eq!(read_drop_chunk_bytes(), READ_DROP_CHUNK_BYTES_DEFAULT);
|
||||
|
||||
// Garbage env value falls back to default.
|
||||
unsafe {
|
||||
std::env::set_var("FREEMKV_READ_DROP_CHUNK_MIB", "not-a-number");
|
||||
}
|
||||
assert_eq!(read_drop_chunk_bytes(), READ_DROP_CHUNK_BYTES_DEFAULT);
|
||||
unsafe {
|
||||
std::env::remove_var("FREEMKV_READ_DROP_CHUNK_MIB");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
//! Fallback for targets without a known sequential-readahead hint
|
||||
//! (BSDs, illumos, etc.). No-op — reads still work, they just don't
|
||||
//! get the OS-level prefetch widening.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn hint_sequential(_file: &File, _len_bytes: u64) {}
|
||||
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
|
||||
pub(super) fn prefetch(_file: &File, _offset: u64, _len: u64) {}
|
||||
@@ -0,0 +1,30 @@
|
||||
//! Windows: the canonical sequential-access hint is
|
||||
//! `FILE_FLAG_SEQUENTIAL_SCAN` passed to `CreateFile` at open time —
|
||||
//! it cannot be set after the fact via `SetFileInformationByHandle`.
|
||||
//! Routing the open call through this module would mean a custom
|
||||
//! `File::from_raw_handle` plumb for every `FileSectorSource::open`
|
||||
//! caller, which is more invasive than the Phase 1 scope.
|
||||
//!
|
||||
//! TODO: replumb `FileSectorSource::open` to take an
|
||||
//! `OpenOptions`-style builder so the Windows path can flip the flag
|
||||
//! at open time. For now this is a no-op stub.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn hint_sequential(_file: &File, _len_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"FileSectorSource hint_sequential: windows stub (TODO: FILE_FLAG_SEQUENTIAL_SCAN at open)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Windows page-cache eviction is not exposed via a posix_fadvise
|
||||
/// equivalent. The kernel does its own working-set management. No-op
|
||||
/// for now.
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
|
||||
/// Windows async-prefetch hint. With FILE_FLAG_SEQUENTIAL_SCAN at
|
||||
/// open the kernel already prefetches aggressively, so there's no
|
||||
/// per-range hint we'd add on top. No-op stub for parity with the
|
||||
/// posix platforms.
|
||||
pub(super) fn prefetch(_file: &File, _offset: u64, _len: u64) {}
|
||||
@@ -0,0 +1,49 @@
|
||||
//! File I/O helpers that bound kernel cache pressure on big writes.
|
||||
//!
|
||||
//! `WritebackFile` is a drop-in wrapper around `std::fs::File` for any
|
||||
//! call site that performs large sequential writes (sweep, patch, mux,
|
||||
//! etc.). It implements `Write` and `Seek` so existing code paths can
|
||||
//! swap `File` for `WritebackFile` with no body changes. Internally it
|
||||
//! drives a `WritebackPipeline` that, on Linux, drains dirty pages
|
||||
//! continuously at 32 MB granularity to avoid the kernel's
|
||||
//! accumulate-then-burst flush behaviour. macOS and Windows use a
|
||||
//! no-op pipeline — their default cache policies have not been shown
|
||||
//! to exhibit the same pathology for this access pattern.
|
||||
//!
|
||||
//! `FileSectorSource` is the read-side dual — it implements
|
||||
//! [`crate::sector::SectorSource`] for an ISO file using direct
|
||||
//! `pread`-equivalent calls so the kernel's own readahead policy runs
|
||||
//! (which interleaves naturally with the concurrent writeback). It
|
||||
//! pairs that with periodic `posix_fadvise(DONTNEED)` drops on the
|
||||
//! consumed window so an 85 GB streaming ISO read doesn't fill the
|
||||
//! page cache and starve the concurrent MKV write.
|
||||
//!
|
||||
//! `Pipeline` + `Sink` (0.18) is the generic producer/consumer primitive
|
||||
//! used by sweep, patch, and mux to overlap reads with writes via a
|
||||
//! bounded channel + dedicated consumer thread.
|
||||
//!
|
||||
//! `byte_channel` is a byte-sized producer/consumer channel for the
|
||||
//! mux pipeline, sized to absorb worst-case input read stalls (see
|
||||
//! `(internal)/memory/project_buffering_architecture.md`).
|
||||
|
||||
pub(crate) mod bounded;
|
||||
pub mod byte_channel;
|
||||
pub mod byte_prefetcher;
|
||||
pub mod file_sector_source;
|
||||
pub mod sink;
|
||||
mod writeback;
|
||||
mod writeback_file;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(crate) mod platform_macos;
|
||||
|
||||
pub mod pipeline;
|
||||
|
||||
pub(crate) use writeback_file::WritebackFile;
|
||||
|
||||
// Re-exports for the 0.18 redesign. Sweep, patch, and mux are all
|
||||
// wired up (disc/sweep.rs, disc/patch.rs, autorip's ripper/mux.rs).
|
||||
pub use pipeline::{
|
||||
DEFAULT_PIPELINE_DEPTH, Flow, Pipeline, READ_PIPELINE_DEPTH, Sink, WRITE_PIPELINE_DEPTH,
|
||||
WRITE_THROUGH_DEPTH,
|
||||
};
|
||||
@@ -0,0 +1,916 @@
|
||||
//! Generic bounded producer/consumer pipeline.
|
||||
//!
|
||||
//! `Pipeline<I, R>` spawns a single consumer thread, hands it items
|
||||
//! through a bounded `mpsc::sync_channel`, and joins it on `finish()`.
|
||||
//! The consumer's behaviour is supplied by a [`Sink`] implementation:
|
||||
//! `apply` is called once per item, `close` is called once at the end.
|
||||
//!
|
||||
//! Three call sites in libfreemkv want a producer/consumer split —
|
||||
//! sweep (migrated to `disc/sweep.rs::SweepSink`), patch, and mux.
|
||||
//! 0.18 collapses all three onto this primitive; sweep is in,
|
||||
//! patch and mux migrate in later 0.18 slices.
|
||||
//! See `(internal)/memory/0_18_redesign.md` for the full picture.
|
||||
//!
|
||||
//! ## Cancellation and error semantics
|
||||
//!
|
||||
//! - Producer dropping the channel (via `Pipeline::finish` dropping
|
||||
//! `tx`) signals end-of-stream; consumer flushes via `close()` and
|
||||
//! returns its `Output`.
|
||||
//! - Consumer returning [`Flow::Stop`] also calls `close()` and
|
||||
//! returns its `Output`. `send()` from the producer will then either
|
||||
//! succeed (if the item already fit in the channel buffer) or fail
|
||||
//! with `Err(item)` once the consumer has dropped its receiver.
|
||||
//! - Consumer returning `Err` from `apply` skips `close()` entirely;
|
||||
//! the consumer keeps draining the channel so the producer never
|
||||
//! blocks on a dead receiver, and the first error is propagated as
|
||||
//! the `JoinHandle` result.
|
||||
//! - Consumer panic is converted into
|
||||
//! `Error::IoError { source: io::Error::other(...) }`.
|
||||
//!
|
||||
//! ## Debug logging
|
||||
//!
|
||||
//! Set `FREEMKV_DEBUG=1` environment variable to enable verbose debug
|
||||
//! logging throughout the pipeline (channel sends/receives, backpressure,
|
||||
//! consumer lag detection). This is critical for diagnosing stalls.
|
||||
|
||||
use std::io;
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crossbeam_channel::{Sender, TrySendError, bounded};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::halt::Halt;
|
||||
|
||||
/// Deadline for [`Pipeline::finish_with_halt`]'s polling join. Chosen
|
||||
/// to be comfortably longer than the autorip hard watchdog
|
||||
/// (`HARD_WATCHDOG_STALL_SECS = 300s`) so the watchdog's `exit(1)`
|
||||
/// fires first when both are racing on the same wedged consumer.
|
||||
///
|
||||
/// 10 minutes is a backstop, not a normal timeout — the consumer is
|
||||
/// expected to drain in seconds. If we hit this, something is wedged
|
||||
/// inside a kernel call the consumer thread can't unwind from, and the
|
||||
/// caller has already lost the rip.
|
||||
pub const JOIN_TIMEOUT_SECS: u64 = 600;
|
||||
|
||||
/// Halt-check cadence for the send loop. Producer blocks on
|
||||
/// [`crossbeam_channel::Sender::send_timeout`] for this slice — the
|
||||
/// kernel wakes it the instant the consumer drains a slot, so on the
|
||||
/// happy path there's no throughput cap from this primitive at all
|
||||
/// (the cap is whatever the underlying medium can sustain). When the
|
||||
/// consumer is genuinely wedged, the timeout fires every
|
||||
/// [`crate::halt::POLL_INTERVAL`] and the producer checks the halt
|
||||
/// token; that's the latency a stop request will observe.
|
||||
///
|
||||
/// Single source of truth lives in [`crate::halt::POLL_INTERVAL`]
|
||||
/// (also used by `bounded_syscall`). Aliased here for readability of
|
||||
/// the send/finish call sites below.
|
||||
///
|
||||
/// 0.21.7 replaced an old `std::sync::mpsc::sync_channel` + 50 ms
|
||||
/// `thread::sleep` polling loop that capped mux throughput at
|
||||
/// ~20 frames/sec ≈ 1 MB/s on saturated channels. See
|
||||
/// (internal)/memory/feedback_send_with_halt_poll_throttle.md
|
||||
/// for the multi-day diagnostic that surfaced it.
|
||||
use crate::halt::POLL_INTERVAL;
|
||||
const SEND_HALT_CHECK_INTERVAL: Duration = POLL_INTERVAL;
|
||||
|
||||
/// Check if verbose debug logging is enabled via FREEMKV_DEBUG env var.
|
||||
pub fn debug_enabled() -> bool {
|
||||
std::env::var("FREEMKV_DEBUG")
|
||||
.ok()
|
||||
.map(|v| v == "1" || v == "true" || v == "yes")
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Default channel depth for callers without a specific reason to
|
||||
/// pick another value. Kept conservative (4) — most callers should
|
||||
/// use READ_PIPELINE_DEPTH or WRITE_PIPELINE_DEPTH instead.
|
||||
pub const DEFAULT_PIPELINE_DEPTH: usize = 4;
|
||||
|
||||
/// Read pipeline depth. Larger buffer compensates for drive variability
|
||||
/// and NFS sync_file_range stalls; keeps ISO reader thread fed even when
|
||||
/// consumer blocks on write.
|
||||
pub const READ_PIPELINE_DEPTH: usize = 32;
|
||||
|
||||
/// Write pipeline depth. Smaller buffer reduces backpressure risk when
|
||||
/// sync_file_range blocks; prevents producer from accumulating too much
|
||||
/// work while consumer waits for NFS to drain.
|
||||
pub const WRITE_PIPELINE_DEPTH: usize = 16;
|
||||
|
||||
/// Channel depth for write-through pipelines. Each `send` fully
|
||||
/// drains before the next can enqueue. Use this when the producer
|
||||
/// must observe consumer side-effects (e.g. mapfile state) before
|
||||
/// emitting the next item. Currently used by `disc::patch`.
|
||||
pub const WRITE_THROUGH_DEPTH: usize = 1;
|
||||
|
||||
/// Outcome of [`Sink::apply`]: either keep feeding items
|
||||
/// ([`Flow::Continue`]), or stop the pipeline early and run `close()`
|
||||
/// ([`Flow::Stop`]).
|
||||
///
|
||||
/// `Stop` has no in-tree caller in this slice — sweep never returns
|
||||
/// it (it always processes the producer's full work-list before the
|
||||
/// channel is dropped). Patch and mux are the intended consumers and
|
||||
/// migrate in later 0.18 slices. The variant ships now so the contract
|
||||
/// is fixed; the targeted `#[allow]` is removed when patch lands.
|
||||
pub enum Flow {
|
||||
Continue,
|
||||
#[allow(dead_code)]
|
||||
Stop,
|
||||
}
|
||||
|
||||
/// Consumer-side of a [`Pipeline`]. The pipeline owns one of these on
|
||||
/// its consumer thread and calls `apply` once per received item, then
|
||||
/// `close` once at end-of-stream.
|
||||
pub trait Sink<I>: Send + 'static {
|
||||
/// Type returned from `close()` and surfaced via
|
||||
/// [`Pipeline::finish`].
|
||||
type Output: Send + 'static;
|
||||
|
||||
/// Apply one item. Returning [`Flow::Continue`] keeps the
|
||||
/// pipeline running; [`Flow::Stop`] ends it cleanly (still calls
|
||||
/// `close()`). An error short-circuits: `close()` is *not* called
|
||||
/// and the error is what `finish()` will return, but the consumer
|
||||
/// keeps draining the channel so the producer never blocks on a
|
||||
/// dead receiver.
|
||||
fn apply(&mut self, item: I) -> Result<Flow, Error>;
|
||||
|
||||
/// Called once at end-of-stream — either because the producer
|
||||
/// dropped `tx` or because `apply` returned [`Flow::Stop`]. Use
|
||||
/// this to flush, fsync, finalise. Skipped if any prior `apply`
|
||||
/// returned `Err`.
|
||||
fn close(self) -> Result<Self::Output, Error>;
|
||||
}
|
||||
|
||||
/// Bounded producer/consumer pipeline. Holds the producer-side
|
||||
/// channel and the consumer thread's join handle.
|
||||
pub struct Pipeline<I: Send + 'static, R: Send + 'static> {
|
||||
tx: Sender<I>,
|
||||
handle: JoinHandle<Result<R, Error>>,
|
||||
}
|
||||
|
||||
impl<I: Send + 'static, R: Send + 'static> Pipeline<I, R> {
|
||||
/// Spawn the consumer thread with the given channel depth and
|
||||
/// [`Sink`].
|
||||
///
|
||||
/// The thread is named `freemkv-pipeline-consumer` so it shows up
|
||||
/// distinctly in stack traces and `top -H`. Callers that want a
|
||||
/// more specific name (e.g. `freemkv-sweep-consumer`) should use
|
||||
/// [`Pipeline::spawn_named`] instead. Returns an `Error::IoError`
|
||||
/// if the OS refuses the thread spawn (resource exhaustion);
|
||||
/// callers already operate in fallible context, so this is
|
||||
/// propagated rather than panicked.
|
||||
///
|
||||
/// Sweep uses [`Pipeline::spawn_named`] directly so the consumer
|
||||
/// thread shows up as `freemkv-sweep-consumer`; mux uses
|
||||
/// `freemkv-mux-consumer`. `Pipeline::spawn` (this function, with
|
||||
/// the default name) is used by `disc::patch` and by the unit
|
||||
/// tests in this module.
|
||||
pub fn spawn<S: Sink<I, Output = R>>(depth: usize, sink: S) -> Result<Self, Error> {
|
||||
Self::spawn_named("freemkv-pipeline-consumer", depth, sink)
|
||||
}
|
||||
|
||||
/// Like [`Pipeline::spawn`] but lets the caller supply the
|
||||
/// consumer thread's name. Useful when several pipelines run in
|
||||
/// the same process and stack traces / `top -H` need to tell them
|
||||
/// apart (e.g. `freemkv-sweep-consumer`, `freemkv-mux-consumer`).
|
||||
pub fn spawn_named<S: Sink<I, Output = R>>(
|
||||
name: &str,
|
||||
depth: usize,
|
||||
sink: S,
|
||||
) -> Result<Self, Error> {
|
||||
let (tx, rx) = bounded::<I>(depth);
|
||||
let handle = thread::Builder::new()
|
||||
.name(name.into())
|
||||
.spawn(move || -> Result<R, Error> {
|
||||
let mut sink = sink;
|
||||
let mut first_err: Option<Error> = None;
|
||||
let mut stopped = false;
|
||||
|
||||
while let Ok(item) = rx.recv() {
|
||||
if debug_enabled() {
|
||||
tracing::debug!("Pipeline receive: item={}", std::any::type_name::<I>());
|
||||
}
|
||||
|
||||
let apply_start = std::time::Instant::now();
|
||||
|
||||
if first_err.is_some() || stopped {
|
||||
// Drain remaining items so the producer never
|
||||
// blocks on a dead receiver. `apply` is not
|
||||
// called once we've decided to stop.
|
||||
continue;
|
||||
}
|
||||
match sink.apply(item) {
|
||||
Ok(Flow::Continue) => {}
|
||||
Ok(Flow::Stop) => {
|
||||
stopped = true;
|
||||
if debug_enabled() {
|
||||
tracing::debug!("Pipeline: consumer returned Flow::Stop");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
if debug_enabled() {
|
||||
tracing::debug!("Pipeline: apply error, stopping, err={:?}", e);
|
||||
}
|
||||
first_err = Some(e);
|
||||
}
|
||||
}
|
||||
|
||||
let apply_elapsed = apply_start.elapsed();
|
||||
if debug_enabled() && apply_elapsed > std::time::Duration::from_millis(100) {
|
||||
tracing::debug!(
|
||||
"Pipeline apply: took {:.2}s, item={}",
|
||||
apply_elapsed.as_secs_f64(),
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
} else if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline apply: OK in {:.3}ms, item={}",
|
||||
apply_elapsed.as_micros(),
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
match first_err {
|
||||
Some(e) => Err(e),
|
||||
None => sink.close(),
|
||||
}
|
||||
})
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
|
||||
Ok(Pipeline { tx, handle })
|
||||
}
|
||||
|
||||
/// Push one item. Blocks if the channel is full — that's the
|
||||
/// back-pressure the whole primitive exists to provide. Returns
|
||||
/// the item back if the consumer thread is gone (panicked or
|
||||
/// already returned).
|
||||
///
|
||||
/// After the consumer returns [`Flow::Stop`], `send` will silently
|
||||
/// buffer items into the channel until the channel fills, then
|
||||
/// return `Err(item)` once the consumer has dropped its receiver.
|
||||
/// Producers that need to stop pushing on `Stop` should track an
|
||||
/// independent signal (e.g. `Halt`) — `send` alone is not the
|
||||
/// notification edge.
|
||||
pub fn send(&self, item: I) -> Result<(), I> {
|
||||
let start = std::time::Instant::now();
|
||||
match self.tx.send(item) {
|
||||
Ok(()) => {
|
||||
let elapsed = start.elapsed();
|
||||
if debug_enabled() && elapsed > std::time::Duration::from_millis(10) {
|
||||
tracing::debug!(
|
||||
"Pipeline send: blocked {:.2}s, item={}",
|
||||
elapsed.as_secs_f64(),
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
} else if debug_enabled() {
|
||||
tracing::debug!("Pipeline send: OK in {:.3}ms", elapsed.as_micros());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
let elapsed = start.elapsed();
|
||||
if debug_enabled() && elapsed > std::time::Duration::from_millis(10) {
|
||||
tracing::debug!(
|
||||
"Pipeline send: blocked {:.2}s before channel closed, item={}",
|
||||
elapsed.as_secs_f64(),
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
} else if debug_enabled() {
|
||||
tracing::debug!("Pipeline send: failed after {:.3}ms", elapsed.as_micros());
|
||||
}
|
||||
Err(e.0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Non-blocking variant of [`Pipeline::send`]. If the channel is
|
||||
/// full or the consumer has hung up, the item is returned in
|
||||
/// `Err`. Useful for best-effort signalling (e.g. sweep's
|
||||
/// throttled `StatsRequest`) where dropping the message is
|
||||
/// preferable to blocking the producer.
|
||||
pub fn try_send(&self, item: I) -> Result<(), TrySendError<I>> {
|
||||
self.tx.try_send(item)
|
||||
}
|
||||
|
||||
/// Halt-aware bounded variant of [`Pipeline::send`].
|
||||
///
|
||||
/// Uses [`crossbeam_channel::Sender::send_timeout`] so the producer
|
||||
/// thread BLOCKS on consumer drain (kernel-wakeup) rather than
|
||||
/// polling. The timeout slice is just the halt-observation cadence
|
||||
/// ([`SEND_HALT_CHECK_INTERVAL`]) — on the happy path the producer
|
||||
/// wakes the instant the consumer drains a slot, so there is no
|
||||
/// throughput cap from this primitive at any medium speed.
|
||||
///
|
||||
/// Returns:
|
||||
///
|
||||
/// - `Ok(())` once the item lands in the channel.
|
||||
/// - `Err(item)` if the consumer disconnected, the halt fired, or
|
||||
/// the deadline elapsed — the caller gets the item back so it
|
||||
/// can decide whether to drop it, route it elsewhere, or unwind.
|
||||
///
|
||||
/// Use this in producer threads that have a `Halt` token threaded
|
||||
/// through (mux, sweep, patch). Plain [`Pipeline::send`] is
|
||||
/// preserved for callers that don't (yet) plumb halt through.
|
||||
///
|
||||
/// Unlike [`Pipeline::send`], this never blocks the producer
|
||||
/// thread inside an unkillable `mpsc::send` — if the consumer is
|
||||
/// wedged inside an unkillable syscall, the producer can still
|
||||
/// observe `/api/stop` and unwind within
|
||||
/// [`SEND_HALT_CHECK_INTERVAL`].
|
||||
pub fn send_with_halt(&self, item: I, halt: &Halt, deadline: Duration) -> Result<(), I> {
|
||||
use crossbeam_channel::SendTimeoutError;
|
||||
let end = Instant::now() + deadline;
|
||||
let mut pending = item;
|
||||
loop {
|
||||
// Pre-check the cheap exit conditions before parking.
|
||||
if halt.is_cancelled() {
|
||||
if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline send_with_halt: halt observed, returning item={}",
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
return Err(pending);
|
||||
}
|
||||
let now = Instant::now();
|
||||
if now >= end {
|
||||
if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline send_with_halt: deadline elapsed, returning item={}",
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
return Err(pending);
|
||||
}
|
||||
// Wait for space-available or halt-check tick, whichever
|
||||
// is sooner. Crossbeam's send_timeout is kernel-wakeup
|
||||
// based: the consumer's recv on a saturated channel
|
||||
// signals this thread the moment a slot opens up.
|
||||
let slice = SEND_HALT_CHECK_INTERVAL.min(end.saturating_duration_since(now));
|
||||
match self.tx.send_timeout(pending, slice) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(SendTimeoutError::Timeout(returned)) => {
|
||||
pending = returned;
|
||||
// loop: re-check halt + deadline, then park again
|
||||
}
|
||||
Err(SendTimeoutError::Disconnected(returned)) => {
|
||||
if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline send_with_halt: consumer disconnected, item={}",
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
return Err(returned);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the producer-side channel and wait for the consumer
|
||||
/// thread to finish. Returns whatever the consumer's `close()`
|
||||
/// produced, or the first `apply` error, or — on consumer panic —
|
||||
/// an `Error::IoError` whose source is `io::Error::other(...)`
|
||||
/// with a "pipeline consumer panicked: <payload>" message
|
||||
/// (callers can match on the constant prefix).
|
||||
pub fn finish(self) -> Result<R, Error> {
|
||||
let Pipeline { tx, handle } = self;
|
||||
// Explicit drop, although the destructure already drops `tx`
|
||||
// at end-of-scope. Being explicit keeps the intent obvious.
|
||||
drop(tx);
|
||||
match handle.join() {
|
||||
Ok(result) => result,
|
||||
Err(payload) => {
|
||||
// Preserve the original panic message when the
|
||||
// consumer's panic payload was a `&str` or `String`
|
||||
// (the two stdlib formats that `panic!` produces).
|
||||
// Anything else falls back to "(no message)".
|
||||
let msg = payload
|
||||
.downcast_ref::<&'static str>()
|
||||
.copied()
|
||||
.or_else(|| payload.downcast_ref::<String>().map(|s| s.as_str()))
|
||||
.unwrap_or("(no message)");
|
||||
Err(Error::IoError {
|
||||
source: io::Error::other(format!("pipeline consumer panicked: {msg}")),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Halt-aware, deadline-bounded variant of [`Pipeline::finish`].
|
||||
///
|
||||
/// Drops the producer-side channel (same as `finish`) and then
|
||||
/// polls `JoinHandle::is_finished()` on a 250 ms cadence. Between
|
||||
/// slices, checks (1) the optional [`Halt`] token and (2) the
|
||||
/// [`JOIN_TIMEOUT_SECS`] deadline. Returns:
|
||||
///
|
||||
/// - `Ok(R)` on a clean consumer exit.
|
||||
/// - `Err(Error::IoError)` with one of three message prefixes for
|
||||
/// wedge cases:
|
||||
/// - `"pipeline join halted"` — halt fired while waiting.
|
||||
/// - `"pipeline join timed out"` — `JOIN_TIMEOUT_SECS` elapsed.
|
||||
/// - `"pipeline consumer panicked"` — same as `finish()`.
|
||||
///
|
||||
/// In the `halted` and `timed out` branches the consumer thread is
|
||||
/// intentionally leaked — exactly the same trade-off the
|
||||
/// `bounded_syscall` primitive makes. The wedged kernel call
|
||||
/// inside the consumer will unwind whenever it does, or at
|
||||
/// process exit. The caller is free to fall back to a degraded
|
||||
/// path (in autorip's case: `exit(1)` after the hard watchdog
|
||||
/// escalation, letting Docker restart the container).
|
||||
///
|
||||
/// Plain [`Pipeline::finish`] is preserved for callers without a
|
||||
/// halt-token plumbed through; that path still blocks indefinitely
|
||||
/// on `join()`, matching pre-0.20.8 behaviour.
|
||||
pub fn finish_with_halt(self, halt: Option<&Halt>) -> Result<R, Error> {
|
||||
let Pipeline { tx, handle } = self;
|
||||
drop(tx);
|
||||
let deadline = Instant::now() + Duration::from_secs(JOIN_TIMEOUT_SECS);
|
||||
loop {
|
||||
if handle.is_finished() {
|
||||
return match handle.join() {
|
||||
Ok(result) => result,
|
||||
Err(payload) => {
|
||||
let msg = payload
|
||||
.downcast_ref::<&'static str>()
|
||||
.copied()
|
||||
.or_else(|| payload.downcast_ref::<String>().map(|s| s.as_str()))
|
||||
.unwrap_or("(no message)");
|
||||
Err(Error::IoError {
|
||||
source: io::Error::other(format!("pipeline consumer panicked: {msg}")),
|
||||
})
|
||||
}
|
||||
};
|
||||
}
|
||||
if let Some(h) = halt {
|
||||
if h.is_cancelled() {
|
||||
// Consumer thread is intentionally leaked.
|
||||
return Err(Error::IoError {
|
||||
source: io::Error::other("pipeline join halted"),
|
||||
});
|
||||
}
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
// Consumer thread is intentionally leaked.
|
||||
return Err(Error::IoError {
|
||||
source: io::Error::other("pipeline join timed out"),
|
||||
});
|
||||
}
|
||||
thread::sleep(POLL_INTERVAL);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Sums u64s; returns the total from `close`.
|
||||
struct SumSink {
|
||||
total: u64,
|
||||
}
|
||||
|
||||
impl Sink<u64> for SumSink {
|
||||
type Output = u64;
|
||||
|
||||
fn apply(&mut self, item: u64) -> Result<Flow, Error> {
|
||||
self.total += item;
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(self) -> Result<u64, Error> {
|
||||
Ok(self.total)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn happy_path_sums_items() {
|
||||
let pipe = Pipeline::spawn(DEFAULT_PIPELINE_DEPTH, SumSink { total: 0 })
|
||||
.expect("spawn should succeed");
|
||||
let mut expected = 0u64;
|
||||
for i in 0..100u64 {
|
||||
expected += i;
|
||||
pipe.send(i).expect("send should succeed");
|
||||
}
|
||||
let total = pipe.finish().expect("finish should succeed");
|
||||
assert_eq!(total, expected);
|
||||
assert_eq!(total, (0..100u64).sum::<u64>());
|
||||
}
|
||||
|
||||
/// Sleeps `delay` per apply; counts how many it received.
|
||||
struct SlowSink {
|
||||
delay: Duration,
|
||||
count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Sink<()> for SlowSink {
|
||||
type Output = usize;
|
||||
|
||||
fn apply(&mut self, _item: ()) -> Result<Flow, Error> {
|
||||
std::thread::sleep(self.delay);
|
||||
self.count.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(self) -> Result<usize, Error> {
|
||||
Ok(self.count.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn back_pressure_blocks_sender() {
|
||||
// depth=2 + 5 sends + 50ms/apply: with the consumer pinned at
|
||||
// 50 ms per item, the producer can buffer 2 (channel cap) +
|
||||
// 1 (consumer in flight) = 3 items before sends 4 and 5 must
|
||||
// block on consumer progress. Wall-clock floor across all 5
|
||||
// sends is therefore ~2 * 50ms = 100ms (sends 4 and 5 each
|
||||
// wait roughly one apply-cycle). Use 80 ms as the assertion
|
||||
// floor to stay above the 50ms-per-item progress floor while
|
||||
// tolerating CI jitter — it still proves blocking is real.
|
||||
let count = Arc::new(AtomicUsize::new(0));
|
||||
let sink = SlowSink {
|
||||
delay: Duration::from_millis(50),
|
||||
count: count.clone(),
|
||||
};
|
||||
let pipe = Pipeline::spawn(2, sink).expect("spawn should succeed");
|
||||
|
||||
let start = Instant::now();
|
||||
for _ in 0..5 {
|
||||
pipe.send(()).expect("send should succeed");
|
||||
}
|
||||
let elapsed_send = start.elapsed();
|
||||
|
||||
let total = pipe.finish().expect("finish should succeed");
|
||||
assert_eq!(total, 5);
|
||||
assert!(
|
||||
elapsed_send >= Duration::from_millis(80),
|
||||
"back-pressure not observed: 5 sends with depth=2 and 50ms/apply \
|
||||
took {elapsed_send:?}, expected ≥ ~100ms (one or more sends \
|
||||
should have blocked behind the consumer)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns `Err` on the Nth apply (1-indexed). Tracks all calls.
|
||||
struct FailOnNthSink {
|
||||
n: usize,
|
||||
seen: Arc<AtomicUsize>,
|
||||
close_called: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Sink<u64> for FailOnNthSink {
|
||||
type Output = ();
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
let i = self.seen.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if i == self.n {
|
||||
Err(Error::DecryptFailed)
|
||||
} else {
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
fn close(self) -> Result<(), Error> {
|
||||
self.close_called.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_error_drains_then_propagates() {
|
||||
let seen = Arc::new(AtomicUsize::new(0));
|
||||
let close_called = Arc::new(AtomicUsize::new(0));
|
||||
let pipe = Pipeline::spawn(
|
||||
DEFAULT_PIPELINE_DEPTH,
|
||||
FailOnNthSink {
|
||||
n: 3,
|
||||
seen: seen.clone(),
|
||||
close_called: close_called.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
|
||||
// Send 10 items. Subsequent sends after the 3rd error must
|
||||
// still succeed (the consumer is draining).
|
||||
for i in 0..10u64 {
|
||||
pipe.send(i).expect("send should succeed even after error");
|
||||
}
|
||||
|
||||
let res = pipe.finish();
|
||||
assert!(matches!(res, Err(Error::DecryptFailed)));
|
||||
assert_eq!(
|
||||
close_called.load(Ordering::SeqCst),
|
||||
0,
|
||||
"close() must not be called when apply returned Err"
|
||||
);
|
||||
// The consumer kept calling `recv` to drain after the error;
|
||||
// it just stopped invoking `apply`. So `seen` is exactly 3
|
||||
// (apply was called for items 1, 2, 3).
|
||||
assert_eq!(seen.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
/// Returns `Flow::Stop` on the Nth apply.
|
||||
struct StopOnNthSink {
|
||||
n: usize,
|
||||
seen: Arc<AtomicUsize>,
|
||||
close_called: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Sink<u64> for StopOnNthSink {
|
||||
type Output = usize;
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
let i = self.seen.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if i >= self.n {
|
||||
Ok(Flow::Stop)
|
||||
} else {
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
fn close(self) -> Result<usize, Error> {
|
||||
self.close_called.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(self.seen.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_stop_calls_close_and_returns_output() {
|
||||
let seen = Arc::new(AtomicUsize::new(0));
|
||||
let close_called = Arc::new(AtomicUsize::new(0));
|
||||
let pipe = Pipeline::spawn(
|
||||
DEFAULT_PIPELINE_DEPTH,
|
||||
StopOnNthSink {
|
||||
n: 3,
|
||||
seen: seen.clone(),
|
||||
close_called: close_called.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
|
||||
// Send 10 items. After Stop, subsequent sends may either
|
||||
// succeed (already buffered) or fail with Err(I) (channel
|
||||
// closed). Both are valid — we don't assert on the send
|
||||
// results.
|
||||
for i in 0..10u64 {
|
||||
let _ = pipe.send(i);
|
||||
}
|
||||
|
||||
let out = pipe.finish().expect("finish should succeed after Stop");
|
||||
assert_eq!(close_called.load(Ordering::SeqCst), 1);
|
||||
// At least 3 items processed (the one that returned Stop).
|
||||
assert!(
|
||||
out >= 3,
|
||||
"expected ≥ 3 applies before Stop took effect, got {out}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Panics on the first apply call.
|
||||
struct PanickingSink;
|
||||
|
||||
impl Sink<u64> for PanickingSink {
|
||||
type Output = ();
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
panic!("synthetic test panic");
|
||||
}
|
||||
|
||||
fn close(self) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consumer_panic_becomes_io_error() {
|
||||
// Silence the panic message that would otherwise pollute the
|
||||
// test output — we expect this panic.
|
||||
let prev = std::panic::take_hook();
|
||||
std::panic::set_hook(Box::new(|_| {}));
|
||||
|
||||
let pipe =
|
||||
Pipeline::spawn(DEFAULT_PIPELINE_DEPTH, PanickingSink).expect("spawn should succeed");
|
||||
// First send may succeed (item buffered before panic) or fail
|
||||
// (channel closed after panic) — either is fine.
|
||||
let _ = pipe.send(1);
|
||||
// Drain a few more sends; once the channel is closed they'll
|
||||
// return Err(I), which we just discard.
|
||||
for i in 0..5u64 {
|
||||
let _ = pipe.send(i);
|
||||
}
|
||||
let res = pipe.finish();
|
||||
|
||||
std::panic::set_hook(prev);
|
||||
|
||||
match res {
|
||||
Err(Error::IoError { source }) => {
|
||||
let msg = source.to_string();
|
||||
// Constant prefix lets callers match without parsing
|
||||
// the variable payload tail.
|
||||
assert!(
|
||||
msg.contains("pipeline consumer panicked"),
|
||||
"expected constant panic prefix, got: {msg}"
|
||||
);
|
||||
// The original `panic!` payload (a `&'static str`) must
|
||||
// be preserved — without the downcast the message
|
||||
// would just be the prefix.
|
||||
assert!(
|
||||
msg.contains("synthetic test panic"),
|
||||
"expected original panic payload, got: {msg}"
|
||||
);
|
||||
}
|
||||
other => panic!("expected Err(IoError), got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Never-completing sink — `apply` blocks until cancelled. Signals
|
||||
/// `started` once it has consumed its first item so the test
|
||||
/// driver knows the consumer thread is wedged in `apply` (and
|
||||
/// will no longer drain the channel). Used to drive the
|
||||
/// halt/timeout paths of `send_with_halt` and `finish_with_halt`
|
||||
/// without depending on real I/O.
|
||||
struct NeverDrainsSink {
|
||||
cancel: Arc<std::sync::atomic::AtomicBool>,
|
||||
started: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
impl Sink<u64> for NeverDrainsSink {
|
||||
type Output = ();
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
self.started.store(true, Ordering::SeqCst);
|
||||
while !self.cancel.load(Ordering::SeqCst) {
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(self) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Spin until `started` flips or `bail` elapses. Used by the
|
||||
/// send_with_halt tests to synchronise with the consumer thread
|
||||
/// before exercising the bounded-send timeout path.
|
||||
fn wait_for_started(started: &Arc<std::sync::atomic::AtomicBool>, bail: Duration) {
|
||||
let end = Instant::now() + bail;
|
||||
while !started.load(Ordering::SeqCst) {
|
||||
assert!(Instant::now() < end, "consumer never started apply()");
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_with_halt_returns_item_on_deadline() {
|
||||
// depth=1 + consumer wedged in apply on the first item, AND
|
||||
// the channel buffer already loaded with a second item, means
|
||||
// any further `try_send` sees Full; with a 200 ms deadline and
|
||||
// no halt fired, send_with_halt must return `Err(item)` within
|
||||
// roughly the deadline. Synchronising on `started` ensures the
|
||||
// consumer has actually started its wedged apply BEFORE we
|
||||
// load the channel-buffer slot — without that, the consumer
|
||||
// could still drain in a race window.
|
||||
let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipe = Pipeline::spawn(
|
||||
1,
|
||||
NeverDrainsSink {
|
||||
cancel: cancel.clone(),
|
||||
started: started.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
// First send: consumer recv()s it and wedges in apply.
|
||||
pipe.send(0u64).expect("first send hands off to consumer");
|
||||
wait_for_started(&started, Duration::from_secs(2));
|
||||
// Second send: lands in the depth=1 buffer slot, consumer
|
||||
// can't pick it up because it's wedged in apply. Channel now
|
||||
// full from the producer's perspective.
|
||||
pipe.send(1u64).expect("second send fills the buffer");
|
||||
|
||||
let halt = crate::halt::Halt::new();
|
||||
let start = Instant::now();
|
||||
let res = pipe.send_with_halt(99u64, &halt, Duration::from_millis(200));
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Release the leaked consumer so the test process winds down.
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
let _ = pipe.finish();
|
||||
|
||||
assert!(matches!(res, Err(99)), "expected item returned on deadline");
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(150),
|
||||
"deadline returned too early: {elapsed:?}"
|
||||
);
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"deadline blew past tolerance: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_with_halt_returns_item_on_halt() {
|
||||
// Same setup, but the halt fires before the deadline elapses.
|
||||
// The send loop must observe the halt within ~50 ms (the
|
||||
// SEND_POLL_INTERVAL) and return the item.
|
||||
let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipe = Pipeline::spawn(
|
||||
1,
|
||||
NeverDrainsSink {
|
||||
cancel: cancel.clone(),
|
||||
started: started.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
pipe.send(0u64).expect("first send hands off to consumer");
|
||||
wait_for_started(&started, Duration::from_secs(2));
|
||||
pipe.send(1u64).expect("second send fills the buffer");
|
||||
|
||||
let halt = crate::halt::Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
halt2.cancel();
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
let res = pipe.send_with_halt(7u64, &halt, Duration::from_secs(10));
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
let _ = pipe.finish();
|
||||
|
||||
assert!(matches!(res, Err(7)), "expected item returned on halt");
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"halt observation took too long: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_with_halt_returns_halted_when_consumer_wedged() {
|
||||
// Consumer wedges on the first apply; halt fires; finish
|
||||
// returns the documented "pipeline join halted" error rather
|
||||
// than blocking forever.
|
||||
let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipe = Pipeline::spawn(
|
||||
DEFAULT_PIPELINE_DEPTH,
|
||||
NeverDrainsSink {
|
||||
cancel: cancel.clone(),
|
||||
started: started.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
pipe.send(0u64).expect("seed item the consumer wedges on");
|
||||
wait_for_started(&started, Duration::from_secs(2));
|
||||
|
||||
let halt = crate::halt::Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(400));
|
||||
halt2.cancel();
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
let res = pipe.finish_with_halt(Some(&halt));
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Release the leaked consumer so the test process exits cleanly.
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
|
||||
match res {
|
||||
Err(Error::IoError { source }) => {
|
||||
assert!(
|
||||
source.to_string().contains("pipeline join halted"),
|
||||
"expected halt-prefix error, got: {source}"
|
||||
);
|
||||
}
|
||||
other => panic!("expected Err(IoError) halted, got {other:?}"),
|
||||
}
|
||||
// Bailed out within ~1 second of the halt firing (worst case
|
||||
// one POLL_INTERVAL = 250 ms of slack).
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"halt observation took too long: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_with_halt_happy_path_returns_output() {
|
||||
// No halt token, sink completes normally — finish_with_halt
|
||||
// must return the same Output that `finish` would.
|
||||
let pipe = Pipeline::spawn(DEFAULT_PIPELINE_DEPTH, SumSink { total: 0 })
|
||||
.expect("spawn should succeed");
|
||||
for i in 0..10u64 {
|
||||
pipe.send(i).expect("send should succeed");
|
||||
}
|
||||
let total = pipe
|
||||
.finish_with_halt(None)
|
||||
.expect("happy-path finish_with_halt should succeed");
|
||||
assert_eq!(total, (0..10u64).sum::<u64>());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
//! Shared macOS `fcntl(F_PREALLOCATE)` definitions.
|
||||
//!
|
||||
//! The `libc` crate doesn't expose these symbols across all macOS SDK
|
||||
//! versions, so we define them locally with values from
|
||||
//! `/usr/include/sys/fcntl.h`. Two call sites (
|
||||
//! [`crate::io::writeback_file`] and [`crate::io::sink::preallocate`])
|
||||
//! need the same constants and `fstore_t` layout — keeping a single
|
||||
//! source of truth here prevents the two copies from drifting.
|
||||
//!
|
||||
//! Module-level cfg gate lives in the parent (`io/mod.rs`); this file
|
||||
//! is only compiled on macOS, so no inner `#![cfg]` is needed.
|
||||
|
||||
/// `fcntl(F_PREALLOCATE)` command number from `sys/fcntl.h`.
|
||||
pub(crate) const F_PREALLOCATE: libc::c_int = 42;
|
||||
|
||||
/// Anchor preallocation at the current physical EOF.
|
||||
pub(crate) const F_PEOFPOSMODE: libc::c_int = 3;
|
||||
|
||||
/// Prefer a contiguous allocation. Try this first; on `EINVAL` (no
|
||||
/// contiguous run of that size), fall back to `F_ALLOCATEALL`.
|
||||
pub(crate) const F_ALLOCATECONTIG: libc::c_uint = 0x0000_0002;
|
||||
|
||||
/// Allow non-contiguous allocation. Stronger guarantee than just
|
||||
/// asking for `F_ALLOCATECONTIG` because the kernel will piece
|
||||
/// together fragments rather than failing.
|
||||
pub(crate) const F_ALLOCATEALL: libc::c_uint = 0x0000_0004;
|
||||
|
||||
/// `fstore_t` from `sys/fcntl.h`. `repr(C)` because we hand it to
|
||||
/// `fcntl(F_PREALLOCATE)` which writes through the pointer.
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct Fstore {
|
||||
pub fst_flags: libc::c_uint,
|
||||
pub fst_posmode: libc::c_int,
|
||||
pub fst_offset: libc::off_t,
|
||||
pub fst_length: libc::off_t,
|
||||
pub fst_bytesalloc: libc::off_t,
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
//! `LocalFileSink` — `BufWriter<File>` for the common local-disk case.
|
||||
//!
|
||||
//! Buffering: 4 MiB internal `BufWriter`. Sized to coalesce the small
|
||||
//! per-PES writes that come out of the muxer into kernel-page-aligned
|
||||
//! flushes without making the buffer big enough to matter for memory
|
||||
//! pressure on a single concurrent rip.
|
||||
//!
|
||||
//! `Seek` flushes the underlying `BufWriter` first; otherwise a seek
|
||||
//! could leapfrog buffered data and silently corrupt the file. This is
|
||||
//! the same shape `BufWriter` itself uses when it impls `Seek` in
|
||||
//! stdlib, and is necessary for MKV's seek-back operations (cluster
|
||||
//! size patch, Cues index, segment header backpatch) to land on the
|
||||
//! right offset.
|
||||
//!
|
||||
//! `RandomAccessSink` is satisfied via the blanket impl in
|
||||
//! [`super::mod`]; no explicit impl needed here.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, BufWriter, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use super::preallocate;
|
||||
|
||||
const BUFFER_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Random-access write sink for local disks.
|
||||
///
|
||||
/// Wraps a `BufWriter<File>` with a 4 MiB internal buffer and forwards
|
||||
/// `Write`/`Seek` so any call site that previously held a `File` or
|
||||
/// `WritebackFile` can drop this in. `finish()` flushes the buffer and
|
||||
/// runs `sync_all` on the underlying file so the caller can drop it
|
||||
/// without losing data.
|
||||
///
|
||||
/// Construction always opens the file `create + truncate + read +
|
||||
/// write`. `read` is enabled so the same handle can be reused for a
|
||||
/// verification re-read after the mux (the existing
|
||||
/// `FileSectorSink::create` pattern). On Linux, [`with_size_hint`]
|
||||
/// additionally calls `fallocate(FALLOC_FL_KEEP_SIZE)` to pre-reserve
|
||||
/// extents.
|
||||
///
|
||||
/// [`with_size_hint`]: Self::with_size_hint
|
||||
pub struct LocalFileSink {
|
||||
inner: BufWriter<File>,
|
||||
}
|
||||
|
||||
impl LocalFileSink {
|
||||
/// Open `path` for writing, truncating any existing contents.
|
||||
pub fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
Ok(Self {
|
||||
inner: BufWriter::with_capacity(BUFFER_BYTES, file),
|
||||
})
|
||||
}
|
||||
|
||||
/// Like [`Self::create`] but additionally calls the per-OS
|
||||
/// preallocate path with `size_bytes`. On Linux this is
|
||||
/// `fallocate(FALLOC_FL_KEEP_SIZE)` so the on-disk extents are
|
||||
/// reserved up front (reducing fragmentation for big sequential
|
||||
/// muxer output); on other OSes it is a no-op today. Failures
|
||||
/// from the preallocate call are non-fatal — the file is still
|
||||
/// returned, just without the size reservation.
|
||||
pub fn with_size_hint(path: &Path, size_bytes: u64) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
preallocate::preallocate(&file, size_bytes);
|
||||
Ok(Self {
|
||||
inner: BufWriter::with_capacity(BUFFER_BYTES, file),
|
||||
})
|
||||
}
|
||||
|
||||
/// Drain the internal buffer and `fsync` the underlying file.
|
||||
/// Idempotent with `Drop` (the `BufWriter` also flushes on drop;
|
||||
/// this call additionally surfaces fsync errors to the caller).
|
||||
#[allow(dead_code)] // exposed for parity with WritebackFile::sync_all
|
||||
pub fn sync_all(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()?;
|
||||
self.inner.get_ref().sync_all()
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for LocalFileSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.inner.write(buf)
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.inner.write_all(buf)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for LocalFileSink {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
// Flush before seeking so buffered bytes land at the offset
|
||||
// they were written for, not the new one.
|
||||
self.inner.flush()?;
|
||||
self.inner.get_mut().seek(from)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
|
||||
#[test]
|
||||
fn write_seek_roundtrip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("rt.bin");
|
||||
let mut s = LocalFileSink::create(&p).unwrap();
|
||||
s.write_all(b"AAAA").unwrap();
|
||||
s.write_all(b"BBBB").unwrap();
|
||||
// Seek back over the second word and overwrite.
|
||||
s.seek(SeekFrom::Start(4)).unwrap();
|
||||
s.write_all(b"CCCC").unwrap();
|
||||
s.sync_all().unwrap();
|
||||
drop(s);
|
||||
|
||||
let mut f = File::open(&p).unwrap();
|
||||
let mut got = Vec::new();
|
||||
f.read_to_end(&mut got).unwrap();
|
||||
assert_eq!(&got[..], b"AAAACCCC");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_flushes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("drop.bin");
|
||||
{
|
||||
let mut s = LocalFileSink::create(&p).unwrap();
|
||||
s.write_all(b"buffered").unwrap();
|
||||
// No explicit flush / sync_all — BufWriter drop runs the
|
||||
// flush and the file should land on disk.
|
||||
}
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..], b"buffered");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_size_hint_creates_writable_file() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("sz.bin");
|
||||
let mut s = LocalFileSink::with_size_hint(&p, 64 * 1024).unwrap();
|
||||
s.write_all(b"hint-ok").unwrap();
|
||||
s.sync_all().unwrap();
|
||||
drop(s);
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..], b"hint-ok");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//! Output-sink trait split for the buffering architecture.
|
||||
//!
|
||||
//! Two traits, one for each capability axis of an output destination:
|
||||
//!
|
||||
//! - [`SequentialSink`] — anything you can `Write` to in order. Sockets,
|
||||
//! pipes, append-only stores, plain files. Containers that don't need
|
||||
//! seek (M2TS, fMP4, HEVC elementary) target this.
|
||||
//! - [`RandomAccessSink`] — everything `SequentialSink` plus a working
|
||||
//! `Seek`. Local files, NFS files, anything with random-write
|
||||
//! semantics. Containers that need backpatch (MKV cluster sizes, Cues
|
||||
//! index, MP4 moov-at-end) target this.
|
||||
//!
|
||||
//! `RandomAccessSink: SequentialSink` — every random-access sink is
|
||||
//! also a valid sequential sink. The muxer is generic over which it
|
||||
//! requires (`MkvMux<S: RandomAccessSink>`, `M2tsMux<S: SequentialSink>`)
|
||||
//! so an attempt to mux MKV to a network socket is a compile error.
|
||||
//!
|
||||
//! Buffering policy belongs to the concrete sink, not to a wrapper at
|
||||
//! the call site. `LocalFileSink` wraps a `BufWriter<File>` with a
|
||||
//! 4 MiB buffer for the common local-disk case; `WritebackFile`
|
||||
//! (separate module) wraps a `File` with the adaptive-chunk
|
||||
//! `sync_file_range` machinery for the Linux+NFS case.
|
||||
//!
|
||||
//! See `(internal)/memory/project_buffering_architecture.md` for
|
||||
//! the full design and the source/sink matrix.
|
||||
|
||||
use std::io::{Seek, Write};
|
||||
|
||||
mod local_file;
|
||||
mod preallocate;
|
||||
mod socket;
|
||||
|
||||
pub use local_file::LocalFileSink;
|
||||
pub use socket::{SocketSink, UdpSocketSink};
|
||||
|
||||
/// Sequential-only write destination. Sockets, pipes, append-only
|
||||
/// stores. No seek. Implementations own their write buffering — the
|
||||
/// trait does not impose or hide any buffering of its own.
|
||||
///
|
||||
/// `finish` drains any internal buffering and signals end-of-stream to
|
||||
/// the underlying transport (close-write on a socket, flush on a
|
||||
/// buffered writer, etc.). The default impl is a no-op; concrete
|
||||
/// implementations that need explicit shutdown can override it but the
|
||||
/// blanket impl below keeps it optional for adapter types like
|
||||
/// `&mut File`.
|
||||
pub trait SequentialSink: Write + Send {
|
||||
fn finish(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Random-access write destination. Local files, NFS files, anything
|
||||
/// with a working `Seek`. Inherits the `SequentialSink` contract — a
|
||||
/// random-access sink is always usable as a sequential sink.
|
||||
pub trait RandomAccessSink: SequentialSink + Seek {}
|
||||
|
||||
// Blanket impls so any `Write + Send` type acts as a `SequentialSink`
|
||||
// (with default `finish`), and any sink that also impls `Seek` is
|
||||
// automatically a `RandomAccessSink`. Keeps call-site ergonomics simple
|
||||
// — `&mut File`, `LocalFileSink`, `WritebackFile`, `BufWriter<File>`,
|
||||
// and `Cursor<Vec<u8>>` all satisfy the right trait without per-type
|
||||
// boilerplate.
|
||||
impl<T: Write + Send + ?Sized> SequentialSink for T {}
|
||||
impl<T: SequentialSink + Seek + ?Sized> RandomAccessSink for T {}
|
||||
|
||||
/// Pick the right `RandomAccessSink` impl for `dest` based on its
|
||||
/// filesystem type.
|
||||
///
|
||||
/// - Linux + NFS path → `WritebackFile` with its adaptive-chunk
|
||||
/// sync_file_range machinery and (when supported) `fallocate` size
|
||||
/// hint.
|
||||
/// - everything else → [`LocalFileSink`] over `BufWriter<File>`. On
|
||||
/// non-Linux there is no `WritebackFile` machinery to opt into, and
|
||||
/// on local Linux the kernel's default writeback policy is already
|
||||
/// fine.
|
||||
///
|
||||
/// `size_hint`, when present, is forwarded to the per-OS preallocate
|
||||
/// path (`fallocate(KEEP_SIZE)` on Linux, `F_PREALLOCATE` on macOS when
|
||||
/// implemented, no-op elsewhere).
|
||||
///
|
||||
/// Returns a boxed trait object so the call site (mux construction)
|
||||
/// stays agnostic of which concrete sink got picked.
|
||||
#[allow(dead_code)] // wiring to mux::resolve is a follow-up commit
|
||||
pub fn open_for_mkv(
|
||||
dest: &std::path::Path,
|
||||
size_hint: Option<u64>,
|
||||
) -> std::io::Result<Box<dyn RandomAccessSink>> {
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
use crate::platform::fs_type::detect;
|
||||
#[cfg(target_os = "linux")]
|
||||
use crate::platform::fs_type::{FsType, detect};
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
if detect(dest) == FsType::Nfs {
|
||||
let wf = match size_hint {
|
||||
Some(n) => crate::io::WritebackFile::create_with_size_hint(dest, n)?,
|
||||
None => crate::io::WritebackFile::create(dest)?,
|
||||
};
|
||||
return Ok(Box::new(wf));
|
||||
}
|
||||
}
|
||||
// Silence the unused-binding warning on non-Linux where the only
|
||||
// branch above is cfg-gated out.
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
let _ = detect(dest);
|
||||
}
|
||||
|
||||
let sink = match size_hint {
|
||||
Some(n) => LocalFileSink::with_size_hint(dest, n)?,
|
||||
None => LocalFileSink::create(dest)?,
|
||||
};
|
||||
Ok(Box::new(sink))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs::File;
|
||||
|
||||
// Type-level assertion: the blanket impls cover the shapes we care
|
||||
// about. These functions never run; they just have to type-check.
|
||||
fn _assert_file_is_sequential(_: &mut dyn SequentialSink) {}
|
||||
fn _assert_file_is_random_access(_: &mut dyn RandomAccessSink) {}
|
||||
|
||||
#[test]
|
||||
fn blanket_impls_cover_file_and_localfilesink() {
|
||||
// `File` directly via blanket impls.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut f = File::create(dir.path().join("a.bin")).unwrap();
|
||||
_assert_file_is_sequential(&mut f);
|
||||
_assert_file_is_random_access(&mut f);
|
||||
|
||||
// `LocalFileSink` ditto.
|
||||
let mut s = LocalFileSink::create(&dir.path().join("b.bin")).unwrap();
|
||||
_assert_file_is_sequential(&mut s);
|
||||
_assert_file_is_random_access(&mut s);
|
||||
|
||||
// `WritebackFile` — confirms the Phase 1 type still satisfies
|
||||
// the trait via the blanket impl without needing an explicit
|
||||
// `impl RandomAccessSink for WritebackFile {}`.
|
||||
let mut wf = crate::io::WritebackFile::create(&dir.path().join("c.bin")).unwrap();
|
||||
_assert_file_is_sequential(&mut wf);
|
||||
_assert_file_is_random_access(&mut wf);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_for_mkv_returns_a_random_access_sink() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("c.bin");
|
||||
let mut sink = open_for_mkv(&p, Some(64 * 1024)).unwrap();
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
sink.write_all(b"hello").unwrap();
|
||||
sink.seek(SeekFrom::Start(0)).unwrap();
|
||||
sink.finish().unwrap();
|
||||
drop(sink);
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..5], b"hello");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
//! Linux `fallocate(FALLOC_FL_KEEP_SIZE)` preallocation.
|
||||
//!
|
||||
//! `KEEP_SIZE` reserves extents without changing the apparent file
|
||||
//! length, which matches the muxer's expectation that writes still grow
|
||||
//! the file naturally.
|
||||
|
||||
use std::fs::File;
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
pub(super) fn preallocate_impl(file: &File, size_bytes: u64) {
|
||||
let fd = file.as_raw_fd();
|
||||
// FALLOC_FL_KEEP_SIZE = 0x01.
|
||||
let rc = unsafe { libc::fallocate(fd, libc::FALLOC_FL_KEEP_SIZE, 0, size_bytes as i64) };
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink fallocate size_hint={size_bytes} rc={rc} ok={}",
|
||||
rc == 0
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
//! macOS `F_PREALLOCATE` extent reservation.
|
||||
//!
|
||||
//! `fcntl(F_PREALLOCATE)` with `F_ALLOCATECONTIG` first (try for a
|
||||
//! contiguous run) and fall back to `F_ALLOCATEALL` (non-contig OK).
|
||||
//! Reported file size is unchanged — the muxer's writes still grow it.
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
use crate::io::platform_macos::{
|
||||
F_ALLOCATEALL, F_ALLOCATECONTIG, F_PEOFPOSMODE, F_PREALLOCATE, Fstore,
|
||||
};
|
||||
|
||||
pub(super) fn preallocate_impl(file: &File, size_bytes: u64) {
|
||||
let fd = file.as_raw_fd();
|
||||
let mut store = Fstore {
|
||||
fst_flags: F_ALLOCATECONTIG,
|
||||
fst_posmode: F_PEOFPOSMODE,
|
||||
fst_offset: 0,
|
||||
fst_length: size_bytes as libc::off_t,
|
||||
fst_bytesalloc: 0,
|
||||
};
|
||||
let mut rc = unsafe { libc::fcntl(fd, F_PREALLOCATE, &mut store as *mut Fstore) };
|
||||
if rc == -1 {
|
||||
// Fall back to non-contiguous.
|
||||
store.fst_flags = F_ALLOCATEALL;
|
||||
rc = unsafe { libc::fcntl(fd, F_PREALLOCATE, &mut store as *mut Fstore) };
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink F_PREALLOCATE size_hint={size_bytes} rc={rc} bytesalloc={}",
|
||||
store.fst_bytesalloc
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! Per-OS extent preallocation. Best-effort; failures are logged at
|
||||
//! debug and otherwise swallowed because the file is still usable
|
||||
//! without the size reservation — only large-file fragmentation gets
|
||||
//! marginally worse.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
mod other;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux::preallocate_impl;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos::preallocate_impl;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
use other::preallocate_impl;
|
||||
|
||||
/// Reserve `size_bytes` of disk space for `file`'s on-disk extents.
|
||||
/// Reported file size is unchanged — writes still grow the file
|
||||
/// naturally; only the allocator's extent map is primed.
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
preallocate_impl(file, size_bytes);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
//! Fallback preallocate impl. No-op.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn preallocate_impl(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink preallocate size_hint={size_bytes} skipped (no platform impl)"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
//! TCP / UDP socket sinks (sequential-only).
|
||||
//!
|
||||
//! [`SocketSink`] wraps a `TcpStream` in a 1 MiB `BufWriter`. Constructor
|
||||
//! tunes `SO_SNDBUF` to a caller hint when provided. `finish()` flushes
|
||||
//! the buffer then `shutdown(Write)`s the socket so the peer sees clean
|
||||
//! end-of-stream.
|
||||
//!
|
||||
//! [`UdpSocketSink`] wraps a connected `UdpSocket`. Each `write` call
|
||||
//! emits exactly one datagram — the caller is responsible for packetizing
|
||||
//! to a reasonable MTU (188 × 7 = 1316 bytes for MPEG-TS-over-UDP is the
|
||||
//! conventional choice). `finish()` is a no-op; UDP has no end-of-stream
|
||||
//! marker.
|
||||
//!
|
||||
//! Both types satisfy [`SequentialSink`] via the blanket impl in
|
||||
//! `super::mod`. Neither implements `Seek`, so neither satisfies
|
||||
//! [`RandomAccessSink`] — using one with `MkvMux` is a compile error,
|
||||
//! which is the design intent.
|
||||
//!
|
||||
//! [`SequentialSink`]: super::SequentialSink
|
||||
//! [`RandomAccessSink`]: super::RandomAccessSink
|
||||
|
||||
use std::io::{self, BufWriter, Write};
|
||||
use std::net::{Shutdown, TcpStream, ToSocketAddrs, UdpSocket};
|
||||
|
||||
/// `BufWriter` capacity for [`SocketSink`]. 1 MiB matches the typical
|
||||
/// kernel send-buffer ceiling and keeps small-write amplification from
|
||||
/// containers (TS = 188-byte packets, fMP4 fragment headers = ~100 bytes)
|
||||
/// from translating into syscall storms.
|
||||
const TCP_BUF_CAPACITY: usize = 1024 * 1024;
|
||||
|
||||
/// Sequential-only sink over a TCP connection.
|
||||
///
|
||||
/// Wraps a `BufWriter<TcpStream>`; the inner `TcpStream` is kept as a
|
||||
/// clone so [`finish`](Self::finish) can call `shutdown(Write)` after
|
||||
/// flushing the buffer (the buffered writer doesn't expose the socket
|
||||
/// directly).
|
||||
pub struct SocketSink {
|
||||
/// Buffered write half. All payload bytes go through this.
|
||||
buf: BufWriter<TcpStream>,
|
||||
/// Shutdown handle — clone of the socket inside `buf`. Used only by
|
||||
/// `finish()` for `shutdown(Write)`; never read or written through.
|
||||
shutdown_handle: TcpStream,
|
||||
}
|
||||
|
||||
impl SocketSink {
|
||||
/// Open a TCP connection to `addr` and wrap it for sequential
|
||||
/// writing. `sndbuf_bytes`, when present, is forwarded to
|
||||
/// `setsockopt(SO_SNDBUF)` as a kernel hint — the OS may clamp it.
|
||||
///
|
||||
/// `addr` accepts anything `ToSocketAddrs` does: `"10.0.0.1:1234"`,
|
||||
/// `("host", 1234)`, a `SocketAddr`, etc.
|
||||
pub fn connect<A: ToSocketAddrs>(addr: A, sndbuf_bytes: Option<usize>) -> io::Result<Self> {
|
||||
let stream = TcpStream::connect(addr)?;
|
||||
// `set_nodelay(true)` keeps small writes (TS packet trains, fMP4
|
||||
// moof headers) from sitting in Nagle's algorithm until the buffer
|
||||
// fills. The BufWriter already absorbs syscall overhead; Nagle
|
||||
// would just add latency without coalescing more.
|
||||
stream.set_nodelay(true)?;
|
||||
if let Some(n) = sndbuf_bytes {
|
||||
set_send_buffer(&stream, n)?;
|
||||
}
|
||||
let shutdown_handle = stream.try_clone()?;
|
||||
Ok(Self {
|
||||
buf: BufWriter::with_capacity(TCP_BUF_CAPACITY, stream),
|
||||
shutdown_handle,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for SocketSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.buf.write(buf)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.buf.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl SocketSink {
|
||||
/// Drain the BufWriter and `shutdown(Write)` the underlying socket
|
||||
/// so the peer sees a clean EOF.
|
||||
///
|
||||
/// Note: [`SequentialSink::finish`](super::SequentialSink::finish)'s
|
||||
/// blanket-impl default is a no-op. Trait-object call sites that
|
||||
/// need socket shutdown should call this inherent method directly
|
||||
/// before dropping the sink, or hold the concrete `SocketSink` type
|
||||
/// (typical pattern: each muxer's `finish()` calls the appropriate
|
||||
/// inherent close method on its captured concrete sink).
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
self.buf.flush()?;
|
||||
// `shutdown(Write)` signals clean EOF to the peer. Errors here
|
||||
// are non-fatal — the connection may have already been torn down
|
||||
// by the peer — but we surface them so callers can log.
|
||||
self.shutdown_handle.shutdown(Shutdown::Write)
|
||||
}
|
||||
}
|
||||
|
||||
/// Sequential-only sink over a connected UDP socket.
|
||||
///
|
||||
/// Each [`write`](Write::write) call sends exactly one datagram. The
|
||||
/// caller is responsible for splitting payload at packet boundaries —
|
||||
/// for MPEG-TS this means 7 × 188 = 1316 bytes per datagram, the
|
||||
/// industry standard for MPEG-TS-over-UDP. No buffering happens here;
|
||||
/// adding it would silently merge datagrams.
|
||||
///
|
||||
/// `finish()` is a no-op: UDP has no end-of-stream marker. Closing the
|
||||
/// socket happens on drop.
|
||||
pub struct UdpSocketSink {
|
||||
socket: UdpSocket,
|
||||
}
|
||||
|
||||
impl UdpSocketSink {
|
||||
/// Bind a local UDP socket to an ephemeral port and `connect` it to
|
||||
/// `peer`. `connect` doesn't open a connection — it just fixes the
|
||||
/// peer address so subsequent `send` calls don't need to repeat it,
|
||||
/// and so receive-side filtering rejects packets from other sources.
|
||||
///
|
||||
/// `sndbuf_bytes`, when present, is a hint to `SO_SNDBUF`.
|
||||
pub fn connect<A: ToSocketAddrs>(peer: A, sndbuf_bytes: Option<usize>) -> io::Result<Self> {
|
||||
// Bind to all-zeros / any port. The kernel picks an ephemeral
|
||||
// source port and the source IP at first send.
|
||||
let socket = UdpSocket::bind("0.0.0.0:0")?;
|
||||
socket.connect(peer)?;
|
||||
if let Some(n) = sndbuf_bytes {
|
||||
set_udp_send_buffer(&socket, n)?;
|
||||
}
|
||||
Ok(Self { socket })
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for UdpSocketSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
// `send` writes the entire datagram or fails — no partial sends
|
||||
// for UDP. Match `Write::write`'s contract by reporting bytes
|
||||
// accepted.
|
||||
self.socket.send(buf)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl UdpSocketSink {
|
||||
/// No-op — UDP has no end-of-stream marker. Provided for parity
|
||||
/// with [`SocketSink::finish`] so call sites can treat them uniformly.
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ── Platform `SO_SNDBUF` tuning ────────────────────────────────────────────
|
||||
//
|
||||
// std's `TcpStream` / `UdpSocket` don't expose `SO_SNDBUF`. We drop to
|
||||
// libc on Linux + macOS (the libc-dep targets in Cargo.toml). On other
|
||||
// targets the hint is silently ignored — the socket still works, the
|
||||
// kernel just picks its own send-buffer size.
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn set_send_buffer(stream: &TcpStream, bytes: usize) -> io::Result<()> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
setsockopt_sndbuf(stream.as_raw_fd(), bytes)
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn set_udp_send_buffer(socket: &UdpSocket, bytes: usize) -> io::Result<()> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
setsockopt_sndbuf(socket.as_raw_fd(), bytes)
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
fn set_send_buffer(_stream: &TcpStream, _bytes: usize) -> io::Result<()> {
|
||||
// Non-Linux-non-macOS targets aren't in Cargo.toml's libc dep list;
|
||||
// silently ignore the hint rather than failing the connect. Callers
|
||||
// can detect via the lack of an explicit "sndbuf applied" signal
|
||||
// (not provided, intentionally — this is a hint, not a guarantee).
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
fn set_udp_send_buffer(_socket: &UdpSocket, _bytes: usize) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn setsockopt_sndbuf(fd: std::os::unix::io::RawFd, bytes: usize) -> io::Result<()> {
|
||||
// Clamp into c_int range; SO_SNDBUF takes an `int` argument.
|
||||
let want: libc::c_int = bytes.try_into().unwrap_or(libc::c_int::MAX);
|
||||
let ret = unsafe {
|
||||
libc::setsockopt(
|
||||
fd,
|
||||
libc::SOL_SOCKET,
|
||||
libc::SO_SNDBUF,
|
||||
&want as *const _ as *const libc::c_void,
|
||||
std::mem::size_of::<libc::c_int>() as libc::socklen_t,
|
||||
)
|
||||
};
|
||||
if ret != 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
use std::net::{TcpListener, UdpSocket};
|
||||
use std::thread;
|
||||
|
||||
/// Bind a listener, accept on a thread, return (listener_addr,
|
||||
/// accepted-bytes future via JoinHandle).
|
||||
#[test]
|
||||
fn socket_sink_round_trips_bytes() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let accept = thread::spawn(move || {
|
||||
let (mut sock, _) = listener.accept().unwrap();
|
||||
let mut buf = Vec::new();
|
||||
sock.read_to_end(&mut buf).unwrap();
|
||||
buf
|
||||
});
|
||||
|
||||
let mut sink = SocketSink::connect(addr, Some(256 * 1024)).unwrap();
|
||||
// Write enough to overflow the BufWriter at least once, then a
|
||||
// tail that lives in the buffer until `finish` flushes.
|
||||
let big: Vec<u8> = (0..(2 * TCP_BUF_CAPACITY))
|
||||
.map(|i| (i & 0xff) as u8)
|
||||
.collect();
|
||||
sink.write_all(&big).unwrap();
|
||||
sink.write_all(b"tail\n").unwrap();
|
||||
sink.finish().unwrap();
|
||||
drop(sink);
|
||||
|
||||
let received = accept.join().unwrap();
|
||||
assert_eq!(received.len(), big.len() + 5);
|
||||
assert_eq!(&received[..big.len()], &big[..]);
|
||||
assert_eq!(&received[big.len()..], b"tail\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn socket_sink_is_sequential_only() {
|
||||
// Compile-time assertion via dyn — if this ever started
|
||||
// satisfying `RandomAccessSink`, the trait split would be broken.
|
||||
fn _assert_seq(_: &mut dyn super::super::SequentialSink) {}
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let _accept = thread::spawn(move || {
|
||||
let _ = listener.accept();
|
||||
});
|
||||
let mut sink = SocketSink::connect(addr, None).unwrap();
|
||||
_assert_seq(&mut sink);
|
||||
// The negative is harder to assert directly (no `is_not<T>`),
|
||||
// but `SocketSink` does not impl `Seek`, so it can't unify with
|
||||
// `RandomAccessSink`'s super-bound. The Phase 2 blanket impl
|
||||
// `impl<T: SequentialSink + Seek> RandomAccessSink for T {}` thus
|
||||
// excludes it by construction.
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn udp_socket_sink_delivers_datagrams() {
|
||||
let receiver = UdpSocket::bind("127.0.0.1:0").unwrap();
|
||||
receiver
|
||||
.set_read_timeout(Some(std::time::Duration::from_secs(2)))
|
||||
.unwrap();
|
||||
let addr = receiver.local_addr().unwrap();
|
||||
let mut sink = UdpSocketSink::connect(addr, Some(128 * 1024)).unwrap();
|
||||
|
||||
sink.write_all(&[1, 2, 3, 4, 5]).unwrap();
|
||||
sink.write_all(&[9, 9, 9]).unwrap();
|
||||
sink.finish().unwrap();
|
||||
|
||||
let mut buf = [0u8; 64];
|
||||
let n1 = receiver.recv(&mut buf).unwrap();
|
||||
assert_eq!(&buf[..n1], &[1, 2, 3, 4, 5]);
|
||||
let n2 = receiver.recv(&mut buf).unwrap();
|
||||
assert_eq!(&buf[..n2], &[9, 9, 9]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
//! Per-platform writeback pipeline. On Linux, drains dirty pages
|
||||
//! continuously at chunk granularity to keep the kernel's writeback
|
||||
//! queue bounded. On macOS and Windows, a no-op stub.
|
||||
//!
|
||||
//! The platform decision lives entirely in this file (the cfg-gated
|
||||
//! `pub use` below). Callers — and `DiskWriter` itself — are
|
||||
//! platform-independent.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
mod noop;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) use linux::WritebackPipeline;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub(super) use noop::WritebackPipeline;
|
||||
@@ -0,0 +1,459 @@
|
||||
//! Linux writeback pipeline using `sync_file_range` + `posix_fadvise`.
|
||||
//!
|
||||
//! Pathology this fixes: the kernel's default `vm.dirty_ratio` (~20 %
|
||||
//! of RAM) lets dirty pages accumulate to hundreds of MB during a
|
||||
//! big sequential write, then bursts a flush at 99 % disk utilisation.
|
||||
//! While the burst runs, app writes block on the writeback queue —
|
||||
//! observed empirically as instantaneous speed dropping from ~15 MB/s
|
||||
//! to ~1 MB/s every ~30 s during a Pass 1 sweep.
|
||||
//!
|
||||
//! Strategy: every `chunk_bytes` of new sequential output, kick async
|
||||
//! writeback (`SYNC_FILE_RANGE_WRITE`) on the just-completed chunk and
|
||||
//! finalise the *previous* chunk via `WAIT_AFTER` + `posix_fadvise
|
||||
//! (DONTNEED)`. By the time we finalise, that previous chunk has had
|
||||
//! a full chunk's worth of work to flush — the wait is near-instant.
|
||||
//! Dirty cache stays bounded at ~2 × `chunk_bytes` and writes drain
|
||||
//! continuously instead of in bursts.
|
||||
//!
|
||||
//! The chunk size is adaptive: we measure the elapsed time of the
|
||||
//! `WAIT_AFTER` call over a rolling window of the last 16 chunks and
|
||||
//! resize the chunk based on the p95. Slow storage (NFS, network
|
||||
//! shares, HDD) sees larger chunks to amortise per-chunk overhead;
|
||||
//! fast storage (NVMe) sees smaller chunks to keep cache pressure
|
||||
//! tight. Bounds: [4 MiB, 256 MiB].
|
||||
//!
|
||||
//! ## NFS escape hatch
|
||||
//!
|
||||
//! `sync_file_range(WAIT_AFTER)` on an NFS-mounted file can block
|
||||
//! indefinitely waiting for the server's commit ack. If the server
|
||||
//! never acks (network partition, server-side hang, slow commit), the
|
||||
//! syscall never returns and the consumer thread is stuck inside the
|
||||
//! kernel — `/api/stop` can't reach it because halt is cooperative.
|
||||
//!
|
||||
//! When `fstatfs` reports the file lives on an NFS mount
|
||||
//! (`f_type == NFS_SUPER_MAGIC`), the pipeline skips the WAIT_AFTER +
|
||||
//! `posix_fadvise(DONTNEED)` dance entirely. NFS clients have their
|
||||
//! own buffering and commit semantics that handle dirty-page bounds
|
||||
//! without us forcing the issue. The async `SYNC_FILE_RANGE_WRITE`
|
||||
//! kickoff still runs (non-blocking by spec) so writeback still gets
|
||||
//! a nudge.
|
||||
//!
|
||||
//! ## Defence in depth: WAIT_AFTER timeout
|
||||
//!
|
||||
//! Even on local storage, a degraded disk or odd filesystem driver
|
||||
//! could in principle wedge inside WAIT_AFTER. Each WAIT_AFTER call
|
||||
//! runs on a worker thread with a 30s recv_timeout on its result
|
||||
//! channel. On timeout we log a loud error, set a `degraded` flag,
|
||||
//! and from then on skip WAIT_AFTER + DONTNEED for the rest of the
|
||||
//! pipeline's life (same shape as the NFS path). The worker thread
|
||||
//! is intentionally leaked — it unwinds whenever the syscall
|
||||
//! eventually returns or the process exits. The mux continues; the
|
||||
//! original dirty-burst pathology re-emerges but the rip can still
|
||||
//! finish instead of freezing.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::{AsRawFd, RawFd};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const ADAPTIVE_WINDOW: usize = 16;
|
||||
const CHUNK_BYTES_MIN: u64 = 4 * 1024 * 1024;
|
||||
const CHUNK_BYTES_MAX: u64 = 256 * 1024 * 1024;
|
||||
const ADAPTIVE_GROW_MS: u64 = 200;
|
||||
const ADAPTIVE_SHRINK_MS: u64 = 20;
|
||||
/// Every N chunks, emit a `debug!` snapshot of the current chunk
|
||||
/// size so operators tailing the log can see where the autoscaler
|
||||
/// settled.
|
||||
const SIZE_LOG_INTERVAL: u64 = 32;
|
||||
/// Hard upper bound on a single `sync_file_range(WAIT_AFTER)` call.
|
||||
/// Beyond this we declare the pipeline degraded and stop calling
|
||||
/// WAIT_AFTER for the rest of its life.
|
||||
const WAIT_AFTER_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
pub(crate) struct WritebackPipeline {
|
||||
/// Aliases the wrapping `WritebackFile::file`. Only valid for the
|
||||
/// lifetime of that struct — moving the `File` independently
|
||||
/// would silently UAF this fd. The pipeline is a private field of
|
||||
/// `WritebackFile` and never exposed outside that wrapper, which
|
||||
/// is what keeps the alias sound.
|
||||
fd: RawFd,
|
||||
chunk_bytes: u64,
|
||||
last_flush_pos: u64,
|
||||
pending: Option<(u64, u64)>,
|
||||
/// Rolling window of recent `WAIT_AFTER` elapsed_ms measurements.
|
||||
wait_after_window: VecDeque<u64>,
|
||||
/// Count of chunks emitted (used to space out periodic
|
||||
/// `debug!` size snapshots).
|
||||
chunk_count: u64,
|
||||
/// True when the underlying file is on an NFS mount. NFS makes
|
||||
/// WAIT_AFTER unsafe (can block forever on missing server ack), so
|
||||
/// we skip it entirely and let the NFS client handle commit on
|
||||
/// close.
|
||||
is_nfs: bool,
|
||||
/// Set the first time WAIT_AFTER exceeds [`WAIT_AFTER_TIMEOUT`].
|
||||
/// Once set, behaviour matches the NFS path for the rest of the
|
||||
/// pipeline's life. Wrapped in `Arc` only because both this
|
||||
/// struct and the spawned worker thread (which itself doesn't
|
||||
/// touch the flag) share-via-fd patterns might one day need it;
|
||||
/// today it's effectively a single-owner cell — the `Arc` shape
|
||||
/// keeps the door open for moving the read side into a worker
|
||||
/// without re-plumbing types.
|
||||
degraded: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl WritebackPipeline {
|
||||
/// Construct a pipeline aliasing `file`'s file descriptor. The
|
||||
/// returned `WritebackPipeline` MUST be dropped before `file`
|
||||
/// itself, or kept inside the same struct that owns `file` — the
|
||||
/// alias is unchecked.
|
||||
pub(crate) fn new(file: &File, start_pos: u64, chunk_bytes: u64) -> Self {
|
||||
let fd = file.as_raw_fd();
|
||||
let is_nfs = detect_nfs(fd);
|
||||
tracing::info!(
|
||||
target: "mux",
|
||||
"WritebackPipeline fd={fd} is_nfs={is_nfs} chunk_bytes={chunk_bytes} strategy={}",
|
||||
if is_nfs { "nfs-skip-wait" } else { "wait+dontneed" }
|
||||
);
|
||||
Self {
|
||||
fd,
|
||||
chunk_bytes,
|
||||
last_flush_pos: start_pos,
|
||||
pending: None,
|
||||
wait_after_window: VecDeque::with_capacity(ADAPTIVE_WINDOW),
|
||||
chunk_count: 0,
|
||||
is_nfs,
|
||||
degraded: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
/// True if we should bypass the WAIT_AFTER + DONTNEED finalisation
|
||||
/// step. NFS always bypasses; local storage bypasses once the
|
||||
/// pipeline has flipped to degraded after a WAIT_AFTER timeout.
|
||||
#[inline]
|
||||
fn skip_wait(&self) -> bool {
|
||||
self.is_nfs || self.degraded.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Caller advanced the file position to `pos`. If a chunk boundary
|
||||
/// was crossed, kick async writeback for the just-completed chunk
|
||||
/// and finalise the previous one.
|
||||
pub(crate) fn note_progress(&mut self, pos: u64) {
|
||||
if pos < self.last_flush_pos.saturating_add(self.chunk_bytes) {
|
||||
return;
|
||||
}
|
||||
let chunk_off = self.last_flush_pos as i64;
|
||||
let chunk_len = (pos - self.last_flush_pos) as i64;
|
||||
let mut wait_ms: u64 = 0;
|
||||
let mut fadvise_ms: u64 = 0;
|
||||
// Async kickoff for the just-completed chunk runs on every
|
||||
// path (NFS, degraded, normal) — it's nominally non-blocking
|
||||
// by spec and gives the kernel an early hint that this range
|
||||
// is ready to flush.
|
||||
unsafe {
|
||||
libc::sync_file_range(self.fd, chunk_off, chunk_len, libc::SYNC_FILE_RANGE_WRITE);
|
||||
}
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
if self.skip_wait() {
|
||||
// NFS branch (or degraded fallback after a prior
|
||||
// timeout): the WAIT_AFTER + DONTNEED dance is what
|
||||
// hangs on NFS — skip it. We still advance `pending`
|
||||
// so the next call has a stable cycle.
|
||||
} else {
|
||||
// Normal local-storage branch with belt-and-braces
|
||||
// timeout. If WAIT_AFTER hangs > WAIT_AFTER_TIMEOUT
|
||||
// we mark the pipeline degraded, log a loud error,
|
||||
// and fall through to the skip path on subsequent
|
||||
// calls.
|
||||
match wait_after_with_timeout(self.fd, prev_off, prev_len) {
|
||||
Some(ms) => {
|
||||
wait_ms = ms;
|
||||
let t_fadv = Instant::now();
|
||||
unsafe {
|
||||
libc::posix_fadvise(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
}
|
||||
fadvise_ms = t_fadv.elapsed().as_millis() as u64;
|
||||
self.record_wait(wait_ms);
|
||||
}
|
||||
None => {
|
||||
// Timeout branch: switch to NFS-style skip
|
||||
// for the rest of the pipeline's life. Do
|
||||
// NOT call DONTNEED — if WAIT_AFTER hasn't
|
||||
// returned, the pages aren't safely flushed.
|
||||
self.degraded.store(true, Ordering::Relaxed);
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackPipeline WAIT_AFTER timed out after {}s on chunk off={} len={}, marking writeback degraded (subsequent chunks will skip WAIT_AFTER + DONTNEED)",
|
||||
WAIT_AFTER_TIMEOUT.as_secs(),
|
||||
prev_off,
|
||||
prev_len
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.pending = Some((chunk_off as u64, chunk_len as u64));
|
||||
self.last_flush_pos = pos;
|
||||
self.chunk_count += 1;
|
||||
tracing::trace!(
|
||||
target: "mux",
|
||||
"WritebackPipeline chunk off={} len={} sync_file_range_ms={wait_ms} fadvise_ms={fadvise_ms} chunk_bytes={} skip_wait={}",
|
||||
chunk_off,
|
||||
chunk_len,
|
||||
self.chunk_bytes,
|
||||
self.skip_wait(),
|
||||
);
|
||||
if self.chunk_count % SIZE_LOG_INTERVAL == 0 {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackPipeline chunk_bytes={} after {} chunks is_nfs={} degraded={}",
|
||||
self.chunk_bytes,
|
||||
self.chunk_count,
|
||||
self.is_nfs,
|
||||
self.degraded.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Push a new `WAIT_AFTER` measurement into the rolling window
|
||||
/// and, if the window is full, adapt `chunk_bytes` based on p95.
|
||||
fn record_wait(&mut self, wait_ms: u64) {
|
||||
if self.wait_after_window.len() == ADAPTIVE_WINDOW {
|
||||
self.wait_after_window.pop_front();
|
||||
}
|
||||
self.wait_after_window.push_back(wait_ms);
|
||||
if self.wait_after_window.len() < ADAPTIVE_WINDOW {
|
||||
return;
|
||||
}
|
||||
// p95 of 16 samples ≈ sorted[14] (5 % of 16 = 0.8 ≈ 1 above).
|
||||
let mut sorted: Vec<u64> = self.wait_after_window.iter().copied().collect();
|
||||
sorted.sort_unstable();
|
||||
let p95 = sorted[14];
|
||||
let old = self.chunk_bytes;
|
||||
let new = if p95 > ADAPTIVE_GROW_MS && self.chunk_bytes < CHUNK_BYTES_MAX {
|
||||
(self.chunk_bytes * 2).min(CHUNK_BYTES_MAX)
|
||||
} else if p95 < ADAPTIVE_SHRINK_MS && self.chunk_bytes > CHUNK_BYTES_MIN {
|
||||
(self.chunk_bytes / 2).max(CHUNK_BYTES_MIN)
|
||||
} else {
|
||||
self.chunk_bytes
|
||||
};
|
||||
if new != old {
|
||||
self.chunk_bytes = new;
|
||||
tracing::info!(
|
||||
target: "mux",
|
||||
"WritebackPipeline adaptive chunk_bytes {} -> {} p95_ms={p95}",
|
||||
old,
|
||||
new
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Caller is about to seek away from the current write region.
|
||||
/// Drain any in-flight chunk and reset tracking.
|
||||
pub(crate) fn handle_seek(&mut self, new_pos: u64) {
|
||||
self.finalize();
|
||||
self.last_flush_pos = new_pos;
|
||||
}
|
||||
|
||||
/// Drain any in-flight chunk. Idempotent. Call before `sync_all()`
|
||||
/// or when discarding the pipeline.
|
||||
pub(crate) fn finalize(&mut self) {
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackPipeline finalize chunk off={prev_off} len={prev_len} skip_wait={} is_nfs={} degraded={}",
|
||||
self.skip_wait(),
|
||||
self.is_nfs,
|
||||
self.degraded.load(Ordering::Relaxed),
|
||||
);
|
||||
if self.skip_wait() {
|
||||
// NFS / degraded: skip WAIT_AFTER + DONTNEED. close()
|
||||
// / sync_all() handle commit through their normal
|
||||
// paths.
|
||||
return;
|
||||
}
|
||||
match wait_after_with_timeout(self.fd, prev_off, prev_len) {
|
||||
Some(_ms) => unsafe {
|
||||
libc::posix_fadvise(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
},
|
||||
None => {
|
||||
self.degraded.store(true, Ordering::Relaxed);
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackPipeline finalize WAIT_AFTER timed out after {}s on chunk off={prev_off} len={prev_len}, marking writeback degraded",
|
||||
WAIT_AFTER_TIMEOUT.as_secs(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe whether `fd` lives on an NFS mount. Thin wrapper around
|
||||
/// [`crate::platform::fs_type::detect_fd`] so writeback policy and
|
||||
/// general-purpose fs-type classification stay in sync (same magic
|
||||
/// numbers, same musl-vs-glibc cast handling).
|
||||
///
|
||||
/// Fails open: any classification other than NFS counts as "not NFS"
|
||||
/// (including `Unknown` on `fstatfs` error) — better to run the
|
||||
/// normal local-storage path on a misdetected NFS mount and surface
|
||||
/// the freeze loudly via [`WAIT_AFTER_TIMEOUT`] than to needlessly
|
||||
/// disable writeback bounding on every local file because of a
|
||||
/// transient stat error.
|
||||
fn detect_nfs(fd: RawFd) -> bool {
|
||||
matches!(
|
||||
crate::platform::fs_type::detect_fd(fd),
|
||||
crate::platform::fs_type::FsType::Nfs
|
||||
)
|
||||
}
|
||||
|
||||
/// Run `sync_file_range(WAIT_AFTER)` on a worker thread and wait up
|
||||
/// to [`WAIT_AFTER_TIMEOUT`] for it to return. `Some(elapsed_ms)` on
|
||||
/// success; `None` on timeout. On timeout the worker thread is
|
||||
/// 0.20.6 generalizes the worker-thread + recv_timeout pattern into
|
||||
/// [`crate::io::bounded::bounded_syscall`]; this helper now just adapts
|
||||
/// the generic primitive to the WAIT_AFTER call shape (returns elapsed_ms
|
||||
/// instead of the syscall's `()` return, treats `WorkerLost` as a benign
|
||||
/// no-op to match the original semantics).
|
||||
fn wait_after_with_timeout(fd: RawFd, off: u64, len: u64) -> Option<u64> {
|
||||
let started = Instant::now();
|
||||
match crate::io::bounded::bounded_syscall(None, WAIT_AFTER_TIMEOUT, move || unsafe {
|
||||
libc::sync_file_range(fd, off as i64, len as i64, libc::SYNC_FILE_RANGE_WAIT_AFTER);
|
||||
}) {
|
||||
Ok(()) => Some(started.elapsed().as_millis() as u64),
|
||||
Err(crate::io::bounded::BoundedError::Timeout)
|
||||
| Err(crate::io::bounded::BoundedError::Halted) => None,
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => {
|
||||
// Worker thread spawn failed or panicked before sending.
|
||||
// Treat as a benign success (no syscall ran) rather than
|
||||
// a degrade trigger — falling through with elapsed_ms=0
|
||||
// matches the no-op behaviour.
|
||||
Some(0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
/// Helper: build a `WritebackPipeline` over a local tempfile. On
|
||||
/// every test rig (linux dev box, CI) the tempfile lives on a
|
||||
/// local FS, so `is_nfs=false` and `skip_wait` returns false until
|
||||
/// we explicitly mark the pipeline degraded.
|
||||
fn local_pipeline(chunk_bytes: u64) -> (NamedTempFile, WritebackPipeline) {
|
||||
let f = NamedTempFile::new().expect("tempfile create");
|
||||
let pipeline = WritebackPipeline::new(f.as_file(), 0, chunk_bytes);
|
||||
(f, pipeline)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_pipeline_starts_active() {
|
||||
let (_f, p) = local_pipeline(32 * 1024 * 1024);
|
||||
assert!(!p.is_nfs, "local tempfile must not classify as NFS");
|
||||
assert!(!p.degraded.load(Ordering::Relaxed));
|
||||
assert!(!p.skip_wait(), "fresh local pipeline must not skip wait");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn degraded_flag_short_circuits_wait() {
|
||||
let (_f, p) = local_pipeline(32 * 1024 * 1024);
|
||||
assert!(!p.skip_wait());
|
||||
p.degraded.store(true, Ordering::Relaxed);
|
||||
assert!(
|
||||
p.skip_wait(),
|
||||
"degraded flag must force the wait+dontneed bypass"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_wait_grows_chunk_on_high_p95() {
|
||||
let (_f, mut p) = local_pipeline(16 * 1024 * 1024);
|
||||
// Fill the window with samples above the grow threshold.
|
||||
for _ in 0..ADAPTIVE_WINDOW {
|
||||
p.record_wait(ADAPTIVE_GROW_MS + 50);
|
||||
}
|
||||
assert!(
|
||||
p.chunk_bytes > 16 * 1024 * 1024,
|
||||
"chunk should have grown; got {}",
|
||||
p.chunk_bytes
|
||||
);
|
||||
assert!(p.chunk_bytes <= CHUNK_BYTES_MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_wait_shrinks_chunk_on_low_p95() {
|
||||
let (_f, mut p) = local_pipeline(64 * 1024 * 1024);
|
||||
for _ in 0..ADAPTIVE_WINDOW {
|
||||
p.record_wait(1); // well under ADAPTIVE_SHRINK_MS
|
||||
}
|
||||
assert!(
|
||||
p.chunk_bytes < 64 * 1024 * 1024,
|
||||
"chunk should have shrunk; got {}",
|
||||
p.chunk_bytes
|
||||
);
|
||||
assert!(p.chunk_bytes >= CHUNK_BYTES_MIN);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_wait_no_op_below_window_fill() {
|
||||
let (_f, mut p) = local_pipeline(16 * 1024 * 1024);
|
||||
let initial = p.chunk_bytes;
|
||||
// Only push a few samples; window not full → no adaptation.
|
||||
for _ in 0..(ADAPTIVE_WINDOW - 1) {
|
||||
p.record_wait(ADAPTIVE_GROW_MS + 100);
|
||||
}
|
||||
assert_eq!(
|
||||
p.chunk_bytes, initial,
|
||||
"chunk must not change before window is full"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_wait_clamps_to_chunk_bounds() {
|
||||
// Grow past the max.
|
||||
let (_f, mut p) = local_pipeline(CHUNK_BYTES_MAX);
|
||||
for _ in 0..ADAPTIVE_WINDOW {
|
||||
p.record_wait(ADAPTIVE_GROW_MS + 1000);
|
||||
}
|
||||
assert_eq!(p.chunk_bytes, CHUNK_BYTES_MAX, "must clamp to MAX");
|
||||
|
||||
// Shrink past the min.
|
||||
let (_f, mut p) = local_pipeline(CHUNK_BYTES_MIN);
|
||||
for _ in 0..ADAPTIVE_WINDOW {
|
||||
p.record_wait(0);
|
||||
}
|
||||
assert_eq!(p.chunk_bytes, CHUNK_BYTES_MIN, "must clamp to MIN");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_nfs_local_file_is_false() {
|
||||
// Local tempfile must not classify as NFS. This locks in the
|
||||
// consolidation through `crate::platform::fs_type::detect_fd`.
|
||||
let f = NamedTempFile::new().expect("tempfile create");
|
||||
use std::os::unix::io::AsRawFd;
|
||||
assert!(!detect_nfs(f.as_file().as_raw_fd()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn note_progress_below_chunk_is_noop() {
|
||||
let (_f, mut p) = local_pipeline(32 * 1024 * 1024);
|
||||
// No-op return before crossing the first chunk boundary.
|
||||
let before = p.chunk_count;
|
||||
p.note_progress(1024); // < 32 MiB
|
||||
assert_eq!(p.chunk_count, before);
|
||||
assert!(p.pending.is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//! No-op writeback pipeline for non-Linux targets. macOS and Windows
|
||||
//! page cache policies have not been shown to exhibit the Linux
|
||||
//! accumulate-then-burst flush pathology for our access pattern.
|
||||
//! If that changes, replace this stub with a real implementation
|
||||
//! (e.g. `F_NOCACHE` on macOS, `FILE_FLAG_WRITE_THROUGH` on Windows).
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(crate) struct WritebackPipeline;
|
||||
|
||||
impl WritebackPipeline {
|
||||
pub(crate) fn new(_file: &File, _start_pos: u64, _chunk_bytes: u64) -> Self {
|
||||
Self
|
||||
}
|
||||
pub(crate) fn note_progress(&mut self, _pos: u64) {}
|
||||
pub(crate) fn handle_seek(&mut self, _new_pos: u64) {}
|
||||
pub(crate) fn finalize(&mut self) {}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//! Linux platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! - `preallocate`: `fallocate(FALLOC_FL_KEEP_SIZE)` — reserve extents
|
||||
//! without growing the reported file size. Reduces extent
|
||||
//! fragmentation on large sequential writes (mux output on NFS in
|
||||
//! particular).
|
||||
//! - `durable_sync`: `fsync` wrapped in
|
||||
//! [`crate::io::bounded::bounded_syscall`] with a 60 s deadline so a
|
||||
//! wedged NFS server can't trap the calling thread indefinitely.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Pre-reserve extents for `size_bytes` of upcoming sequential writes.
|
||||
/// Best-effort: a non-zero rc is logged but not propagated, since the
|
||||
/// caller would just continue with the unreserved file anyway.
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
// FALLOC_FL_KEEP_SIZE = 0x01 — keep the reported file size at 0
|
||||
// (writes grow it normally) while still pre-reserving the extents.
|
||||
let rc = unsafe {
|
||||
libc::fallocate(
|
||||
file.as_raw_fd(),
|
||||
libc::FALLOC_FL_KEEP_SIZE,
|
||||
0,
|
||||
size_bytes as i64,
|
||||
)
|
||||
};
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile fallocate size_hint={size_bytes} rc={rc} ok={}",
|
||||
rc == 0
|
||||
);
|
||||
}
|
||||
|
||||
/// Run `fsync` on `file` with a 60 s deadline. On timeout we log loudly
|
||||
/// and return `Ok(())` — the kernel will still flush on close, so the
|
||||
/// data is best-effort durable; the alternative (trap the thread for
|
||||
/// the rest of the rip) defeats `/api/stop`.
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
let fd = file.as_raw_fd();
|
||||
match crate::io::bounded::bounded_syscall(
|
||||
None,
|
||||
Duration::from_secs(60),
|
||||
move || -> io::Result<()> {
|
||||
let rc = unsafe { libc::fsync(fd) };
|
||||
if rc == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::last_os_error())
|
||||
}
|
||||
},
|
||||
) {
|
||||
Ok(inner) => inner,
|
||||
Err(crate::io::bounded::BoundedError::Timeout) => {
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all fsync timed out after 60s; kernel will flush on close (best-effort)"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(crate::io::bounded::BoundedError::Halted) => Ok(()),
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => Ok(()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
//! macOS platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! - `preallocate`: `fcntl(F_PREALLOCATE)` — macOS's fallocate-equiv.
|
||||
//! Reserves a contiguous extent when possible, falling back to a
|
||||
//! non-contiguous reservation if the FS can't satisfy it. Reported
|
||||
//! file size is unchanged (`F_ALLOCATEALL` is not set, so allocation
|
||||
//! is "best effort up to length"; growth happens via writes).
|
||||
//! - `durable_sync`: `fcntl(F_FULLFSYNC)` wrapped in
|
||||
//! [`crate::io::bounded::bounded_syscall`] with a 60 s deadline.
|
||||
//! F_FULLFSYNC is HFS+/APFS's true-fsync (flushes the disk's own
|
||||
//! write cache) — what `fsync` should have been on macOS. Falls back
|
||||
//! to plain `fsync` if F_FULLFSYNC returns ENOTSUP.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::io::platform_macos::{
|
||||
F_ALLOCATEALL, F_ALLOCATECONTIG, F_PEOFPOSMODE, F_PREALLOCATE, Fstore,
|
||||
};
|
||||
|
||||
/// `fcntl(F_FULLFSYNC)` opcode. Documented in `man 2 fcntl` on macOS;
|
||||
/// not in the `libc` crate as a named constant.
|
||||
const F_FULLFSYNC: libc::c_int = 51;
|
||||
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
let mut fst = Fstore {
|
||||
fst_flags: F_ALLOCATECONTIG | F_ALLOCATEALL,
|
||||
fst_posmode: F_PEOFPOSMODE,
|
||||
fst_offset: 0,
|
||||
fst_length: size_bytes as libc::off_t,
|
||||
fst_bytesalloc: 0,
|
||||
};
|
||||
// First attempt: contiguous.
|
||||
let mut rc = unsafe { libc::fcntl(file.as_raw_fd(), F_PREALLOCATE, &mut fst) };
|
||||
if rc == -1 {
|
||||
// Fall back: drop the contiguous hint, allow scattered extents.
|
||||
fst.fst_flags = F_ALLOCATEALL;
|
||||
rc = unsafe { libc::fcntl(file.as_raw_fd(), F_PREALLOCATE, &mut fst) };
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile F_PREALLOCATE size_hint={size_bytes} rc={rc} bytes_allocated={} ok={}",
|
||||
fst.fst_bytesalloc,
|
||||
rc != -1
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
let fd = file.as_raw_fd();
|
||||
match crate::io::bounded::bounded_syscall(
|
||||
None,
|
||||
Duration::from_secs(60),
|
||||
move || -> io::Result<()> {
|
||||
// Try F_FULLFSYNC first. If it isn't supported on this
|
||||
// filesystem (older HFS, some network mounts) fall back to
|
||||
// plain fsync — better than nothing.
|
||||
let rc = unsafe { libc::fcntl(fd, F_FULLFSYNC, 0) };
|
||||
if rc == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let err = io::Error::last_os_error();
|
||||
if err.raw_os_error() == Some(libc::ENOTSUP) {
|
||||
let rc = unsafe { libc::fsync(fd) };
|
||||
if rc == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::last_os_error())
|
||||
}
|
||||
} else {
|
||||
Err(err)
|
||||
}
|
||||
},
|
||||
) {
|
||||
Ok(inner) => inner,
|
||||
Err(crate::io::bounded::BoundedError::Timeout) => {
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all F_FULLFSYNC timed out after 60s; kernel will flush on close (best-effort)"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(crate::io::bounded::BoundedError::Halted) => Ok(()),
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => Ok(()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
//! `WritebackFile` — a `File` wrapper whose reason for existing is the
|
||||
//! bounded-cache writeback pipeline.
|
||||
//!
|
||||
//! Why: large sequential writes (sweep, patch, mux on UHD-scale output)
|
||||
//! left to the kernel's default writeback policy accumulate hundreds of
|
||||
//! megabytes of dirty pages and then burst-flush, stalling subsequent
|
||||
//! writes for seconds at a time. `WritebackFile` drives a continuous
|
||||
//! [`super::writeback::WritebackPipeline`] that on Linux issues
|
||||
//! incremental `sync_file_range` + `posix_fadvise(DONTNEED)` calls at
|
||||
//! 32 MB granularity so dirty pages drain at the same rate they're
|
||||
//! produced. macOS and Windows fall through to a no-op pipeline — their
|
||||
//! default cache policies have not been shown to exhibit the same
|
||||
//! pathology for this access pattern.
|
||||
//!
|
||||
//! It implements `Write` and `Seek` so any call site that wrote to a
|
||||
//! plain `File` through those traits (sweep, patch, mux) can swap in
|
||||
//! `WritebackFile` without touching the body of the loop. The wrapper
|
||||
//! also tracks the current file position to feed the pipeline with
|
||||
//! progress + seek boundaries.
|
||||
//!
|
||||
//! See `super::writeback::linux` for the underlying pathology and the
|
||||
//! strategy.
|
||||
//!
|
||||
//! ## Platform split
|
||||
//!
|
||||
//! The platform-specific pieces of this wrapper — extent preallocation
|
||||
//! (Linux `fallocate(KEEP_SIZE)`, macOS `F_PREALLOCATE`, Windows
|
||||
//! `SetFileValidData`) and the durable-flush primitive (Linux/macOS
|
||||
//! `fsync`/`F_FULLFSYNC` wrapped in a bounded syscall, Windows
|
||||
//! `FlushFileBuffers`) — live in per-OS sibling modules. The dispatch
|
||||
//! happens once at the bottom of this file via cfg-gated `mod` decls.
|
||||
//! No inline `#[cfg(target_os = "...")]` in the business-logic above.
|
||||
//!
|
||||
//! ## Write path
|
||||
//!
|
||||
//! Writes are direct passthrough to the underlying `File` (no writer
|
||||
//! thread, no ring, no batching). Empirically the Phase-2.5
|
||||
//! writer-thread architecture introduced a ~60% mux throughput
|
||||
//! regression on NFS bidirectional workloads; reverting the write path
|
||||
//! to direct passthrough restores the 0.20.7 baseline. The writeback
|
||||
//! pipeline still runs (it's called inline from `write` / `write_all` /
|
||||
//! `seek`) so the bounded-cache invariant on Linux is preserved.
|
||||
//!
|
||||
//! ## Halt-safety
|
||||
//!
|
||||
//! `sync_all` runs the per-OS durable-flush primitive, which on
|
||||
//! Linux/macOS is wrapped in [`crate::io::bounded::bounded_syscall`]
|
||||
//! with a 60 s deadline. A wedged NFS server cannot trap the muxer
|
||||
//! indefinitely on the final fsync.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
mod other;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux as platform;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos as platform;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
use other as platform;
|
||||
#[cfg(target_os = "windows")]
|
||||
use windows as platform;
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use super::writeback::WritebackPipeline;
|
||||
|
||||
/// Granularity at which the Linux writeback pipeline issues
|
||||
/// `sync_file_range` pairs. 32 MiB is the empirically best value on
|
||||
/// the rip1 test bed (NFS to unraid-1 over 1 GbE, single-disk SAS):
|
||||
/// 8 MiB / 64 MiB / 128 MiB all measured worse in the 0.21.x mux
|
||||
/// iteration runs. Override via `FREEMKV_WRITEBACK_CHUNK_MIB` —
|
||||
/// faster backends (NVMe, RAID) may tolerate larger windows.
|
||||
const WRITEBACK_CHUNK_BYTES_DEFAULT: u64 = 32 * 1024 * 1024;
|
||||
|
||||
fn writeback_chunk_bytes() -> u64 {
|
||||
std::env::var("FREEMKV_WRITEBACK_CHUNK_MIB")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.filter(|&n| n > 0)
|
||||
.map(|n| n * 1024 * 1024)
|
||||
.unwrap_or(WRITEBACK_CHUNK_BYTES_DEFAULT)
|
||||
}
|
||||
|
||||
pub(crate) struct WritebackFile {
|
||||
file: File,
|
||||
pipeline: WritebackPipeline,
|
||||
pos: u64,
|
||||
}
|
||||
|
||||
impl WritebackFile {
|
||||
/// Wrap an open `File`. The current OS file position is queried
|
||||
/// once so the pipeline starts tracking from wherever the file
|
||||
/// already is (typically 0 for fresh files; non-zero for resumed
|
||||
/// or appended files).
|
||||
pub(crate) fn new(mut file: File) -> io::Result<Self> {
|
||||
let pos = file.stream_position()?;
|
||||
let pipeline = WritebackPipeline::new(&file, pos, writeback_chunk_bytes());
|
||||
Ok(Self {
|
||||
file,
|
||||
pipeline,
|
||||
pos,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new file at `path` (truncating any existing contents)
|
||||
/// and wrap it. Convenience for the common
|
||||
/// `File::create(path)` + `WritebackFile::new(file)` pair so callers
|
||||
/// don't have to assemble a `File` first.
|
||||
///
|
||||
/// Callers that know the target output size should prefer
|
||||
/// [`Self::create_with_size_hint`] so the kernel can pre-reserve
|
||||
/// extents.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = File::create(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Like [`Self::create`] but pre-reserves `size_bytes` of disk
|
||||
/// space via the platform's extent-preallocation primitive (Linux
|
||||
/// `fallocate(KEEP_SIZE)`, macOS `F_PREALLOCATE`, Windows
|
||||
/// `SetFileValidData` stub). The reported file size is unchanged
|
||||
/// (writes still grow the file naturally) — only the on-disk extent
|
||||
/// allocation is preallocated, which reduces extent fragmentation
|
||||
/// on large sequential writes (mux output, especially on slow
|
||||
/// storage / NFS).
|
||||
///
|
||||
/// On platforms without an extent-preallocation primitive this is
|
||||
/// equivalent to `create` — the size hint is dropped after a debug
|
||||
/// log.
|
||||
pub(crate) fn create_with_size_hint(path: &Path, size_bytes: u64) -> io::Result<Self> {
|
||||
let file = File::create(path)?;
|
||||
platform::preallocate(&file, size_bytes);
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Open an existing file at `path` for writing (no truncation) and
|
||||
/// wrap it. Mirrors `File::open` semantics for the writable case
|
||||
/// — used by patch / resume paths that mutate an existing ISO in
|
||||
/// place.
|
||||
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new().write(true).open(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Drain in-flight writeback then issue a full fsync. Use this in
|
||||
/// place of `File::sync_all`.
|
||||
///
|
||||
/// The final durable flush is wrapped in
|
||||
/// [`crate::io::bounded::bounded_syscall`] (per the per-OS module)
|
||||
/// with a 60 s deadline on Linux/macOS — a wedged NFS server cannot
|
||||
/// trap the calling thread indefinitely. On timeout the page cache
|
||||
/// is left to the kernel's normal flush-on-close path — best
|
||||
/// effort, but bounded.
|
||||
pub(crate) fn sync_all(&mut self) -> io::Result<()> {
|
||||
self.pipeline.finalize();
|
||||
platform::durable_sync(&self.file)
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for WritebackFile {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let n = self.file.write(buf)?;
|
||||
self.pos += n as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.file.write_all(buf)?;
|
||||
self.pos += buf.len() as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for WritebackFile {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
let p = self.file.seek(from)?;
|
||||
// Only treat seeks that actually move the position as
|
||||
// boundaries — sweep does a redundant `seek(Current(pos))`
|
||||
// before every write, and we don't want that to drain the
|
||||
// pipeline on every iteration.
|
||||
if p != self.pos {
|
||||
// Diagnostic for the NFS mux hang: the MKV format requires
|
||||
// the muxer to seek back occasionally (cluster size
|
||||
// patching, Cues index write, Segment header backpatch).
|
||||
// Each such seek invalidates the writeback chunk tracking
|
||||
// and forces a finalize → WAIT_AFTER on the in-flight
|
||||
// chunk. Logging the seek delta lets us correlate hang
|
||||
// offsets with specific muxer operations.
|
||||
let from_pos = self.pos;
|
||||
let to_pos = p;
|
||||
let delta: i64 = (to_pos as i64).wrapping_sub(from_pos as i64);
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile seek from={from_pos} to={to_pos} delta={delta}"
|
||||
);
|
||||
self.pipeline.handle_seek(p);
|
||||
self.pos = p;
|
||||
}
|
||||
Ok(p)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WritebackFile {
|
||||
fn drop(&mut self) {
|
||||
// Run the pipeline's tail finalize so the last in-flight chunk
|
||||
// gets its `WAIT_AFTER` + `posix_fadvise(DONTNEED)`. Without
|
||||
// this, callers that drop a `WritebackFile` without calling
|
||||
// `sync_all` (panic, early-return, idiomatic `let _ = w;`)
|
||||
// leave the trailing chunk in cache; the kernel still flushes
|
||||
// on close, but the bounded-cache invariant fails at the tail.
|
||||
// We deliberately do *not* call `self.file.sync_all()` here —
|
||||
// close already triggers a flush, and an `fsync` from `Drop`
|
||||
// would silently swallow its `io::Error` anyway. `finalize` is
|
||||
// idempotent so an explicit `sync_all` followed by drop is
|
||||
// still safe.
|
||||
self.pipeline.finalize();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
|
||||
fn read_back(path: &Path) -> Vec<u8> {
|
||||
let mut f = File::open(path).unwrap();
|
||||
let mut v = Vec::new();
|
||||
f.read_to_end(&mut v).unwrap();
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_then_drop_persists_bytes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("a.bin");
|
||||
{
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"hello world").unwrap();
|
||||
// Drop drains the pipeline tail.
|
||||
}
|
||||
assert_eq!(read_back(&p), b"hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_all_drains_and_flushes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("b.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
for _ in 0..32 {
|
||||
w.write_all(&[0x5au8; 1024]).unwrap();
|
||||
}
|
||||
// After sync_all, the bytes MUST be visible to a separate
|
||||
// reader. The pipeline has been finalised and durable-sync has
|
||||
// run.
|
||||
w.sync_all().unwrap();
|
||||
let bytes = read_back(&p);
|
||||
assert_eq!(bytes.len(), 32 * 1024);
|
||||
assert!(bytes.iter().all(|&b| b == 0x5a));
|
||||
drop(w);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seek_then_patch_roundtrip() {
|
||||
// Write A; seek back; patch with B; read back; the patch lands
|
||||
// at the right offset.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("c.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
let big = vec![b'A'; 4096];
|
||||
w.write_all(&big).unwrap();
|
||||
// Seek back to offset 1000 and overwrite 8 bytes.
|
||||
w.seek(SeekFrom::Start(1000)).unwrap();
|
||||
w.write_all(b"PATCHED!").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
let bytes = read_back(&p);
|
||||
assert_eq!(bytes.len(), 4096);
|
||||
assert_eq!(&bytes[1000..1008], b"PATCHED!");
|
||||
// Bytes outside the patch are still 'A'.
|
||||
assert_eq!(bytes[999], b'A');
|
||||
assert_eq!(bytes[1008], b'A');
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flush_is_observed_in_order() {
|
||||
// `Write::flush` should not panic or reorder; verify the bytes
|
||||
// land in order through interleaved flushes.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("f.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"one").unwrap();
|
||||
w.flush().unwrap();
|
||||
w.write_all(b"two").unwrap();
|
||||
w.flush().unwrap();
|
||||
w.write_all(b"three").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
assert_eq!(read_back(&p), b"onetwothree");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//! Fallback platform impl for [`super::WritebackFile`] on targets
|
||||
//! without a dedicated implementation (BSDs, illumos, etc.).
|
||||
//!
|
||||
//! - `preallocate` is a logged no-op.
|
||||
//! - `durable_sync` calls `File::sync_all` directly (no bounded-syscall
|
||||
//! wrapper — the wrapper depends on Linux/macOS unix idioms that
|
||||
//! aren't universally portable). If a future BSD impl needs the
|
||||
//! 60-s deadline, it should land in its own per-OS file rather than
|
||||
//! bloat this fallback.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
|
||||
pub(super) fn preallocate(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile preallocate size_hint={size_bytes} skipped (no impl on this target)"
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
file.sync_all()
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//! Windows platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! TODO: this stub matches the design's "validate without a Windows
|
||||
//! build env, leave a stub" carve-out. The real impl should use:
|
||||
//!
|
||||
//! - `SetEndOfFile` + `SetFileValidData` for extent preallocation
|
||||
//! (caller needs `SE_MANAGE_VOLUME_NAME` privilege; if unavailable
|
||||
//! fall back to a write-zero path or just skip).
|
||||
//! - `FlushFileBuffers` for fsync-equivalent durable flush.
|
||||
//!
|
||||
//! Until then: preallocate is a debug-logged no-op; durable_sync calls
|
||||
//! the std `File::sync_all` (which on Windows maps to
|
||||
//! `FlushFileBuffers` internally).
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
|
||||
pub(super) fn preallocate(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile preallocate size_hint={size_bytes} skipped (windows stub; TODO: SetFileValidData)"
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
// `File::sync_all` on Windows is `FlushFileBuffers`. Acceptable
|
||||
// for now; the bounded-syscall wrapper is not used here because
|
||||
// the stub also skips the worker-thread + leak machinery (the
|
||||
// wrapper would need an `unsafe impl Send` for `RawHandle`, and
|
||||
// designing that without a Windows test env is asking for it).
|
||||
file.sync_all()
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
//! BDMV disc-library metadata (`/BDMV/META/DL/bdmt_<lang>.xml`).
|
||||
//!
|
||||
//! Every commercial Blu-ray carries a disc-library metadata directory
|
||||
//! with one XML file per shipped language. The schema is the Blu-ray
|
||||
//! "disc library metadata" namespace (`urn:BDA:bdmv;disclibmeta`),
|
||||
//! conventionally prefixed `di:`. Fields commonly present:
|
||||
//!
|
||||
//! - `<di:title>` or `<di:name>` — the title string. Vendor practice
|
||||
//! varies (Paramount discs tend to use `<di:name>`).
|
||||
//! - `<di:description>` — optional synopsis (often absent on retail
|
||||
//! discs; common on box sets and special editions).
|
||||
//! - `<di:discNumber>` / `<di:numSets>` (or `<di:numberOfSets>`) —
|
||||
//! set position for multi-disc releases.
|
||||
//!
|
||||
//! This module is intentionally separate from the BD-J `StreamLabel`
|
||||
//! parsers under `labels/*.rs`. The XML here is disc-level (title,
|
||||
//! description, set position), not per-stream — wiring into the main
|
||||
//! parser registry happens elsewhere.
|
||||
//!
|
||||
//! Real-world XML is irregular: missing description elements, multiple
|
||||
//! title elements (first one wins), and occasional malformed content.
|
||||
//! Extraction is best-effort — a malformed file is treated as "no
|
||||
//! metadata" (returns `None` from the helper), and the caller can
|
||||
//! still get metadata from sibling-language XML files.
|
||||
|
||||
// The module wiring (registry hook + public re-export) is added
|
||||
// separately. Until then the parse/detect entry points have no
|
||||
use super::xml;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Disc-level metadata extracted from `/BDMV/META/DL/bdmt_*.xml`.
|
||||
///
|
||||
/// All maps are keyed by 3-char ISO 639-2 language code (e.g.
|
||||
/// `"eng"`, `"fra"`, `"jpn"`) — the same key segment used in the
|
||||
/// `bdmt_<lang>.xml` filename.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)]
|
||||
pub struct DiscMetadata {
|
||||
/// Localized titles, keyed by 3-char ISO 639-2 lang code
|
||||
/// (e.g. "eng" → "Dune Part Two")
|
||||
pub titles: BTreeMap<String, String>,
|
||||
/// First-line / short description, per lang
|
||||
pub descriptions: BTreeMap<String, String>,
|
||||
/// Disc N of M for box sets (None if not a box set)
|
||||
pub disc_number: Option<(u32, u32)>,
|
||||
}
|
||||
|
||||
/// True if `/BDMV/META/DL/` exists and contains at least one
|
||||
/// `bdmt_*.xml` file.
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
let Some(dir) = udf.find_dir("/BDMV/META/DL") else {
|
||||
return false;
|
||||
};
|
||||
dir.entries
|
||||
.iter()
|
||||
.any(|e| !e.is_dir && is_bdmt_filename(&e.name))
|
||||
}
|
||||
|
||||
/// Read every `bdmt_<lang>.xml` under `/BDMV/META/DL/` and return the
|
||||
/// aggregated [`DiscMetadata`]. Returns `None` if no titles could be
|
||||
/// extracted from any file.
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<DiscMetadata> {
|
||||
let dir = udf.find_dir("/BDMV/META/DL")?;
|
||||
let mut out = DiscMetadata::default();
|
||||
|
||||
for entry in &dir.entries {
|
||||
if entry.is_dir {
|
||||
continue;
|
||||
}
|
||||
let Some(lang) = lang_code_from_filename(&entry.name) else {
|
||||
continue;
|
||||
};
|
||||
let path = format!("/BDMV/META/DL/{}", entry.name);
|
||||
let Ok(bytes) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(text) = std::str::from_utf8(&bytes) else {
|
||||
continue;
|
||||
};
|
||||
let Some((title, description, disc_set)) = parse_bdmt_xml(&lang, text) else {
|
||||
continue;
|
||||
};
|
||||
out.titles.insert(lang.clone(), title);
|
||||
if let Some(desc) = description {
|
||||
out.descriptions.insert(lang.clone(), desc);
|
||||
}
|
||||
// Disc-set position is disc-global; first one we successfully
|
||||
// read wins. (All bdmt_*.xml on a given disc carry the same
|
||||
// value in practice.)
|
||||
if out.disc_number.is_none() {
|
||||
if let Some(ds) = disc_set {
|
||||
out.disc_number = Some(ds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if out.titles.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// True if `name` matches the `bdmt_<lang>.xml` convention with a
|
||||
/// 3-character ISO 639-2 lang code segment. Case-insensitive.
|
||||
fn is_bdmt_filename(name: &str) -> bool {
|
||||
lang_code_from_filename(name).is_some()
|
||||
}
|
||||
|
||||
/// Extract the 3-char language code from a `bdmt_<lang>.xml` filename.
|
||||
/// Returns `None` if the filename doesn't match. Lang code is
|
||||
/// lowercased so callers always see e.g. `"eng"` not `"ENG"`.
|
||||
fn lang_code_from_filename(name: &str) -> Option<String> {
|
||||
let lower = name.to_ascii_lowercase();
|
||||
let stem = lower.strip_suffix(".xml")?;
|
||||
let lang = stem.strip_prefix("bdmt_")?;
|
||||
// ISO 639-2 codes are exactly 3 ASCII letters. Be strict — keeps
|
||||
// us from picking up unrelated `bdmt_foo.xml` siblings.
|
||||
if lang.len() != 3 || !lang.chars().all(|c| c.is_ascii_alphabetic()) {
|
||||
return None;
|
||||
}
|
||||
Some(lang.to_string())
|
||||
}
|
||||
|
||||
/// Tuple returned by [`parse_bdmt_xml`]: `(title, description?, disc_set?)`.
|
||||
/// Aliased so the function signature isn't a clippy::type-complexity offender.
|
||||
pub(crate) type BdmtFields = (String, Option<String>, Option<(u32, u32)>);
|
||||
|
||||
/// Parse one `bdmt_<lang>.xml` document and return
|
||||
/// `(title, description?, disc_set?)`. Returns `None` if no title
|
||||
/// could be located — the caller treats this as "skip this file".
|
||||
///
|
||||
/// Title-element preference: `<di:name>` → `<di:title>` →
|
||||
/// `<di:tableOfContents>/<di:titleName>` (first match wins, per the
|
||||
/// authoring-tool conventions documented at the module level).
|
||||
pub(crate) fn parse_bdmt_xml(_lang_code: &str, xml_text: &str) -> Option<BdmtFields> {
|
||||
let title = extract_title(xml_text)?;
|
||||
let description = xml::text(xml_text, "description")
|
||||
.filter(|s| !s.is_empty())
|
||||
.filter(|s| !looks_like_xml(s))
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty());
|
||||
let disc_set = extract_disc_set(xml_text);
|
||||
Some((title, description, disc_set))
|
||||
}
|
||||
|
||||
/// Reject candidate description strings that are themselves XML
|
||||
/// fragments — observed on disc-04 (Top Gun: Maverick), where
|
||||
/// `<di:description>` contained `<di:thumbnail href="…"/>` child
|
||||
/// elements and no actual prose. Surfacing that raw to the JSON
|
||||
/// output is worse than dropping the field entirely.
|
||||
fn looks_like_xml(s: &str) -> bool {
|
||||
let t = s.trim_start();
|
||||
t.starts_with('<')
|
||||
}
|
||||
|
||||
/// Try title-bearing element variants in priority order. The `xml`
|
||||
/// helpers are case- and namespace-insensitive, so callers pass the
|
||||
/// bare local name (no `di:` prefix).
|
||||
fn extract_title(xml_text: &str) -> Option<String> {
|
||||
// Order matches the module-level convention: <di:name> first
|
||||
// (Paramount-style), then <di:title>, then the nested
|
||||
// tableOfContents/titleName form.
|
||||
for tag in ["name", "title"] {
|
||||
if let Some(s) = xml::text(xml_text, tag) {
|
||||
let trimmed = s.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
// tableOfContents/titleName: search inside the toc block so we
|
||||
// don't accidentally pick a stray <titleName> from elsewhere.
|
||||
if let Some((s, e)) = xml::find_element(xml_text, "tableOfContents", 0) {
|
||||
let block = &xml_text[s..e];
|
||||
if let Some(t) = xml::text(block, "titleName") {
|
||||
let trimmed = t.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Extract `(discNumber, numSets)` if both are present and parse as
|
||||
/// `u32`. Accepts either `<di:numSets>` or `<di:numberOfSets>` for
|
||||
/// the denominator (both forms appear in the wild).
|
||||
fn extract_disc_set(xml_text: &str) -> Option<(u32, u32)> {
|
||||
let n = xml::text(xml_text, "discNumber")?
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.ok()?;
|
||||
let total = xml::text(xml_text, "numSets")
|
||||
.or_else(|| xml::text(xml_text, "numberOfSets"))?
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.ok()?;
|
||||
Some((n, total))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn extract_simple_title() {
|
||||
// Minimal Paramount-style document: <di:name> as the title
|
||||
// carrier inside a <discInfo> root.
|
||||
let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Dune Part Two</di:name>
|
||||
</discInfo>"#;
|
||||
let (title, desc, set) = parse_bdmt_xml("eng", xml).expect("title should parse");
|
||||
assert_eq!(title, "Dune Part Two");
|
||||
assert_eq!(desc, None);
|
||||
assert_eq!(set, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_title_element_variant() {
|
||||
// <di:title> is the alternate carrier; should be picked up
|
||||
// when <di:name> is absent.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:title>The Matrix</di:title>
|
||||
<di:description>A film about computers.</di:description>
|
||||
</discInfo>"#;
|
||||
let (title, desc, _) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(title, "The Matrix");
|
||||
assert_eq!(desc.as_deref(), Some("A film about computers."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_title_from_table_of_contents_fallback() {
|
||||
// Some authoring tools nest the title under tableOfContents.
|
||||
// No <di:name> or <di:title> at top level → fall back to
|
||||
// titleName inside tableOfContents.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:tableOfContents>
|
||||
<di:titleName>Inside Out 2</di:titleName>
|
||||
</di:tableOfContents>
|
||||
</discInfo>"#;
|
||||
let (title, _, _) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(title, "Inside Out 2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_box_set_position() {
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>LOTR Disc 2</di:name>
|
||||
<di:discNumber>2</di:discNumber>
|
||||
<di:numSets>5</di:numSets>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(set, Some((2, 5)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_box_set_position_alternate_total_tag() {
|
||||
// <di:numberOfSets> is an alternate spelling we've seen.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>X</di:name>
|
||||
<di:discNumber>3</di:discNumber>
|
||||
<di:numberOfSets>6</di:numberOfSets>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(set, Some((3, 6)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_box_set_requires_both_fields() {
|
||||
// discNumber alone (no total) yields None — we don't fabricate
|
||||
// a denominator.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>X</di:name>
|
||||
<di:discNumber>1</di:discNumber>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(set, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_languages_keyed_correctly() {
|
||||
// Simulate driving parse_bdmt_xml from two synthetic XML
|
||||
// blobs and aggregating into DiscMetadata the same way parse()
|
||||
// would. This exercises the BTreeMap key handling without
|
||||
// needing a UdfFs.
|
||||
let eng_xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Dune Part Two</di:name>
|
||||
</discInfo>"#;
|
||||
let fra_xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Dune Deuxième Partie</di:name>
|
||||
<di:description>Suite du film de 2021.</di:description>
|
||||
</discInfo>"#;
|
||||
|
||||
let mut meta = DiscMetadata::default();
|
||||
for (lang, blob) in [("eng", eng_xml), ("fra", fra_xml)] {
|
||||
let (title, desc, ds) = parse_bdmt_xml(lang, blob).unwrap();
|
||||
meta.titles.insert(lang.to_string(), title);
|
||||
if let Some(d) = desc {
|
||||
meta.descriptions.insert(lang.to_string(), d);
|
||||
}
|
||||
if meta.disc_number.is_none() {
|
||||
if let Some(d) = ds {
|
||||
meta.disc_number = Some(d);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
meta.titles.get("eng").map(String::as_str),
|
||||
Some("Dune Part Two")
|
||||
);
|
||||
assert_eq!(
|
||||
meta.titles.get("fra").map(String::as_str),
|
||||
Some("Dune Deuxième Partie")
|
||||
);
|
||||
assert!(meta.descriptions.get("eng").is_none());
|
||||
assert_eq!(
|
||||
meta.descriptions.get("fra").map(String::as_str),
|
||||
Some("Suite du film de 2021.")
|
||||
);
|
||||
assert_eq!(meta.disc_number, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_xml_returns_none() {
|
||||
// Random gibberish has no recognizable title element. We
|
||||
// document the contract: parse_bdmt_xml returns None, and
|
||||
// parse() (the caller) skips the file. Aggregating across
|
||||
// zero files leaves DiscMetadata::default() — which parse()
|
||||
// surfaces as None to its caller. Either is documented as
|
||||
// acceptable per the module spec.
|
||||
let bad = "this is not xml &&& <<< nope";
|
||||
assert!(parse_bdmt_xml("eng", bad).is_none());
|
||||
|
||||
// Half-open tag, no body, no close: also yields no title.
|
||||
let truncated = "<discInfo><di:name>";
|
||||
assert!(parse_bdmt_xml("eng", truncated).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn description_with_only_child_xml_is_dropped() {
|
||||
// Real-world bug from disc-04 (Top Gun: Maverick, 2026-05-11
|
||||
// capture): <di:description> contained only <di:thumbnail/>
|
||||
// child elements with no actual prose. The previous parser
|
||||
// surfaced the raw XML fragment as the description string.
|
||||
// Now we reject candidates that begin with `<`.
|
||||
let xml = r#"<discInfo>
|
||||
<di:name>Top Gun: Maverick</di:name>
|
||||
<di:description>
|
||||
<di:thumbnail href="tgm_meta_sm.jpg" />
|
||||
<di:thumbnail href="tgm_meta_lg.jpg" />
|
||||
</di:description>
|
||||
</discInfo>"#;
|
||||
let (title, description, _) =
|
||||
parse_bdmt_xml("eng", xml).expect("title is present so parse must succeed");
|
||||
assert_eq!(title, "Top Gun: Maverick");
|
||||
assert!(
|
||||
description.is_none(),
|
||||
"description containing only XML children must be dropped, got {description:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn description_with_plain_text_passes_through() {
|
||||
// The legitimate case still works: a description with actual
|
||||
// prose survives the looks_like_xml filter.
|
||||
let xml = r#"<discInfo>
|
||||
<di:name>Some Movie</di:name>
|
||||
<di:description>An epic tale of one man's quest for tea.</di:description>
|
||||
</discInfo>"#;
|
||||
let (_, description, _) = parse_bdmt_xml("eng", xml).expect("must parse");
|
||||
assert_eq!(
|
||||
description.as_deref(),
|
||||
Some("An epic tale of one man's quest for tea.")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whitespace_in_title_is_trimmed() {
|
||||
let xml = r#"<discInfo><di:name>
|
||||
Dune Part Two
|
||||
</di:name></discInfo>"#;
|
||||
let (title, _, _) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(title, "Dune Part Two");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_code_extraction() {
|
||||
assert_eq!(lang_code_from_filename("bdmt_eng.xml"), Some("eng".into()));
|
||||
assert_eq!(lang_code_from_filename("BDMT_FRA.XML"), Some("fra".into()));
|
||||
assert_eq!(lang_code_from_filename("bdmt_jpn.xml"), Some("jpn".into()));
|
||||
// Non-matching cases:
|
||||
assert_eq!(lang_code_from_filename("bdmt_.xml"), None);
|
||||
assert_eq!(lang_code_from_filename("bdmt_engl.xml"), None);
|
||||
assert_eq!(lang_code_from_filename("bdmt_e1g.xml"), None);
|
||||
assert_eq!(lang_code_from_filename("bdmt_eng.txt"), None);
|
||||
assert_eq!(lang_code_from_filename("foo.xml"), None);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,291 @@
|
||||
//! CLPI vs MPLS cross-validation diagnostic.
|
||||
//!
|
||||
//! Empirical question (raised 2026-05-11): is CLPI's per-stream
|
||||
//! language and codec data truly redundant with MPLS's STN-table data
|
||||
//! on real-world Blu-rays?
|
||||
//!
|
||||
//! Build a quick audit that walks both sources, normalizes their stream
|
||||
//! lists by (PID, language, coding_type), and flags any disagreement.
|
||||
//!
|
||||
//! Three classes of mismatch we want to detect:
|
||||
//!
|
||||
//! 1. **CLPI has streams MPLS doesn't reference.** Orphan streams in
|
||||
//! the .m2ts that no playlist's STN table includes. Means the user
|
||||
//! can't reach them through the menu but they're physically on the
|
||||
//! disc.
|
||||
//! 2. **MPLS has streams CLPI doesn't list.** Should never happen if
|
||||
//! both parsers are correct — playlists reference clips which
|
||||
//! reference streams. If it happens, one of our parsers has a bug.
|
||||
//! 3. **Same PID, different language / coding_type.** The playlist re-
|
||||
//! tagged a stream's metadata. Rare but spec-permitted. Means CLPI
|
||||
//! and MPLS disagree about the same physical stream's properties.
|
||||
//!
|
||||
//! If audits across the corpus show zero mismatches of any class, CLPI
|
||||
//! program_info extraction is **empirically redundant** for labels and
|
||||
//! we can leave it out of the registry. If even one mismatch surfaces,
|
||||
//! we add a CLPI label parser to the registry as belt-and-suspenders.
|
||||
//!
|
||||
//! This module exposes `audit(reader, udf)` returning a structured
|
||||
//! report. Surfaced via the labels-analyze tool — not part of the
|
||||
//! `analyze()` pipeline (no impact on the label output).
|
||||
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// One row in the audit: a stream PID that's known to one source or
|
||||
/// both, with the fields each source reported.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ClpiVsMplsRow {
|
||||
pub pid: u16,
|
||||
pub clpi_coding_type: Option<u8>,
|
||||
pub clpi_language: Option<String>,
|
||||
pub mpls_coding_type: Option<u8>,
|
||||
pub mpls_language: Option<String>,
|
||||
}
|
||||
|
||||
impl ClpiVsMplsRow {
|
||||
/// Three rules for classification:
|
||||
/// - both sources missing (impossible — caller wouldn't insert)
|
||||
/// - one source missing → class A or B (orphan-on-disc / playlist-only)
|
||||
/// - both present but fields differ → class C (metadata divergence)
|
||||
/// - both present and identical → no mismatch
|
||||
pub fn class(&self) -> ClpiVsMplsClass {
|
||||
match (
|
||||
self.clpi_coding_type.is_some(),
|
||||
self.mpls_coding_type.is_some(),
|
||||
) {
|
||||
(true, false) => ClpiVsMplsClass::ClpiOnly,
|
||||
(false, true) => ClpiVsMplsClass::MplsOnly,
|
||||
(true, true) => {
|
||||
let coding_match = self.clpi_coding_type == self.mpls_coding_type;
|
||||
let lang_match = self.clpi_language == self.mpls_language;
|
||||
if coding_match && lang_match {
|
||||
ClpiVsMplsClass::Match
|
||||
} else {
|
||||
ClpiVsMplsClass::Divergent
|
||||
}
|
||||
}
|
||||
(false, false) => ClpiVsMplsClass::Match,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Classification of one (PID) row.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ClpiVsMplsClass {
|
||||
/// PID seen in CLPI ProgramInfo but no MPLS STN table references
|
||||
/// it. Orphan on disc.
|
||||
ClpiOnly,
|
||||
/// PID seen in MPLS STN table but no CLPI ProgramInfo includes it.
|
||||
/// One of our parsers probably has a bug.
|
||||
MplsOnly,
|
||||
/// Both sources see this PID with the same coding_type + language.
|
||||
Match,
|
||||
/// Both sources see this PID but disagree on coding_type or language.
|
||||
/// MPLS wins for label rendering (playlist-authoritative view); CLPI
|
||||
/// is the per-clip ground truth.
|
||||
Divergent,
|
||||
}
|
||||
|
||||
/// Full audit report.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ClpiVsMplsAudit {
|
||||
pub rows: Vec<ClpiVsMplsRow>,
|
||||
}
|
||||
|
||||
impl ClpiVsMplsAudit {
|
||||
pub fn class_counts(&self) -> (usize, usize, usize, usize) {
|
||||
let mut clpi_only = 0;
|
||||
let mut mpls_only = 0;
|
||||
let mut matches = 0;
|
||||
let mut divergent = 0;
|
||||
for r in &self.rows {
|
||||
match r.class() {
|
||||
ClpiVsMplsClass::ClpiOnly => clpi_only += 1,
|
||||
ClpiVsMplsClass::MplsOnly => mpls_only += 1,
|
||||
ClpiVsMplsClass::Match => matches += 1,
|
||||
ClpiVsMplsClass::Divergent => divergent += 1,
|
||||
}
|
||||
}
|
||||
(clpi_only, mpls_only, matches, divergent)
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk `/BDMV/CLIPINF/*.clpi` and `/BDMV/PLAYLIST/*.mpls`, build a
|
||||
/// dedup-by-PID table of (CLPI fields, MPLS fields), return the
|
||||
/// merged view. Missing files (read errors, parse failures) are
|
||||
/// silently skipped — this is diagnostic, not correctness-critical.
|
||||
pub fn audit(reader: &mut dyn SectorSource, udf: &UdfFs) -> ClpiVsMplsAudit {
|
||||
// Aggregate by PID across all CLPI files. If a PID appears in
|
||||
// multiple clips (typical — main movie clip + trailers reference
|
||||
// the same audio stream PIDs), first encountered wins (they should
|
||||
// all agree per BD spec).
|
||||
let mut clpi_by_pid: BTreeMap<u16, (u8, String)> = BTreeMap::new();
|
||||
if let Some(dir) = udf.find_dir("/BDMV/CLIPINF") {
|
||||
let names: Vec<String> = dir
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && e.name.to_ascii_lowercase().ends_with(".clpi"))
|
||||
.map(|e| e.name.clone())
|
||||
.collect();
|
||||
for name in names {
|
||||
let path = format!("/BDMV/CLIPINF/{}", name);
|
||||
let Ok(data) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(clip) = crate::clpi::parse(&data) else {
|
||||
continue;
|
||||
};
|
||||
for s in clip.streams {
|
||||
clpi_by_pid
|
||||
.entry(s.pid)
|
||||
.or_insert((s.coding_type, s.language));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Same for MPLS streams.
|
||||
let mut mpls_by_pid: BTreeMap<u16, (u8, String)> = BTreeMap::new();
|
||||
if let Some(dir) = udf.find_dir("/BDMV/PLAYLIST") {
|
||||
let names: Vec<String> = dir
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && e.name.to_ascii_lowercase().ends_with(".mpls"))
|
||||
.map(|e| e.name.clone())
|
||||
.collect();
|
||||
for name in names {
|
||||
let path = format!("/BDMV/PLAYLIST/{}", name);
|
||||
let Ok(data) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(pl) = crate::mpls::parse(&data) else {
|
||||
continue;
|
||||
};
|
||||
for s in pl.streams {
|
||||
if s.pid == 0 {
|
||||
// PID 0 means "no PID in stream entry" — skip rather
|
||||
// than collide with other entries.
|
||||
continue;
|
||||
}
|
||||
mpls_by_pid
|
||||
.entry(s.pid)
|
||||
.or_insert((s.coding_type, s.language));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Merge views: every PID seen anywhere gets a row.
|
||||
let mut all_pids: std::collections::BTreeSet<u16> = std::collections::BTreeSet::new();
|
||||
all_pids.extend(clpi_by_pid.keys().copied());
|
||||
all_pids.extend(mpls_by_pid.keys().copied());
|
||||
|
||||
let mut rows = Vec::with_capacity(all_pids.len());
|
||||
for pid in all_pids {
|
||||
let clpi = clpi_by_pid.get(&pid);
|
||||
let mpls = mpls_by_pid.get(&pid);
|
||||
rows.push(ClpiVsMplsRow {
|
||||
pid,
|
||||
clpi_coding_type: clpi.map(|(c, _)| *c),
|
||||
clpi_language: clpi.map(|(_, l)| l.clone()),
|
||||
mpls_coding_type: mpls.map(|(c, _)| *c),
|
||||
mpls_language: mpls.map(|(_, l)| l.clone()),
|
||||
});
|
||||
}
|
||||
|
||||
ClpiVsMplsAudit { rows }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn class_match_when_identical() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("eng".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::Match);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_clpi_only_when_mpls_missing() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: None,
|
||||
mpls_language: None,
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::ClpiOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_mpls_only_when_clpi_missing() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: None,
|
||||
clpi_language: None,
|
||||
mpls_coding_type: Some(0x90),
|
||||
mpls_language: Some("fra".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::MplsOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_divergent_on_lang_disagreement() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("und".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::Divergent);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_counts_sum_rows() {
|
||||
let audit = ClpiVsMplsAudit {
|
||||
rows: vec![
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("eng".into()),
|
||||
}, // Match
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1101,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("fra".into()),
|
||||
mpls_coding_type: None,
|
||||
mpls_language: None,
|
||||
}, // ClpiOnly
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1102,
|
||||
clpi_coding_type: None,
|
||||
clpi_language: None,
|
||||
mpls_coding_type: Some(0x90),
|
||||
mpls_language: Some("eng".into()),
|
||||
}, // MplsOnly
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1103,
|
||||
clpi_coding_type: Some(0x86),
|
||||
clpi_language: Some("spa".into()),
|
||||
mpls_coding_type: Some(0x86),
|
||||
mpls_language: Some("ita".into()),
|
||||
}, // Divergent
|
||||
],
|
||||
};
|
||||
let (co, mo, m, d) = audit.class_counts();
|
||||
assert_eq!(co, 1);
|
||||
assert_eq!(mo, 1);
|
||||
assert_eq!(m, 1);
|
||||
assert_eq!(d, 1);
|
||||
}
|
||||
}
|
||||
+56
-92
@@ -3,8 +3,8 @@
|
||||
//! Clean structured XML with Content/Qualifier per stream and
|
||||
//! stream number mapping via playbackconfig.
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, xml};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -12,7 +12,7 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "streamproperties.xml")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let sp_data = super::read_jar_file(reader, udf, "streamproperties.xml")?;
|
||||
let sp_text = std::str::from_utf8(&sp_data).ok()?;
|
||||
|
||||
@@ -66,7 +66,8 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(labels)
|
||||
// High confidence: streamproperties.xml is fully structured.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
|
||||
struct StreamInfo {
|
||||
@@ -78,105 +79,68 @@ struct StreamInfo {
|
||||
qualifier: LabelQualifier,
|
||||
}
|
||||
|
||||
fn parse_stream_infos(xml: &str) -> Vec<StreamInfo> {
|
||||
fn parse_stream_infos(text: &str) -> Vec<StreamInfo> {
|
||||
let mut infos = Vec::new();
|
||||
let mut pos = 0;
|
||||
|
||||
while pos < xml.len() {
|
||||
let (tag, stream_type) = if let Some(p) = xml[pos..].find("<AudioStreamInfos>") {
|
||||
(p + pos, StreamLabelType::Audio)
|
||||
} else if let Some(p) = xml[pos..].find("<SubtitleStreamInfos>") {
|
||||
(p + pos, StreamLabelType::Subtitle)
|
||||
} else {
|
||||
break;
|
||||
};
|
||||
for (tag_name, stream_type) in [
|
||||
("AudioStreamInfos", StreamLabelType::Audio),
|
||||
("SubtitleStreamInfos", StreamLabelType::Subtitle),
|
||||
] {
|
||||
let mut from = 0;
|
||||
while let Some((start, end)) = xml::find_element(text, tag_name, from) {
|
||||
let block = &text[start..end];
|
||||
let id = xml::text(block, "ID").unwrap_or_default();
|
||||
let lang_id = xml::text(block, "LangInfoID").unwrap_or_default();
|
||||
let content = xml::text(block, "Content").unwrap_or_default();
|
||||
let qualifier_str = xml::text(block, "Qualifier").unwrap_or_default();
|
||||
|
||||
let end_tag = match stream_type {
|
||||
StreamLabelType::Audio => "</AudioStreamInfos>",
|
||||
StreamLabelType::Subtitle => "</SubtitleStreamInfos>",
|
||||
};
|
||||
let (language, variant) = if lang_id.contains('_') {
|
||||
let parts: Vec<&str> = lang_id.splitn(2, '_').collect();
|
||||
(parts[0].to_lowercase(), parts[1].to_string())
|
||||
} else {
|
||||
(lang_id.to_lowercase(), String::new())
|
||||
};
|
||||
|
||||
let block_end = match xml[tag..].find(end_tag) {
|
||||
Some(p) => tag + p + end_tag.len(),
|
||||
None => break,
|
||||
};
|
||||
let purpose = if content.eq_ignore_ascii_case("COMMENTARY") {
|
||||
LabelPurpose::Commentary
|
||||
} else {
|
||||
LabelPurpose::Normal
|
||||
};
|
||||
|
||||
let block = &xml[tag..block_end];
|
||||
let id = extract_tag(block, "ID").unwrap_or_default();
|
||||
let lang_id = extract_tag(block, "LangInfoID").unwrap_or_default();
|
||||
let content = extract_tag(block, "Content").unwrap_or_default();
|
||||
let qualifier_str = extract_tag(block, "Qualifier").unwrap_or_default();
|
||||
let qualifier = match qualifier_str.to_ascii_uppercase().as_str() {
|
||||
"SDH" => LabelQualifier::Sdh,
|
||||
"DS" => LabelQualifier::DescriptiveService,
|
||||
_ => LabelQualifier::None,
|
||||
};
|
||||
|
||||
let (language, variant) = if lang_id.contains('_') {
|
||||
let parts: Vec<&str> = lang_id.splitn(2, '_').collect();
|
||||
(parts[0].to_lowercase(), parts[1].to_string())
|
||||
} else {
|
||||
(lang_id.to_lowercase(), String::new())
|
||||
};
|
||||
|
||||
let purpose = match content.as_str() {
|
||||
"COMMENTARY" => LabelPurpose::Commentary,
|
||||
_ => LabelPurpose::Normal,
|
||||
};
|
||||
|
||||
let qualifier = match qualifier_str.as_str() {
|
||||
"SDH" => LabelQualifier::Sdh,
|
||||
"DS" => LabelQualifier::DescriptiveService,
|
||||
_ => LabelQualifier::None,
|
||||
};
|
||||
|
||||
infos.push(StreamInfo {
|
||||
id,
|
||||
stream_type,
|
||||
language,
|
||||
variant,
|
||||
purpose,
|
||||
qualifier,
|
||||
});
|
||||
pos = block_end;
|
||||
infos.push(StreamInfo {
|
||||
id,
|
||||
stream_type,
|
||||
language,
|
||||
variant,
|
||||
purpose,
|
||||
qualifier,
|
||||
});
|
||||
from = end;
|
||||
}
|
||||
}
|
||||
infos
|
||||
}
|
||||
|
||||
fn parse_playback_config(xml: &str, map: &mut HashMap<String, u16>) {
|
||||
let mut pos = 0;
|
||||
while pos < xml.len() {
|
||||
let tag_start = if let Some(p) = xml[pos..].find("<AudioStreams>") {
|
||||
Some(p + pos)
|
||||
} else {
|
||||
xml[pos..].find("<SubtitlesStreams>").map(|p| p + pos)
|
||||
};
|
||||
|
||||
let tag_start = match tag_start {
|
||||
Some(p) => p,
|
||||
None => break,
|
||||
};
|
||||
|
||||
let block_end = xml[tag_start..]
|
||||
.find("</AudioStreams>")
|
||||
.or_else(|| xml[tag_start..].find("</SubtitlesStreams>"))
|
||||
.map(|p| tag_start + p + 20)
|
||||
.unwrap_or(xml.len());
|
||||
|
||||
let block = &xml[tag_start..block_end];
|
||||
|
||||
if let (Some(stream_id_str), Some(info_id)) = (
|
||||
extract_tag(block, "StreamID"),
|
||||
extract_tag(block, "StreamInfo_ID"),
|
||||
) {
|
||||
if let Ok(stream_num) = stream_id_str.parse::<u16>() {
|
||||
map.insert(info_id, stream_num);
|
||||
fn parse_playback_config(text: &str, map: &mut HashMap<String, u16>) {
|
||||
for tag_name in ["AudioStreams", "SubtitlesStreams"] {
|
||||
let mut from = 0;
|
||||
while let Some((start, end)) = xml::find_element(text, tag_name, from) {
|
||||
let block = &text[start..end];
|
||||
if let (Some(stream_id_str), Some(info_id)) = (
|
||||
xml::text(block, "StreamID"),
|
||||
xml::text(block, "StreamInfo_ID"),
|
||||
) {
|
||||
if let Ok(stream_num) = stream_id_str.parse::<u16>() {
|
||||
map.insert(info_id, stream_num);
|
||||
}
|
||||
}
|
||||
from = end;
|
||||
}
|
||||
|
||||
pos = block_end;
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_tag(xml: &str, tag: &str) -> Option<String> {
|
||||
let open = format!("<{tag}>");
|
||||
let close = format!("</{tag}>");
|
||||
let start = xml.find(&open)? + open.len();
|
||||
let end = xml[start..].find(&close)? + start;
|
||||
Some(xml[start..end].trim().to_string())
|
||||
}
|
||||
|
||||
+185
-17
@@ -4,8 +4,8 @@
|
||||
//! When both exist, language_streams.txt provides structured types while
|
||||
//! menu_base.prop provides stream number → button name mapping.
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType, vocab};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, vocab};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -14,7 +14,7 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
|| super::jar_file_exists(udf, "language_streams.txt")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
// Try language_streams.txt first (richer structured data)
|
||||
let ls_labels = parse_language_streams(reader, udf);
|
||||
|
||||
@@ -22,12 +22,18 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
let mb_labels = parse_menu_base(reader, udf);
|
||||
|
||||
// If we have both, merge: language_streams for structure, menu_base for names
|
||||
match (ls_labels, mb_labels) {
|
||||
(Some(ls), Some(mb)) => Some(merge(ls, mb)),
|
||||
(Some(ls), None) => Some(ls),
|
||||
(None, Some(mb)) => Some(mb),
|
||||
(None, None) => None,
|
||||
let labels = match (ls_labels, mb_labels) {
|
||||
(Some(ls), Some(mb)) => merge(ls, mb),
|
||||
(Some(ls), None) => ls,
|
||||
(None, Some(mb)) => mb,
|
||||
(None, None) => return None,
|
||||
};
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// High confidence: both language_streams.txt and menu_base.prop
|
||||
// are structured key-value formats with documented types.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
|
||||
fn merge(ls: Vec<StreamLabel>, mb: Vec<StreamLabel>) -> Vec<StreamLabel> {
|
||||
@@ -49,7 +55,7 @@ fn merge(ls: Vec<StreamLabel>, mb: Vec<StreamLabel>) -> Vec<StreamLabel> {
|
||||
|
||||
// ── language_streams.txt parser ────────────────────────────────────────────
|
||||
|
||||
fn parse_language_streams(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
fn parse_language_streams(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
let data = super::read_jar_file(reader, udf, "language_streams.txt")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
@@ -172,9 +178,167 @@ fn parse_language_streams(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<
|
||||
Some(labels)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Build a minimal menu_base.prop text and run `parse_menu_base`'s
|
||||
/// inner logic via a temporary closure. This isolates the prop
|
||||
/// parsing without needing a SectorSource.
|
||||
fn parse_props(text: &str) -> Vec<StreamLabel> {
|
||||
// Mirror the inner loop of parse_menu_base exactly. Kept
|
||||
// separate so the test doesn't need disc fixtures.
|
||||
use std::collections::HashMap;
|
||||
let mut entries: HashMap<String, HashMap<String, String>> = HashMap::new();
|
||||
for line in text.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let Some(eq_pos) = line.find('=') else {
|
||||
continue;
|
||||
};
|
||||
let full_key = &line[..eq_pos];
|
||||
let value = &line[eq_pos + 1..];
|
||||
if let Some(dot_pos) = full_key.rfind('.') {
|
||||
entries
|
||||
.entry(full_key[..dot_pos].to_string())
|
||||
.or_default()
|
||||
.insert(full_key[dot_pos + 1..].to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
let mut labels = Vec::new();
|
||||
for (prefix, props) in &entries {
|
||||
let is_audio = props
|
||||
.get("class")
|
||||
.is_some_and(|c| c.contains("AudioButton"))
|
||||
|| prefix.starts_with("audio_");
|
||||
let is_subtitle = props
|
||||
.get("class")
|
||||
.is_some_and(|c| c.contains("SubtitleButton"))
|
||||
|| prefix.starts_with("subtitle_");
|
||||
let stream_num_str = props
|
||||
.get("streamNumber")
|
||||
.or_else(|| props.get("audioStream"))
|
||||
.or_else(|| props.get("subtitleStream"));
|
||||
let stream_num: u16 = match stream_num_str.and_then(|s| s.parse().ok()) {
|
||||
Some(n) if n > 0 => n,
|
||||
_ => continue,
|
||||
};
|
||||
if !is_audio && !is_subtitle {
|
||||
continue;
|
||||
}
|
||||
let name = props.get("name").cloned().unwrap_or_default();
|
||||
let purpose = match vocab::purpose(&name) {
|
||||
LabelPurpose::Normal if prefix.contains("comm") => LabelPurpose::Commentary,
|
||||
p => p,
|
||||
};
|
||||
let qualifier = if is_subtitle {
|
||||
vocab::qualifier(&name)
|
||||
} else {
|
||||
LabelQualifier::None
|
||||
};
|
||||
let stream_type = if is_audio {
|
||||
StreamLabelType::Audio
|
||||
} else {
|
||||
StreamLabelType::Subtitle
|
||||
};
|
||||
let language = props
|
||||
.get("audioLanguage")
|
||||
.or_else(|| props.get("subtitleLanguage"))
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
labels.push(StreamLabel {
|
||||
stream_number: stream_num,
|
||||
stream_type,
|
||||
language,
|
||||
name,
|
||||
purpose,
|
||||
qualifier,
|
||||
codec_hint: String::new(),
|
||||
variant: String::new(),
|
||||
});
|
||||
}
|
||||
labels.sort_by_key(|l| (l.stream_type as u8, l.stream_number));
|
||||
labels
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commentary_via_name() {
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=2\n\
|
||||
audio_1.name=Director's Commentary\n\
|
||||
audio_1.audioLanguage=eng\n",
|
||||
);
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Commentary);
|
||||
assert_eq!(labels[0].language, "eng");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commentary_via_prefix_when_name_silent() {
|
||||
let labels = parse_props(
|
||||
"audio_commentary_1.class=AudioButton\n\
|
||||
audio_commentary_1.streamNumber=2\n\
|
||||
audio_commentary_1.name=Track 2\n\
|
||||
audio_commentary_1.audioLanguage=eng\n",
|
||||
);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commenter_does_not_false_match_commentary() {
|
||||
// Regression for the pre-refactor `name.contains("comment")`
|
||||
// bug: this would wrongly classify a "Commenter Pro" track as
|
||||
// Commentary. vocab::purpose enforces a word boundary.
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=2\n\
|
||||
audio_1.name=Commenter Pro Track\n\
|
||||
audio_1.audioLanguage=eng\n",
|
||||
);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptive_via_name() {
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=3\n\
|
||||
audio_1.name=English Descriptive Audio\n",
|
||||
);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sdh_only_on_subtitles() {
|
||||
// SDH applied to a subtitle stream.
|
||||
let labels = parse_props(
|
||||
"subtitle_1.class=SubtitleButton\n\
|
||||
subtitle_1.streamNumber=4\n\
|
||||
subtitle_1.name=English SDH\n",
|
||||
);
|
||||
assert_eq!(labels[0].qualifier, LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sdh_not_applied_to_audio_stream_even_if_name_contains_sdh() {
|
||||
// Audio streams should not pick up SDH (it's a subtitle
|
||||
// concept). Edge case: badly-authored name happens to include
|
||||
// "SDH" — we don't propagate it to audio metadata.
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=5\n\
|
||||
audio_1.name=English SDH (track?)\n",
|
||||
);
|
||||
assert_eq!(labels[0].qualifier, LabelQualifier::None);
|
||||
}
|
||||
}
|
||||
|
||||
// ── menu_base.prop parser ──────────────────────────────────────────────────
|
||||
|
||||
fn parse_menu_base(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
fn parse_menu_base(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
let data = super::read_jar_file(reader, udf, "menu_base.prop")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
@@ -231,16 +395,20 @@ fn parse_menu_base(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<Str
|
||||
}
|
||||
|
||||
let name = props.get("name").cloned().unwrap_or_default();
|
||||
let name_lower = name.to_lowercase();
|
||||
|
||||
let purpose = if name_lower.contains("comment") || prefix.contains("comm") {
|
||||
LabelPurpose::Commentary
|
||||
} else {
|
||||
LabelPurpose::Normal
|
||||
// Purpose: ask vocab first (word-boundary matched — avoids the
|
||||
// "Commenter" false positive the prior `name.contains("comment")`
|
||||
// had). Then fall back to the structural prefix check
|
||||
// (`audio_commentary.foo`-style keys group commentary streams
|
||||
// regardless of display name).
|
||||
let purpose = match vocab::purpose(&name) {
|
||||
LabelPurpose::Normal if prefix.contains("comm") => LabelPurpose::Commentary,
|
||||
p => p,
|
||||
};
|
||||
|
||||
let qualifier = if is_subtitle && name_lower.contains("sdh") {
|
||||
LabelQualifier::Sdh
|
||||
// Qualifier: only apply to subtitles (SDH is a subtitle concept).
|
||||
let qualifier = if is_subtitle {
|
||||
vocab::qualifier(&name)
|
||||
} else {
|
||||
LabelQualifier::None
|
||||
};
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
//! "dbp" framework — Magnolia Pictures BD-J authoring shop (per
|
||||
//! `bd-live.magpictures.com` referenced in the disc's
|
||||
//! `com/dbp/bluray.MenuXlet.perm`). Detected on UHD discs whose
|
||||
//! `/BDMV/JAR/<x>.jar` (top-level, not in a subdir) contains
|
||||
//! `com/dbp/` package paths.
|
||||
//!
|
||||
//! Stream labels live as plain ASCII strings inside compiled `.class`
|
||||
//! files in the jar — a quirk of the menu-rendering layer encoding
|
||||
//! its TextField positions and content as constant strings the
|
||||
//! Java compiler retained in the class string pool. Format observed
|
||||
//! in the corpus (Civil War UHD, 2024):
|
||||
//!
|
||||
//! ```text
|
||||
//! LTextField,Audio1,English Dolby Atmos,Fontstrip_Composite,...
|
||||
//! RTextField,Audio2,English Descriptive Audio,Fontstrip_Composite,...
|
||||
//! HTextField,Subtitle1,English SDH,Fontstrip_Composite,...
|
||||
//! ATextField,Subtitle0,None,Fontstrip_Composite,...
|
||||
//! ```
|
||||
//!
|
||||
//! The single uppercase letter before `TextField` is string-pool
|
||||
//! prefix noise — the parser anchors on `TextField,` regardless of
|
||||
//! what precedes it. `Subtitle0` is the disable-subtitles menu
|
||||
//! button and is skipped (not a real subtitle stream).
|
||||
//!
|
||||
//! ## Implementation
|
||||
//!
|
||||
//! v2 (2026-05-10): rewritten on top of [`super::class_reader`] —
|
||||
//! iterates `CpInfo::Utf8` constant-pool entries instead of raw byte
|
||||
//! scanning each class file. Equivalent label coverage (the literal
|
||||
//! `TextField,...` strings live in the CP as Utf8 entries), but
|
||||
//! structurally cleaner: no false-positive risk from method bytecode
|
||||
//! or attribute names happening to contain `TextField,`. Language /
|
||||
//! purpose / qualifier classification moved to [`super::vocab`] so all
|
||||
//! Java-parser families share one source of truth.
|
||||
|
||||
use super::class_reader::CpInfo;
|
||||
use super::{ParseResult, StreamLabel, StreamLabelType, jar, vocab};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// dbp detect can't peek inside a jar without a SectorSource (the
|
||||
/// trait function only takes `&UdfFs`), so we trigger on the cheap
|
||||
/// signal "any top-level .jar in /BDMV/JAR/." That fires on every
|
||||
/// BD-J disc, but parse() does the real `com/dbp/` check and
|
||||
/// returns None on a mismatch — so this parser only ever consumes
|
||||
/// time on discs that fell through every earlier parser.
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
jar::has_any_top_level_jar(udf)
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
jar::for_each_jar(reader, udf, |_entry_name, archive| {
|
||||
if !jar::has_path_prefix(archive, "com/dbp/") {
|
||||
return None;
|
||||
}
|
||||
let labels = scan_jar(archive);
|
||||
if labels.is_empty() {
|
||||
None
|
||||
} else {
|
||||
// High confidence: TextField,Audio1,... is a stable anchor
|
||||
// pattern + vocab routes language/purpose/qualifier.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn scan_jar(archive: &mut jar::Jar) -> Vec<StreamLabel> {
|
||||
// BTreeMap so we keep the highest-numbered (last-written) label
|
||||
// for each stream slot deterministic across runs. The same
|
||||
// TextField,Audio1,... string can appear in multiple classes
|
||||
// (button-state variants, localization fallbacks). Last write
|
||||
// wins — they should all agree, but the structure is defensive.
|
||||
let mut audios: BTreeMap<u16, String> = BTreeMap::new();
|
||||
let mut subs: BTreeMap<u16, String> = BTreeMap::new();
|
||||
|
||||
jar::for_each_class(archive, |_class_name, class| {
|
||||
for (_idx, cp) in class.constant_pool.iter() {
|
||||
if let CpInfo::Utf8(s) = cp {
|
||||
collect_textfield(s, &mut audios, &mut subs);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut out = Vec::new();
|
||||
for (num, label) in audios {
|
||||
out.push(make_label(num, label, StreamLabelType::Audio));
|
||||
}
|
||||
for (num, label) in subs {
|
||||
out.push(make_label(num, label, StreamLabelType::Subtitle));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn collect_textfield(
|
||||
s: &str,
|
||||
audios: &mut BTreeMap<u16, String>,
|
||||
subs: &mut BTreeMap<u16, String>,
|
||||
) {
|
||||
// Anchor on "TextField," — the prefix character before it varies
|
||||
// (string-pool ordering inside compiled Java) and is irrelevant.
|
||||
let Some(idx) = s.find("TextField,") else {
|
||||
return;
|
||||
};
|
||||
let after = &s[idx + "TextField,".len()..];
|
||||
let mut parts = after.splitn(3, ',');
|
||||
let kind_n = parts.next().unwrap_or("").trim();
|
||||
let label = parts.next().unwrap_or("").trim();
|
||||
if label.is_empty() {
|
||||
return;
|
||||
}
|
||||
if let Some(rest) = kind_n.strip_prefix("Audio") {
|
||||
if let Ok(n) = rest.parse::<u16>() {
|
||||
audios.insert(n, label.to_string());
|
||||
}
|
||||
} else if let Some(rest) = kind_n.strip_prefix("Subtitle") {
|
||||
if let Ok(n) = rest.parse::<u16>() {
|
||||
// Subtitle0 is conventionally the "None / Off" disable
|
||||
// button, not an actual subtitle stream.
|
||||
if n > 0 {
|
||||
subs.insert(n, label.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn make_label(num: u16, label: String, stream_type: StreamLabelType) -> StreamLabel {
|
||||
let lang_info = vocab::lang(&label);
|
||||
let language = lang_info.map(|l| l.code).unwrap_or("").to_string();
|
||||
let variant = lang_info.map(|l| l.variant).unwrap_or("").to_string();
|
||||
let qualifier = vocab::qualifier(&label);
|
||||
let purpose = vocab::purpose(&label);
|
||||
StreamLabel {
|
||||
stream_number: num,
|
||||
stream_type,
|
||||
language,
|
||||
name: label,
|
||||
purpose,
|
||||
qualifier,
|
||||
codec_hint: String::new(),
|
||||
variant,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::{LabelPurpose, LabelQualifier};
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn collect_extracts_audio_and_subtitle_indices() {
|
||||
let mut audios = BTreeMap::new();
|
||||
let mut subs = BTreeMap::new();
|
||||
let lines = [
|
||||
"LTextField,Audio1,English Dolby Atmos,Fontstrip_Composite,296,763,275,25,left",
|
||||
"RTextField,Audio2,English Descriptive Audio,Fontstrip_Composite,296,803,275,25,left",
|
||||
"RTextField,Audio3,Spanish 5.1 Dolby Digital,Fontstrip_Composite,296,843,275,25,left",
|
||||
"ATextField,Subtitle0,None,Fontstrip_Composite,1312,843,275,25,left",
|
||||
"HTextField,Subtitle1,English SDH,Fontstrip_Composite,1312,763,275,25,left",
|
||||
"DTextField,Subtitle2,Spanish,Fontstrip_Composite,1312,803,275,25,left",
|
||||
];
|
||||
for s in &lines {
|
||||
collect_textfield(s, &mut audios, &mut subs);
|
||||
}
|
||||
assert_eq!(audios.len(), 3);
|
||||
assert_eq!(audios[&1], "English Dolby Atmos");
|
||||
assert_eq!(audios[&2], "English Descriptive Audio");
|
||||
assert_eq!(audios[&3], "Spanish 5.1 Dolby Digital");
|
||||
// Subtitle0 ("None") is skipped — disable button, not a stream.
|
||||
assert_eq!(subs.len(), 2);
|
||||
assert_eq!(subs[&1], "English SDH");
|
||||
assert_eq!(subs[&2], "Spanish");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_ignores_non_textfield_strings() {
|
||||
let mut audios = BTreeMap::new();
|
||||
let mut subs = BTreeMap::new();
|
||||
for s in [
|
||||
"GraphicButton,SU_Audio",
|
||||
"AudioMenu",
|
||||
"CommentaryMenuAlternateScenes",
|
||||
"PrimaryAudioControl",
|
||||
] {
|
||||
collect_textfield(s, &mut audios, &mut subs);
|
||||
}
|
||||
assert!(audios.is_empty());
|
||||
assert!(subs.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_routes_via_vocab() {
|
||||
let l = make_label(1, "English SDH".to_string(), StreamLabelType::Subtitle);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.qualifier, LabelQualifier::Sdh);
|
||||
assert_eq!(l.purpose, LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_descriptive_audio() {
|
||||
let l = make_label(
|
||||
2,
|
||||
"English Descriptive Audio".to_string(),
|
||||
StreamLabelType::Audio,
|
||||
);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.purpose, LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_commentary() {
|
||||
let l = make_label(
|
||||
3,
|
||||
"English Director's Commentary".to_string(),
|
||||
StreamLabelType::Audio,
|
||||
);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_compound_languages_populate_variant() {
|
||||
let brazilian = make_label(1, "Brazilian Portuguese 5.1".into(), StreamLabelType::Audio);
|
||||
assert_eq!(brazilian.language, "por");
|
||||
assert_eq!(brazilian.variant, "Brazilian");
|
||||
|
||||
let castilian = make_label(1, "Castilian Spanish".into(), StreamLabelType::Audio);
|
||||
assert_eq!(castilian.language, "spa");
|
||||
assert_eq!(castilian.variant, "Castilian");
|
||||
|
||||
let canadian = make_label(
|
||||
1,
|
||||
"Canadian French Dolby Digital".into(),
|
||||
StreamLabelType::Audio,
|
||||
);
|
||||
assert_eq!(canadian.language, "fra");
|
||||
assert_eq!(canadian.variant, "Canadian");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_bare_language_has_empty_variant() {
|
||||
let l = make_label(1, "English Dolby Atmos".into(), StreamLabelType::Audio);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.variant, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_unknown_language_is_empty() {
|
||||
// vocab::lang returns None — make_label converts both fields to "".
|
||||
let l = make_label(1, "Klingon Dolby Atmos".into(), StreamLabelType::Audio);
|
||||
assert_eq!(l.language, "");
|
||||
assert_eq!(l.variant, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_rnib_descriptive_service() {
|
||||
let l = make_label(1, "English RNIB".into(), StreamLabelType::Subtitle);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.qualifier, LabelQualifier::DescriptiveService);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,151 @@
|
||||
//! BD-J jar utilities — common scaffolding for parsers that read
|
||||
//! `/BDMV/JAR/*.jar`.
|
||||
//!
|
||||
//! Composes with [`class_reader`](super::class_reader) for structured
|
||||
//! `.class` access. Used by `dbp` (string-pool scan via constant pool)
|
||||
//! and `deluxe` (bytecode pattern matching) — those parsers express
|
||||
//! "open every top-level jar, look at every .class inside" without
|
||||
//! repeating the zip-archive boilerplate.
|
||||
|
||||
// `try_each_class` is staged for `labels::deluxe`, which needs the
|
||||
// early-return form to short-circuit class iteration on a match.
|
||||
// dead-code allow comes off when deluxe lands.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use super::class_reader::ClassFile;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::io::Cursor;
|
||||
use zip::ZipArchive;
|
||||
|
||||
/// In-memory zip archive: backed by a `Vec<u8>` read from UDF. Owns
|
||||
/// the buffer; callers pass it to [`has_path_prefix`], [`for_each_class`],
|
||||
/// etc.
|
||||
pub type Jar = ZipArchive<Cursor<Vec<u8>>>;
|
||||
|
||||
/// True if `/BDMV/JAR/` contains at least one top-level `.jar` file
|
||||
/// (not under a subdir). Used by `detect()` in parsers whose real
|
||||
/// signal lives inside a jar — they can't open the jar without a
|
||||
/// `SectorSource`, so they use this cheap pre-check and do the real
|
||||
/// `com/<vendor>/` discriminator in `parse()`.
|
||||
pub fn has_any_top_level_jar(udf: &UdfFs) -> bool {
|
||||
let Some(jar_dir) = udf.find_dir("/BDMV/JAR") else {
|
||||
return false;
|
||||
};
|
||||
jar_dir
|
||||
.entries
|
||||
.iter()
|
||||
.any(|e| !e.is_dir && e.name.to_lowercase().ends_with(".jar"))
|
||||
}
|
||||
|
||||
/// Open every top-level `*.jar` entry in `/BDMV/JAR/` and yield each
|
||||
/// `(entry_name, Jar)` to `f`. Returns the first `Some(R)` the callback
|
||||
/// produces, or `None` if every jar was visited without a hit.
|
||||
///
|
||||
/// "Top-level" means entries directly under `/BDMV/JAR/`, not nested
|
||||
/// under a subdir. (Pixelogic, Criterion, Paramount, etc. put their
|
||||
/// data files inside `/BDMV/JAR/<x>/`; dbp and Deluxe put their jar
|
||||
/// directly at `/BDMV/JAR/<name>.jar`.)
|
||||
///
|
||||
/// Entries that fail to read from UDF or that aren't valid zips are
|
||||
/// silently skipped — same defensive shape as the existing dbp parser.
|
||||
pub fn for_each_jar<R, F>(reader: &mut dyn SectorSource, udf: &UdfFs, mut f: F) -> Option<R>
|
||||
where
|
||||
F: FnMut(&str, &mut Jar) -> Option<R>,
|
||||
{
|
||||
let jar_dir = udf.find_dir("/BDMV/JAR")?;
|
||||
for entry in &jar_dir.entries {
|
||||
if entry.is_dir {
|
||||
continue;
|
||||
}
|
||||
if !entry.name.to_lowercase().ends_with(".jar") {
|
||||
continue;
|
||||
}
|
||||
let path = format!("/BDMV/JAR/{}", entry.name);
|
||||
let Ok(bytes) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(mut archive) = ZipArchive::new(Cursor::new(bytes)) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(r) = f(&entry.name, &mut archive) {
|
||||
return Some(r);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// True if any entry in this jar's central directory starts with
|
||||
/// `prefix`. Fast — only reads filenames, never extracts bytes.
|
||||
///
|
||||
/// Used by parsers as a cheap "is this MY framework's jar?" check
|
||||
/// (e.g. `has_path_prefix(archive, "com/dbp/")` for dbp,
|
||||
/// `has_path_prefix(archive, "com/bydeluxe/")` for Deluxe).
|
||||
pub fn has_path_prefix(archive: &mut Jar, prefix: &str) -> bool {
|
||||
for i in 0..archive.len() {
|
||||
if let Ok(f) = archive.by_index(i) {
|
||||
if f.name().starts_with(prefix) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Iterate every `.class` entry in the jar, parse it with
|
||||
/// [`class_reader`], and call `f` with `(entry_name, &ClassFile)`.
|
||||
///
|
||||
/// Entries that fail to read or parse are silently skipped — this is
|
||||
/// label-extraction code, robustness matters more than completeness.
|
||||
/// Callers that need to know which classes failed should use the
|
||||
/// lower-level [`class_reader`] API directly.
|
||||
pub fn for_each_class<F>(archive: &mut Jar, mut f: F)
|
||||
where
|
||||
F: FnMut(&str, &ClassFile),
|
||||
{
|
||||
for i in 0..archive.len() {
|
||||
let Ok(mut entry) = archive.by_index(i) else {
|
||||
continue;
|
||||
};
|
||||
if !entry.name().ends_with(".class") {
|
||||
continue;
|
||||
}
|
||||
let name = entry.name().to_string();
|
||||
let mut bytes = Vec::with_capacity(entry.size() as usize);
|
||||
if std::io::Read::read_to_end(&mut entry, &mut bytes).is_err() {
|
||||
continue;
|
||||
}
|
||||
let Ok(class) = ClassFile::parse(&bytes) else {
|
||||
continue;
|
||||
};
|
||||
f(&name, &class);
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`for_each_class`] but allows the callback to short-circuit
|
||||
/// iteration. Returns the first `Some(R)` the callback produces.
|
||||
pub fn try_each_class<R, F>(archive: &mut Jar, mut f: F) -> Option<R>
|
||||
where
|
||||
F: FnMut(&str, &ClassFile) -> Option<R>,
|
||||
{
|
||||
for i in 0..archive.len() {
|
||||
let Ok(mut entry) = archive.by_index(i) else {
|
||||
continue;
|
||||
};
|
||||
if !entry.name().ends_with(".class") {
|
||||
continue;
|
||||
}
|
||||
let name = entry.name().to_string();
|
||||
let mut bytes = Vec::with_capacity(entry.size() as usize);
|
||||
if std::io::Read::read_to_end(&mut entry, &mut bytes).is_err() {
|
||||
continue;
|
||||
}
|
||||
let Ok(class) = ClassFile::parse(&bytes) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(r) = f(&name, &class) {
|
||||
return Some(r);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
+1130
-15
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,597 @@
|
||||
//! Universal MPLS-based stream labels.
|
||||
//!
|
||||
//! Unlike the framework-specific parsers in this directory (dbp,
|
||||
//! pixelogic, ctrm, criterion, ...), this module is the *floor*:
|
||||
//! every Blu-ray ships with MPLS playlists under `/BDMV/PLAYLIST/`,
|
||||
//! and every MPLS file has an STN table with per-stream ISO 639-2
|
||||
//! language codes plus coding-type / channel-layout / sample-rate
|
||||
//! bytes from the BD spec.
|
||||
//!
|
||||
//! The framework parsers extract richer editorial labels ("English
|
||||
//! Dolby Atmos", "Director's Commentary") when the disc was authored
|
||||
//! with a recognized tool. When none of them match (e.g. a "no BD-J"
|
||||
//! disc, or an authoring framework we haven't catalogued), MPLS still
|
||||
//! gives us language + codec on every stream — enough to render
|
||||
//! something more useful than the bare PID.
|
||||
//!
|
||||
//! Output confidence is Low: MPLS carries language + codec but
|
||||
//! never purpose/qualifier info (no way to tell "Commentary" from
|
||||
//! "Normal" from the STN table alone). Higher-confidence framework
|
||||
//! parsers, when present, always win on the registry's max-by-confidence
|
||||
//! tiebreaker — MPLS is only chosen when nothing else matched.
|
||||
|
||||
use super::{
|
||||
LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType,
|
||||
vocab::{self, LangInfo},
|
||||
};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
/// True iff `/BDMV/PLAYLIST/` exists and contains at least one
|
||||
/// `.mpls` file. Cheap directory walk only — no sector reads.
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
let Some(dir) = udf.find_dir("/BDMV/PLAYLIST") else {
|
||||
return false;
|
||||
};
|
||||
dir.entries
|
||||
.iter()
|
||||
.any(|e| !e.is_dir && has_mpls_extension(&e.name))
|
||||
}
|
||||
|
||||
/// Walk every `*.mpls` in `/BDMV/PLAYLIST/`, parse it, and convert
|
||||
/// each StreamEntry to a [`StreamLabel`]. Streams shared across
|
||||
/// playlists (same PID) are deduped.
|
||||
///
|
||||
/// Returns `None` if no labels could be produced (e.g. no .mpls files
|
||||
/// parsed successfully, or every parsed stream was a type we skip
|
||||
/// like IG / DV EL).
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let playlist_dir = udf.find_dir("/BDMV/PLAYLIST")?;
|
||||
|
||||
// Collect mpls filenames first so we don't hold a borrow on udf
|
||||
// while we call udf.read_file (which takes &self).
|
||||
let mpls_names: Vec<String> = playlist_dir
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && has_mpls_extension(&e.name))
|
||||
.map(|e| e.name.clone())
|
||||
.collect();
|
||||
|
||||
if mpls_names.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut labels: Vec<StreamLabel> = Vec::new();
|
||||
// (stream_type_tag, language, codec_hint, pid) — PID is the
|
||||
// canonical "same physical stream" key; type+lang+codec round
|
||||
// out the rare case where two distinct logical streams happen
|
||||
// to share a PID across playlists with different metadata.
|
||||
let mut seen: Vec<(u8, String, String, u16)> = Vec::new();
|
||||
|
||||
// Global 1-based counters keyed by StreamLabelType. Incremented
|
||||
// only when an entry survives dedup, so stream_numbers are dense
|
||||
// (1, 2, 3, ...) per type across the whole disc — not reset per
|
||||
// playlist. A disc with 2 MPLS files that each list the same
|
||||
// 8 audio streams ends up with audio_1..audio_8, not audio_1..
|
||||
// audio_16 or audio_1..audio_8 with audio_1 duplicated.
|
||||
let mut audio_idx: u16 = 0;
|
||||
let mut sub_idx: u16 = 0;
|
||||
|
||||
for name in &mpls_names {
|
||||
let path = format!("/BDMV/PLAYLIST/{}", name);
|
||||
let Ok(data) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(playlist) = crate::mpls::parse(&data) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
for entry in &playlist.streams {
|
||||
let label_type = match entry.stream_type {
|
||||
2 | 5 => StreamLabelType::Audio, // primary + secondary audio
|
||||
3 => StreamLabelType::Subtitle, // PG subtitle
|
||||
// 1 = primary video, 6 = secondary video, 7 = DV EL
|
||||
// → no StreamLabelType variant for video, skip.
|
||||
// 4 = IG (interactive graphics) — not a user-facing
|
||||
// stream, skip.
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
let language = normalize_language(&entry.language);
|
||||
let name = language_display_name(&language);
|
||||
let codec_hint = build_codec_hint(label_type, entry);
|
||||
|
||||
let type_tag = type_tag(label_type);
|
||||
let key = (type_tag, language.clone(), codec_hint.clone(), entry.pid);
|
||||
if seen.contains(&key) {
|
||||
continue;
|
||||
}
|
||||
seen.push(key);
|
||||
|
||||
let stream_number = match label_type {
|
||||
StreamLabelType::Audio => {
|
||||
audio_idx += 1;
|
||||
audio_idx
|
||||
}
|
||||
StreamLabelType::Subtitle => {
|
||||
sub_idx += 1;
|
||||
sub_idx
|
||||
}
|
||||
};
|
||||
|
||||
labels.push(StreamLabel {
|
||||
stream_number,
|
||||
stream_type: label_type,
|
||||
language,
|
||||
name,
|
||||
purpose: LabelPurpose::Normal,
|
||||
qualifier: LabelQualifier::None,
|
||||
codec_hint,
|
||||
variant: String::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// MPLS gives language + codec but never editorial info (no
|
||||
// commentary/SDH/director's cut). Low confidence means framework
|
||||
// parsers (paramount, criterion, pixelogic, ctrm, dbp, deluxe) always
|
||||
// win when they match. MPLS only gets chosen as the parser when
|
||||
// nothing else fired — exactly the universal-fallback role we want.
|
||||
Some(ParseResult::low(labels))
|
||||
}
|
||||
|
||||
fn has_mpls_extension(name: &str) -> bool {
|
||||
// Case-insensitive ".mpls" suffix. Some discs use uppercase,
|
||||
// some lowercase; UDF filenames preserve case but we don't.
|
||||
let n = name.len();
|
||||
if n < 5 {
|
||||
return false;
|
||||
}
|
||||
name[n - 5..].eq_ignore_ascii_case(".mpls")
|
||||
}
|
||||
|
||||
/// Lowercase + trim the raw 3-char ISO 639-2 code. If the lowered
|
||||
/// string maps via [`vocab::lang`] (it won't for plain "eng" — that
|
||||
/// matcher is for English-name fragments, not codes) use its
|
||||
/// canonical code; otherwise return the trimmed lowercase string.
|
||||
fn normalize_language(raw: &str) -> String {
|
||||
let trimmed = raw.trim().to_ascii_lowercase();
|
||||
if trimmed.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
// vocab::lang() matches free-form English names, not ISO 639-2
|
||||
// codes — so for the typical MPLS payload ("eng", "fra", ...)
|
||||
// it returns None and we keep the trimmed code.
|
||||
if let Some(LangInfo { code, .. }) = vocab::lang(&trimmed) {
|
||||
return code.to_string();
|
||||
}
|
||||
trimmed
|
||||
}
|
||||
|
||||
/// Human-readable English name for an ISO 639-2 code, or empty if
|
||||
/// the code is unknown. Kept inline rather than in vocab because
|
||||
/// vocab is the *reverse* mapping (name → code).
|
||||
pub(crate) fn language_display_name(iso: &str) -> String {
|
||||
match iso {
|
||||
"eng" => "English",
|
||||
"fra" | "fre" => "French",
|
||||
"spa" => "Spanish",
|
||||
"deu" | "ger" => "German",
|
||||
"ita" => "Italian",
|
||||
"jpn" => "Japanese",
|
||||
"zho" | "chi" => "Chinese",
|
||||
"kor" => "Korean",
|
||||
"por" => "Portuguese",
|
||||
"pol" => "Polish",
|
||||
"ces" | "cze" => "Czech",
|
||||
"hun" => "Hungarian",
|
||||
"nld" | "dut" => "Dutch",
|
||||
"ara" => "Arabic",
|
||||
"hin" => "Hindi",
|
||||
"tur" => "Turkish",
|
||||
"tha" => "Thai",
|
||||
"swe" => "Swedish",
|
||||
"nor" => "Norwegian",
|
||||
"dan" => "Danish",
|
||||
"fin" => "Finnish",
|
||||
"heb" => "Hebrew",
|
||||
"rus" => "Russian",
|
||||
"ell" | "gre" => "Greek",
|
||||
"vie" => "Vietnamese",
|
||||
"ind" => "Indonesian",
|
||||
"msa" | "may" => "Malay",
|
||||
"ukr" => "Ukrainian",
|
||||
"ron" | "rum" => "Romanian",
|
||||
"bul" => "Bulgarian",
|
||||
"hrv" => "Croatian",
|
||||
"srp" => "Serbian",
|
||||
"slk" | "slo" => "Slovak",
|
||||
"slv" => "Slovenian",
|
||||
"est" => "Estonian",
|
||||
"lav" => "Latvian",
|
||||
"lit" => "Lithuanian",
|
||||
"isl" | "ice" => "Icelandic",
|
||||
"eus" | "baq" => "Basque",
|
||||
"cat" => "Catalan",
|
||||
"glg" => "Galician",
|
||||
_ => "",
|
||||
}
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Map BD coding_type byte → codec name. Returns empty for unknown
|
||||
/// bytes (the table covers everything the spec defines, but unknown
|
||||
/// values are still possible on malformed discs).
|
||||
pub(crate) fn codec_name(coding_type: u8) -> &'static str {
|
||||
match coding_type {
|
||||
0x02 => "MPEG-2",
|
||||
0x1B => "H.264",
|
||||
0x24 => "HEVC",
|
||||
0x80 => "LPCM",
|
||||
0x81 => "AC-3",
|
||||
0x82 => "DTS",
|
||||
0x83 => "TrueHD",
|
||||
0x84 => "AC-3+",
|
||||
0x85 => "DTS-HD",
|
||||
0x86 => "DTS-HD MA",
|
||||
0x90 => "PG",
|
||||
0x91 => "IG",
|
||||
0xA1 => "AC-3+ Secondary",
|
||||
0xA2 => "DTS-HD Secondary",
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the final `codec_hint`. For audio streams, optionally
|
||||
/// append " <channels>" and/or " <rate>" suffixes. Sample rate is
|
||||
/// only spelled out for non-48k (the universal default).
|
||||
fn build_codec_hint(label_type: StreamLabelType, entry: &crate::mpls::StreamEntry) -> String {
|
||||
let base = codec_name(entry.coding_type);
|
||||
if base.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
if label_type != StreamLabelType::Audio {
|
||||
return base.to_string();
|
||||
}
|
||||
|
||||
let mut out = base.to_string();
|
||||
|
||||
let channels = match entry.audio_format {
|
||||
1 => Some("mono"),
|
||||
3 => Some("2.0"),
|
||||
6 => Some("5.1"),
|
||||
12 => Some("7.1"),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(ch) = channels {
|
||||
out.push(' ');
|
||||
out.push_str(ch);
|
||||
}
|
||||
|
||||
// 1 = 48 kHz (universal default, omit). Only call out higher rates.
|
||||
let rate = match entry.audio_rate {
|
||||
4 => Some("96kHz"),
|
||||
5 => Some("192kHz"),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(r) = rate {
|
||||
out.push(' ');
|
||||
out.push_str(r);
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
/// Dedup-tag for the label type. `u8` instead of `StreamLabelType`
|
||||
/// itself because the enum does not derive `Hash` / `Eq`-by-discriminant
|
||||
/// in a way that we want to couple to (and `==` works fine for the
|
||||
/// linear `Vec::contains` lookup we do).
|
||||
fn type_tag(t: StreamLabelType) -> u8 {
|
||||
match t {
|
||||
StreamLabelType::Audio => 1,
|
||||
StreamLabelType::Subtitle => 2,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::mpls::{Playlist, StreamEntry};
|
||||
|
||||
fn audio_entry(pid: u16, coding: u8, fmt: u8, rate: u8, lang: &str) -> StreamEntry {
|
||||
StreamEntry {
|
||||
stream_type: 2,
|
||||
pid,
|
||||
coding_type: coding,
|
||||
video_format: 0,
|
||||
video_rate: 0,
|
||||
audio_format: fmt,
|
||||
audio_rate: rate,
|
||||
language: lang.to_string(),
|
||||
dynamic_range: 0,
|
||||
color_space: 0,
|
||||
secondary: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn pg_entry(pid: u16, lang: &str) -> StreamEntry {
|
||||
StreamEntry {
|
||||
stream_type: 3,
|
||||
pid,
|
||||
coding_type: 0x90,
|
||||
video_format: 0,
|
||||
video_rate: 0,
|
||||
audio_format: 0,
|
||||
audio_rate: 0,
|
||||
language: lang.to_string(),
|
||||
dynamic_range: 0,
|
||||
color_space: 0,
|
||||
secondary: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn playlist_with(streams: Vec<StreamEntry>) -> Playlist {
|
||||
Playlist {
|
||||
version: "0200".to_string(),
|
||||
play_items: Vec::new(),
|
||||
streams,
|
||||
marks: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Drive the same conversion logic that `parse()` runs on real
|
||||
/// disc data, but starting from already-parsed Playlists so we
|
||||
/// don't have to synthesize valid MPLS bytes.
|
||||
fn labels_from_playlists(playlists: &[Playlist]) -> Vec<StreamLabel> {
|
||||
let mut labels: Vec<StreamLabel> = Vec::new();
|
||||
let mut seen: Vec<(u8, String, String, u16)> = Vec::new();
|
||||
|
||||
for playlist in playlists {
|
||||
let mut audio_idx: u16 = 0;
|
||||
let mut sub_idx: u16 = 0;
|
||||
|
||||
for entry in &playlist.streams {
|
||||
let label_type = match entry.stream_type {
|
||||
2 | 5 => StreamLabelType::Audio,
|
||||
3 => StreamLabelType::Subtitle,
|
||||
_ => continue,
|
||||
};
|
||||
let stream_number = match label_type {
|
||||
StreamLabelType::Audio => {
|
||||
audio_idx += 1;
|
||||
audio_idx
|
||||
}
|
||||
StreamLabelType::Subtitle => {
|
||||
sub_idx += 1;
|
||||
sub_idx
|
||||
}
|
||||
};
|
||||
let language = normalize_language(&entry.language);
|
||||
let name = language_display_name(&language);
|
||||
let codec_hint = build_codec_hint(label_type, entry);
|
||||
let key = (
|
||||
type_tag(label_type),
|
||||
language.clone(),
|
||||
codec_hint.clone(),
|
||||
entry.pid,
|
||||
);
|
||||
if seen.contains(&key) {
|
||||
continue;
|
||||
}
|
||||
seen.push(key);
|
||||
labels.push(StreamLabel {
|
||||
stream_number,
|
||||
stream_type: label_type,
|
||||
language,
|
||||
name,
|
||||
purpose: LabelPurpose::Normal,
|
||||
qualifier: LabelQualifier::None,
|
||||
codec_hint,
|
||||
variant: String::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
labels
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mpls_audio_streams_become_labels() {
|
||||
// Two audio streams: English TrueHD 7.1 48k, French AC-3 5.1 48k.
|
||||
let pl = playlist_with(vec![
|
||||
audio_entry(0x1100, 0x83, 12, 1, "eng"),
|
||||
audio_entry(0x1101, 0x81, 6, 1, "fra"),
|
||||
]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert_eq!(labels.len(), 2);
|
||||
|
||||
// English TrueHD 7.1
|
||||
let a = &labels[0];
|
||||
assert_eq!(a.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(a.stream_number, 1);
|
||||
assert_eq!(a.language, "eng");
|
||||
assert_eq!(a.name, "English");
|
||||
assert_eq!(a.codec_hint, "TrueHD 7.1");
|
||||
assert_eq!(a.purpose, LabelPurpose::Normal);
|
||||
assert_eq!(a.qualifier, LabelQualifier::None);
|
||||
assert_eq!(a.variant, "");
|
||||
|
||||
// French AC-3 5.1
|
||||
let b = &labels[1];
|
||||
assert_eq!(b.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(b.stream_number, 2);
|
||||
assert_eq!(b.language, "fra");
|
||||
assert_eq!(b.name, "French");
|
||||
assert_eq!(b.codec_hint, "AC-3 5.1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mpls_pg_streams_become_subtitle_labels() {
|
||||
let pl = playlist_with(vec![
|
||||
pg_entry(0x1200, "eng"),
|
||||
pg_entry(0x1201, "spa"),
|
||||
pg_entry(0x1202, "fra"),
|
||||
]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert_eq!(labels.len(), 3);
|
||||
for label in &labels {
|
||||
assert_eq!(label.stream_type, StreamLabelType::Subtitle);
|
||||
assert_eq!(label.codec_hint, "PG");
|
||||
}
|
||||
assert_eq!(labels[0].stream_number, 1);
|
||||
assert_eq!(labels[0].language, "eng");
|
||||
assert_eq!(labels[0].name, "English");
|
||||
assert_eq!(labels[1].stream_number, 2);
|
||||
assert_eq!(labels[1].language, "spa");
|
||||
assert_eq!(labels[1].name, "Spanish");
|
||||
assert_eq!(labels[2].stream_number, 3);
|
||||
assert_eq!(labels[2].language, "fra");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_streams_across_playlists() {
|
||||
// Two playlists, same English TrueHD 7.1 PID 0x1100 in both.
|
||||
// Expect one Audio label, not two.
|
||||
let pl1 = playlist_with(vec![
|
||||
audio_entry(0x1100, 0x83, 12, 1, "eng"),
|
||||
audio_entry(0x1101, 0x81, 6, 1, "fra"),
|
||||
]);
|
||||
let pl2 = playlist_with(vec![
|
||||
audio_entry(0x1100, 0x83, 12, 1, "eng"), // duplicate
|
||||
audio_entry(0x1102, 0x82, 6, 1, "deu"), // new
|
||||
]);
|
||||
let labels = labels_from_playlists(&[pl1, pl2]);
|
||||
// Expected: eng@0x1100, fra@0x1101, deu@0x1102 — three uniques.
|
||||
assert_eq!(labels.len(), 3);
|
||||
// PID isn't stored on StreamLabel, so assert on the surviving
|
||||
// language set instead.
|
||||
let mut langs: Vec<String> = labels.iter().map(|l| l.language.clone()).collect();
|
||||
langs.sort();
|
||||
assert_eq!(langs, vec!["deu", "eng", "fra"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coding_type_to_codec_hint_table() {
|
||||
// Spot-check every entry in the spec table. Audio entries
|
||||
// come back bare (no channels/rate set) so codec_hint is the
|
||||
// codec name alone.
|
||||
let cases: &[(u8, &str)] = &[
|
||||
(0x02, "MPEG-2"),
|
||||
(0x1B, "H.264"),
|
||||
(0x24, "HEVC"),
|
||||
(0x80, "LPCM"),
|
||||
(0x81, "AC-3"),
|
||||
(0x82, "DTS"),
|
||||
(0x83, "TrueHD"),
|
||||
(0x84, "AC-3+"),
|
||||
(0x85, "DTS-HD"),
|
||||
(0x86, "DTS-HD MA"),
|
||||
(0x90, "PG"),
|
||||
(0x91, "IG"),
|
||||
(0xA1, "AC-3+ Secondary"),
|
||||
(0xA2, "DTS-HD Secondary"),
|
||||
];
|
||||
for (ct, expected) in cases {
|
||||
assert_eq!(
|
||||
codec_name(*ct),
|
||||
*expected,
|
||||
"coding_type 0x{:02X} should map to {}",
|
||||
ct,
|
||||
expected
|
||||
);
|
||||
}
|
||||
// Unknown bytes return empty.
|
||||
assert_eq!(codec_name(0x00), "");
|
||||
assert_eq!(codec_name(0xFF), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_format_appends_channel_layout() {
|
||||
let mono = audio_entry(1, 0x83, 1, 1, "eng");
|
||||
let stereo = audio_entry(2, 0x83, 3, 1, "eng");
|
||||
let surround_51 = audio_entry(3, 0x83, 6, 1, "eng");
|
||||
let surround_71 = audio_entry(4, 0x83, 12, 1, "eng");
|
||||
let unknown = audio_entry(5, 0x83, 0, 1, "eng");
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &mono),
|
||||
"TrueHD mono"
|
||||
);
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &stereo),
|
||||
"TrueHD 2.0"
|
||||
);
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &surround_51),
|
||||
"TrueHD 5.1"
|
||||
);
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &surround_71),
|
||||
"TrueHD 7.1"
|
||||
);
|
||||
assert_eq!(build_codec_hint(StreamLabelType::Audio, &unknown), "TrueHD");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_rate_only_shows_above_48k() {
|
||||
// 48 kHz (rate=1) is the universal default → not surfaced.
|
||||
let r48 = audio_entry(1, 0x83, 6, 1, "eng");
|
||||
// 96 kHz (rate=4) → surfaced.
|
||||
let r96 = audio_entry(2, 0x83, 6, 4, "eng");
|
||||
// 192 kHz (rate=5) → surfaced.
|
||||
let r192 = audio_entry(3, 0x83, 6, 5, "eng");
|
||||
assert_eq!(build_codec_hint(StreamLabelType::Audio, &r48), "TrueHD 5.1");
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &r96),
|
||||
"TrueHD 5.1 96kHz"
|
||||
);
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &r192),
|
||||
"TrueHD 5.1 192kHz"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_iso_code_passes_through_without_display_name() {
|
||||
// Made-up code: keep the raw lowercase code as `language`,
|
||||
// but `name` is empty because we don't know it.
|
||||
let pl = playlist_with(vec![audio_entry(0x1100, 0x83, 6, 1, "xyz")]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].language, "xyz");
|
||||
assert_eq!(labels[0].name, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ig_and_dv_streams_are_skipped() {
|
||||
// stream_type 4 = IG, 7 = DV EL — both must not surface.
|
||||
let mut ig = pg_entry(0x1400, "eng");
|
||||
ig.stream_type = 4;
|
||||
let mut dv = audio_entry(0x1011, 0x24, 0, 0, "");
|
||||
dv.stream_type = 7;
|
||||
let pl = playlist_with(vec![ig, dv]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert!(labels.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secondary_audio_becomes_audio_label() {
|
||||
// stream_type 5 = secondary audio. The conversion should
|
||||
// still produce an Audio label (the registry's apply path
|
||||
// can ignore secondary if it wants — this module just
|
||||
// surfaces what's there).
|
||||
let mut sec = audio_entry(0x1A00, 0x83, 3, 1, "eng");
|
||||
sec.stream_type = 5;
|
||||
sec.secondary = true;
|
||||
let pl = playlist_with(vec![sec]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(labels[0].codec_hint, "TrueHD 2.0");
|
||||
}
|
||||
}
|
||||
+20
-31
@@ -12,15 +12,15 @@
|
||||
//! sub_com1_idx="23,24,25" />
|
||||
//! ```
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, xml};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "playlists.xml")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let data = super::read_jar_file(reader, udf, "playlists.xml")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
@@ -30,8 +30,8 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
let mut labels = Vec::new();
|
||||
|
||||
// Parse audio streams
|
||||
if let Some(aud) = extract_attr(&feature, "aud") {
|
||||
let com_idx = extract_attr(&feature, "aud_com1_idx").and_then(|s| s.parse::<usize>().ok());
|
||||
if let Some(aud) = xml::attr(&feature, "aud") {
|
||||
let com_idx = xml::attr(&feature, "aud_com1_idx").and_then(|s| s.parse::<usize>().ok());
|
||||
|
||||
for (i, lang) in aud.split(',').enumerate() {
|
||||
let lang = lang.trim();
|
||||
@@ -57,12 +57,12 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
}
|
||||
|
||||
// Parse subtitle streams
|
||||
if let Some(sub) = extract_attr(&feature, "sub") {
|
||||
let forced: Vec<bool> = extract_attr(&feature, "forced_sub")
|
||||
if let Some(sub) = xml::attr(&feature, "sub") {
|
||||
let forced: Vec<bool> = xml::attr(&feature, "forced_sub")
|
||||
.map(|s| s.split(',').map(|f| f.trim() == "1").collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
let com_indices: Vec<usize> = extract_attr(&feature, "sub_com1_idx")
|
||||
let com_indices: Vec<usize> = xml::attr(&feature, "sub_com1_idx")
|
||||
.map(|s| s.split(',').filter_map(|i| i.trim().parse().ok()).collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
@@ -100,32 +100,29 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(labels)
|
||||
// High confidence: paramount's playlists.xml is fully structured
|
||||
// and we extract every documented field.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
|
||||
/// Find the feature playlist element (the one with the most audio tracks).
|
||||
fn find_feature_playlist(xml: &str) -> Option<String> {
|
||||
fn find_feature_playlist(text: &str) -> Option<String> {
|
||||
let mut best: Option<String> = None;
|
||||
let mut best_aud_count = 0;
|
||||
let mut from = 0;
|
||||
|
||||
let mut pos = 0;
|
||||
while let Some(start) = xml[pos..].find("<playlist ") {
|
||||
let abs_start = pos + start;
|
||||
let end = match xml[abs_start..].find("/>") {
|
||||
Some(p) => abs_start + p + 2,
|
||||
None => break,
|
||||
};
|
||||
let element = &xml[abs_start..end];
|
||||
while let Some((start, end)) = xml::find_element(text, "playlist", from) {
|
||||
let element = &text[start..end];
|
||||
|
||||
// Prefer name="Feature" explicitly
|
||||
if let Some(name) = extract_attr(element, "name") {
|
||||
// Prefer name="Feature" explicitly.
|
||||
if let Some(name) = xml::attr(element, "name") {
|
||||
if name.eq_ignore_ascii_case("Feature") {
|
||||
return Some(element.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise pick the one with the most audio streams
|
||||
if let Some(aud) = extract_attr(element, "aud") {
|
||||
// Otherwise pick the one with the most audio streams.
|
||||
if let Some(aud) = xml::attr(element, "aud") {
|
||||
let count = aud.split(',').count();
|
||||
if count > best_aud_count {
|
||||
best_aud_count = count;
|
||||
@@ -133,15 +130,7 @@ fn find_feature_playlist(xml: &str) -> Option<String> {
|
||||
}
|
||||
}
|
||||
|
||||
pos = end;
|
||||
from = end;
|
||||
}
|
||||
best
|
||||
}
|
||||
|
||||
/// Extract an XML attribute value from an element string.
|
||||
fn extract_attr(element: &str, name: &str) -> Option<String> {
|
||||
let needle = format!("{name}=\"");
|
||||
let start = element.find(&needle)? + needle.len();
|
||||
let end = element[start..].find('"')? + start;
|
||||
Some(element[start..end].to_string())
|
||||
}
|
||||
|
||||
+102
-23
@@ -5,9 +5,13 @@
|
||||
//!
|
||||
//! Token format: `{lang}_{codec?}_{purpose?}_{region?}_`
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType, vocab};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{
|
||||
Confidence, LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, text,
|
||||
vocab,
|
||||
};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
/// Known audio codec tokens
|
||||
const AUDIO_CODECS: &[&str] = &["MLP", "AC3", "DTS", "DDL", "WAV", "AC"];
|
||||
@@ -20,9 +24,18 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "bluray_project.bin")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let data = super::read_jar_file(reader, udf, "bluray_project.bin")?;
|
||||
let strings = extract_strings(&data);
|
||||
// min_len=4 matches the prior local extract_strings impl. The token
|
||||
// grammar is `{lang3}_{codec?}_{purpose?}_{region?}_` so the
|
||||
// shortest meaningful run is 4 chars (lang + underscore).
|
||||
let strings = text::extract_ascii_strings(&data, 4);
|
||||
|
||||
// Tracked across all parse_token calls in this run: did any stream
|
||||
// hit an unrecognized token component (skip-unknown path)? If yes
|
||||
// we downgrade confidence to Medium — the labels are still valid
|
||||
// but the corpus surfaced something we don't catalogue.
|
||||
let saw_unknown = AtomicBool::new(false);
|
||||
|
||||
let mut labels = Vec::new();
|
||||
let mut in_feature = false;
|
||||
@@ -50,7 +63,7 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(label) = parse_token(s) {
|
||||
if let Some(label) = parse_token_inner(s, Some(&saw_unknown)) {
|
||||
match label.stream_type {
|
||||
StreamLabelType::Audio => {
|
||||
audio_num += 1;
|
||||
@@ -73,10 +86,15 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(labels)
|
||||
let confidence = if saw_unknown.load(Ordering::Relaxed) {
|
||||
Confidence::Medium
|
||||
} else {
|
||||
Confidence::High
|
||||
};
|
||||
Some(ParseResult { labels, confidence })
|
||||
}
|
||||
|
||||
fn parse_token(s: &str) -> Option<StreamLabel> {
|
||||
fn parse_token_inner(s: &str, saw_unknown: Option<&AtomicBool>) -> Option<StreamLabel> {
|
||||
let clean = s.trim().trim_start_matches('\t').trim_end_matches('_');
|
||||
let parts: Vec<&str> = clean.split('_').collect();
|
||||
if parts.len() < 2 {
|
||||
@@ -127,7 +145,18 @@ fn parse_token(s: &str) -> Option<StreamLabel> {
|
||||
} else if part.starts_with("PGStream") {
|
||||
is_subtitle = true;
|
||||
} else {
|
||||
return None;
|
||||
// Unknown token component — skip this single part rather
|
||||
// than discarding the entire stream record. Pre-refactor
|
||||
// behavior was `return None` here, which silently dropped
|
||||
// any stream containing a single uncatalogued token (e.g.
|
||||
// a new codec ID or framework variant). Better to surface
|
||||
// what we know than discard a whole stream over one part,
|
||||
// but flag the parse as Medium-confidence so callers know
|
||||
// some data was elided.
|
||||
tracing::debug!(part = %part, "pixelogic: unrecognized token component, skipping");
|
||||
if let Some(flag) = saw_unknown {
|
||||
flag.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,22 +182,72 @@ fn parse_token(s: &str) -> Option<StreamLabel> {
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_strings(data: &[u8]) -> Vec<String> {
|
||||
let mut strings = Vec::new();
|
||||
let mut current = String::new();
|
||||
// extract_strings removed — replaced by super::text::extract_ascii_strings(data, 4).
|
||||
|
||||
for &b in data {
|
||||
if (0x20..0x7f).contains(&b) {
|
||||
current.push(b as char);
|
||||
} else {
|
||||
if current.len() > 3 {
|
||||
strings.push(current.clone());
|
||||
}
|
||||
current.clear();
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_token_basic_audio() {
|
||||
let l = parse_token_inner("eng_MLP_", None).unwrap();
|
||||
assert_eq!(l.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.codec_hint, "TrueHD");
|
||||
assert_eq!(l.purpose, LabelPurpose::Normal);
|
||||
}
|
||||
if current.len() > 3 {
|
||||
strings.push(current);
|
||||
|
||||
#[test]
|
||||
fn parse_token_basic_subtitle_sdh() {
|
||||
let l = parse_token_inner("eng_SDH_", None).unwrap();
|
||||
assert_eq!(l.stream_type, StreamLabelType::Subtitle);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.qualifier, LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_commentary() {
|
||||
let l = parse_token_inner("eng_MLP_ACOM_", None).unwrap();
|
||||
assert_eq!(l.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(l.purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_descriptive() {
|
||||
let l = parse_token_inner("eng_AC3_ADES_", None).unwrap();
|
||||
assert_eq!(l.purpose, LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_with_region() {
|
||||
let l = parse_token_inner("eng_MLP_US_", None).unwrap();
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.variant, "US");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_unknown_component_does_not_kill_stream() {
|
||||
// Regression: pre-refactor, an unrecognized token part returned
|
||||
// None for the whole stream, silently dropping it. New
|
||||
// behavior: skip the unknown part, surface what we know.
|
||||
let l = parse_token_inner("eng_MLP_FUTUREFLAG_FOR_", None).unwrap();
|
||||
assert_eq!(l.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.codec_hint, "TrueHD");
|
||||
assert_eq!(l.qualifier, LabelQualifier::Forced);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_no_audio_or_subtitle_signal_returns_none() {
|
||||
// A token that has only a language and an unknown part with
|
||||
// no audio/subtitle classifier should still return None —
|
||||
// there's no way to file it as a stream.
|
||||
assert!(parse_token_inner("eng_UNKNOWN_", None).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_rejects_non_lang_prefix() {
|
||||
assert!(parse_token_inner("XX_MLP_", None).is_none());
|
||||
assert!(parse_token_inner("ENG_MLP_", None).is_none()); // uppercase not accepted as ISO 639-2
|
||||
}
|
||||
strings
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
//! Text-extraction helpers used by parsers that scan binary blobs for
|
||||
//! embedded label strings.
|
||||
//!
|
||||
//! Promoted from two near-duplicate implementations:
|
||||
//! - `pixelogic::extract_strings` (`bluray_project.bin`, min_len=4)
|
||||
//! - `dbp::extract_printable` (`.class` files in jars, min_len=5)
|
||||
//!
|
||||
//! Single implementation, threshold passed in. Callers that have a
|
||||
//! more structured parse path (e.g. `class_reader` for .class) should
|
||||
//! prefer that — this helper is for genuinely unstructured input.
|
||||
|
||||
/// Walk `data`, emit every maximal run of printable-ASCII bytes
|
||||
/// (`0x20..=0x7E`) whose length is at least `min_len`.
|
||||
///
|
||||
/// Non-printable bytes (including `\t`, `\n`, NUL) terminate the
|
||||
/// current run. Output strings are guaranteed valid UTF-8 (they're
|
||||
/// pure 7-bit ASCII). Strings shorter than `min_len` are dropped.
|
||||
pub fn extract_ascii_strings(data: &[u8], min_len: usize) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
let mut current = String::new();
|
||||
for &b in data {
|
||||
if (0x20..=0x7E).contains(&b) {
|
||||
current.push(b as char);
|
||||
} else if current.len() >= min_len {
|
||||
out.push(std::mem::take(&mut current));
|
||||
} else {
|
||||
current.clear();
|
||||
}
|
||||
}
|
||||
if current.len() >= min_len {
|
||||
out.push(current);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn extracts_simple_runs() {
|
||||
let got = extract_ascii_strings(b"hello\0world\0", 3);
|
||||
assert_eq!(got, vec!["hello", "world"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn applies_minimum_length() {
|
||||
let got = extract_ascii_strings(b"hi\0ok\0longer\0", 5);
|
||||
assert_eq!(got, vec!["longer"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn treats_tab_and_newline_as_separators() {
|
||||
// \t (0x09) and \n (0x0A) are below 0x20, so they break runs.
|
||||
let got = extract_ascii_strings(b"alpha\tbeta\ngamma", 3);
|
||||
assert_eq!(got, vec!["alpha", "beta", "gamma"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handles_trailing_run_without_terminator() {
|
||||
// Run at the very end of the buffer should still be emitted.
|
||||
let got = extract_ascii_strings(b"prefix\0tail", 3);
|
||||
assert_eq!(got, vec!["prefix", "tail"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_high_bit_bytes() {
|
||||
// 0x80+ is non-printable per this helper's definition.
|
||||
let mut buf = b"good".to_vec();
|
||||
buf.push(0xC3);
|
||||
buf.push(0xA9);
|
||||
buf.extend_from_slice(b"more");
|
||||
let got = extract_ascii_strings(&buf, 3);
|
||||
assert_eq!(got, vec!["good", "more"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_returns_empty() {
|
||||
let got = extract_ascii_strings(&[], 1);
|
||||
assert!(got.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn min_len_zero_emits_singletons() {
|
||||
// Pathological but well-defined.
|
||||
let got = extract_ascii_strings(b"a\0b", 0);
|
||||
assert_eq!(got, vec!["a", "b"]);
|
||||
}
|
||||
}
|
||||
+432
-9
@@ -1,19 +1,40 @@
|
||||
//! Shared label vocabulary — values we are 100% confident about.
|
||||
//! Shared label vocabulary — canonical mappings used by ≥1 label parser.
|
||||
//!
|
||||
//! Labels come from BD-J authoring tool files (bluray_project.bin,
|
||||
//! playlists.xml, menu_base.prop, etc.) — NOT from BD spec fields.
|
||||
//! This is NOT for MPLS/CLPI/STN data. Those follow the BD spec directly.
|
||||
//! playlists.xml, menu_base.prop, .class string pools, etc.) — NOT
|
||||
//! from BD spec fields. This module is the central, regression-tested
|
||||
//! source of truth for:
|
||||
//!
|
||||
//! Rules:
|
||||
//! - Only map values we are 100% certain about (published codec names).
|
||||
//! - Unknown codes (csp, eda, cf, etc.) pass through raw from disc.
|
||||
//! - The app/CLI handles display text, not the lib.
|
||||
//! - Codec brand name aliases (`MLP` → `TrueHD`).
|
||||
//! - English / multi-word language name → ISO 639-2 code.
|
||||
//! - English text → [`LabelPurpose`] (Commentary / Descriptive / etc.).
|
||||
//! - English text → [`LabelQualifier`] (SDH / Forced / Descriptive Service).
|
||||
//!
|
||||
//! Rules of engagement (carried over from
|
||||
//! `(internal)/memory/feedback_label_data_rules.md`):
|
||||
//!
|
||||
//! 1. Only map values we are 100% certain about — published codec
|
||||
//! names, well-known ISO 639-2 mappings, vendor-documented purpose
|
||||
//! keywords.
|
||||
//! 2. Unknown codes / unrecognized phrases pass through raw or return
|
||||
//! `None`. We never guess.
|
||||
//! 3. Matching is case-insensitive and word-boundary-aware where
|
||||
//! relevant (so "Commenter" doesn't match "commentary"). Anchoring
|
||||
//! on whole tokens is the responsibility of this module — callers
|
||||
//! pass raw text, we handle it.
|
||||
//!
|
||||
//! This module is NOT for BD spec STN codec IDs; those decode in
|
||||
//! `mpls.rs` separately.
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier};
|
||||
|
||||
// ── Codec aliases ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Map a codec identifier found in label data to its display name.
|
||||
///
|
||||
/// These are well-known codec identifiers used across multiple BD-J
|
||||
/// authoring tools. NOT BD spec STN codec IDs — those are decoded
|
||||
/// separately in mpls.rs.
|
||||
/// authoring tools. Unknown codes pass through unchanged so callers
|
||||
/// can still surface vendor-specific tokens we haven't catalogued.
|
||||
pub fn codec(code: &str) -> &str {
|
||||
match code {
|
||||
"MLP" => "TrueHD",
|
||||
@@ -25,3 +46,405 @@ pub fn codec(code: &str) -> &str {
|
||||
_ => code,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Language: English / multi-word names → ISO 639-2 ─────────────────────────
|
||||
|
||||
/// Result of [`lang`] — ISO code + human-readable regional variant.
|
||||
///
|
||||
/// `code` is ISO 639-2 (always 3 lowercase letters).
|
||||
/// `variant` is the regional dialect as a human-readable English word
|
||||
/// (`"Brazilian"`, `"Castilian"`, `"Canadian"`, `"Simplified"`, ...)
|
||||
/// or `""` when the input names just a bare language without
|
||||
/// dialect ("Spanish" → variant=""). The variant matches the
|
||||
/// convention pixelogic / ctrm / criterion already use for their
|
||||
/// `StreamLabel::variant` field: a short display token the UI can
|
||||
/// surface verbatim.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct LangInfo {
|
||||
pub code: &'static str,
|
||||
pub variant: &'static str,
|
||||
}
|
||||
|
||||
/// Map a free-form language label fragment to an ISO 639-2 code AND
|
||||
/// (where applicable) its regional variant.
|
||||
///
|
||||
/// Handles both bare English names ("English", "Spanish") and the
|
||||
/// multi-word vendor variants we've seen in the corpus ("Brazilian
|
||||
/// Portuguese", "Castilian Spanish", "Canadian French"). Match is
|
||||
/// case-insensitive; longer compound phrases win over their bare
|
||||
/// counterparts (so "Brazilian Portuguese" returns
|
||||
/// `LangInfo { code: "por", variant: "Brazilian" }`, not consumed by
|
||||
/// the bare "Portuguese" entry).
|
||||
///
|
||||
/// Bare-name matches return `variant: ""`.
|
||||
///
|
||||
/// Returns `None` for unrecognized input — callers decide whether to
|
||||
/// fall back to MPLS spec codes, pass through raw, or drop the stream.
|
||||
/// Never guesses.
|
||||
///
|
||||
/// Why the variant: the prior `lang() -> Option<&str>` shape silently
|
||||
/// dropped regional dialect info. "Brazilian Portuguese 5.1" became
|
||||
/// `language="por", variant=""` — UI displayed plain "Portuguese"
|
||||
/// even though the disc had explicitly labeled this stream Brazilian.
|
||||
/// Capturing the variant here parallels how pixelogic and ctrm
|
||||
/// populate `StreamLabel::variant` from their own region tables.
|
||||
pub fn lang(text: &str) -> Option<LangInfo> {
|
||||
let lower = text.to_lowercase();
|
||||
// Multi-word compounds first — longest-match wins.
|
||||
for (needle, code, variant) in COMPOUND_LANGS {
|
||||
if lower.contains(needle) {
|
||||
return Some(LangInfo { code, variant });
|
||||
}
|
||||
}
|
||||
// Bare names: word-boundary match (avoid "english" inside "englishman"
|
||||
// or any other accidental substring).
|
||||
for (needle, code) in BARE_LANGS {
|
||||
if has_word(&lower, needle) {
|
||||
return Some(LangInfo { code, variant: "" });
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
const COMPOUND_LANGS: &[(&str, &str, &str)] = &[
|
||||
("brazilian portuguese", "por", "Brazilian"),
|
||||
("euro portuguese", "por", "European"),
|
||||
("european portuguese", "por", "European"),
|
||||
("castilian spanish", "spa", "Castilian"),
|
||||
("latin american spanish", "spa", "Latin American"),
|
||||
("latin spanish", "spa", "Latin American"),
|
||||
("canadian french", "fra", "Canadian"),
|
||||
("parisian french", "fra", "Parisian"),
|
||||
("australian english", "eng", "Australian"),
|
||||
("austrailian english", "eng", "Australian"), // disc-corpus typo, keep matching
|
||||
("british english", "eng", "British"),
|
||||
("simplified chinese", "zho", "Simplified"),
|
||||
("traditional chinese", "zho", "Traditional"),
|
||||
("mandarin chinese", "zho", "Mandarin"),
|
||||
("cantonese chinese", "zho", "Cantonese"),
|
||||
];
|
||||
|
||||
const BARE_LANGS: &[(&str, &str)] = &[
|
||||
("english", "eng"),
|
||||
("spanish", "spa"),
|
||||
("french", "fra"),
|
||||
("german", "deu"),
|
||||
("italian", "ita"),
|
||||
("japanese", "jpn"),
|
||||
("chinese", "zho"),
|
||||
("mandarin", "zho"),
|
||||
("cantonese", "zho"),
|
||||
("portuguese", "por"),
|
||||
("polish", "pol"),
|
||||
("czech", "ces"),
|
||||
("hungarian", "hun"),
|
||||
("dutch", "nld"),
|
||||
("korean", "kor"),
|
||||
("arabic", "ara"),
|
||||
("hindi", "hin"),
|
||||
("turkish", "tur"),
|
||||
("thai", "tha"),
|
||||
("swedish", "swe"),
|
||||
("norwegian", "nor"),
|
||||
("danish", "dan"),
|
||||
("finnish", "fin"),
|
||||
("hebrew", "heb"),
|
||||
("russian", "rus"),
|
||||
("greek", "ell"),
|
||||
("vietnamese", "vie"),
|
||||
("indonesian", "ind"),
|
||||
("malay", "msa"),
|
||||
("ukrainian", "ukr"),
|
||||
("romanian", "ron"),
|
||||
("bulgarian", "bul"),
|
||||
("croatian", "hrv"),
|
||||
("serbian", "srp"),
|
||||
("slovak", "slk"),
|
||||
("slovenian", "slv"),
|
||||
("estonian", "est"),
|
||||
("latvian", "lav"),
|
||||
("lithuanian", "lit"),
|
||||
("icelandic", "isl"),
|
||||
("basque", "eus"),
|
||||
("catalan", "cat"),
|
||||
("galician", "glg"),
|
||||
];
|
||||
|
||||
// ── Purpose ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Classify a free-form English label string into a [`LabelPurpose`].
|
||||
///
|
||||
/// Recognized keywords (case-insensitive, word-boundary matched):
|
||||
/// - "commentary", "director's commentary" → `Commentary`
|
||||
/// - "descriptive", "description", "audio description", "described" → `Descriptive`
|
||||
/// - "score", "music only" → `Score`
|
||||
/// - "ime" (alternate music for closing themes etc.) → `Ime`
|
||||
/// - anything else → `Normal`
|
||||
///
|
||||
/// Word-boundary matching means "Commentary track" matches but
|
||||
/// "Commenter Pro audio" does not.
|
||||
pub fn purpose(text: &str) -> LabelPurpose {
|
||||
let lower = text.to_lowercase();
|
||||
// Multi-word compounds first — they're more specific.
|
||||
if lower.contains("audio description") || lower.contains("descriptive service") {
|
||||
return LabelPurpose::Descriptive;
|
||||
}
|
||||
if lower.contains("music only") {
|
||||
return LabelPurpose::Score;
|
||||
}
|
||||
if has_word(&lower, "commentary") {
|
||||
return LabelPurpose::Commentary;
|
||||
}
|
||||
if has_word(&lower, "descriptive")
|
||||
|| has_word(&lower, "description")
|
||||
|| has_word(&lower, "described")
|
||||
{
|
||||
return LabelPurpose::Descriptive;
|
||||
}
|
||||
if has_word(&lower, "score") {
|
||||
return LabelPurpose::Score;
|
||||
}
|
||||
if has_word(&lower, "ime") {
|
||||
return LabelPurpose::Ime;
|
||||
}
|
||||
LabelPurpose::Normal
|
||||
}
|
||||
|
||||
// ── Qualifier ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Classify a free-form English label string into a [`LabelQualifier`].
|
||||
///
|
||||
/// Recognized keywords (case-insensitive, word-boundary matched):
|
||||
/// - "sdh", "captions" → `Sdh`
|
||||
/// - "forced", "forced narrative" → `Forced`
|
||||
/// - "rnib", "descriptive service" → `DescriptiveService`
|
||||
/// - anything else → `None`
|
||||
///
|
||||
/// SDH (Subtitles for the Deaf and Hard of hearing) wins over Forced
|
||||
/// when both keywords are present, because an SDH track is its own
|
||||
/// stream regardless of whether the player flags it as "forced".
|
||||
pub fn qualifier(text: &str) -> LabelQualifier {
|
||||
let lower = text.to_lowercase();
|
||||
if has_word(&lower, "sdh") || has_word(&lower, "captions") {
|
||||
return LabelQualifier::Sdh;
|
||||
}
|
||||
if lower.contains("descriptive service") || has_word(&lower, "rnib") {
|
||||
return LabelQualifier::DescriptiveService;
|
||||
}
|
||||
if has_word(&lower, "forced") {
|
||||
return LabelQualifier::Forced;
|
||||
}
|
||||
LabelQualifier::None
|
||||
}
|
||||
|
||||
// ── Internal: word-boundary matching ────────────────────────────────────────
|
||||
|
||||
/// True if `needle` appears in `haystack` surrounded by non-alphanumeric
|
||||
/// boundaries (or string ends). `haystack` is assumed lowercase already.
|
||||
///
|
||||
/// This is the load-bearing primitive for `lang` / `purpose` /
|
||||
/// `qualifier`: bare-token matchers MUST use it, otherwise we match
|
||||
/// "english" inside "englishman" and "sdh" inside "lambdash". The
|
||||
/// existing parsers used `.contains()` and got lucky on the corpus;
|
||||
/// vocab guarantees the boundary.
|
||||
fn has_word(haystack: &str, needle: &str) -> bool {
|
||||
if needle.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let bytes = haystack.as_bytes();
|
||||
let nb = needle.as_bytes();
|
||||
let mut i = 0;
|
||||
while i + nb.len() <= bytes.len() {
|
||||
if &bytes[i..i + nb.len()] == nb {
|
||||
let before = if i == 0 { None } else { Some(bytes[i - 1]) };
|
||||
let after = bytes.get(i + nb.len()).copied();
|
||||
let bound = |c: Option<u8>| match c {
|
||||
None => true,
|
||||
Some(b) => !b.is_ascii_alphanumeric(),
|
||||
};
|
||||
if bound(before) && bound(after) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn codec_known_aliases() {
|
||||
assert_eq!(codec("MLP"), "TrueHD");
|
||||
assert_eq!(codec("AC3"), "Dolby Digital");
|
||||
assert_eq!(codec("AC"), "Dolby Digital");
|
||||
assert_eq!(codec("DDL"), "Dolby Digital Plus");
|
||||
assert_eq!(codec("atmos"), "Dolby Atmos");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_unknown_passes_through() {
|
||||
assert_eq!(codec("FX9"), "FX9");
|
||||
assert_eq!(codec(""), "");
|
||||
}
|
||||
|
||||
fn li(code: &'static str, variant: &'static str) -> LangInfo {
|
||||
LangInfo { code, variant }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_bare_names_have_empty_variant() {
|
||||
assert_eq!(lang("English"), Some(li("eng", "")));
|
||||
assert_eq!(lang("english"), Some(li("eng", "")));
|
||||
assert_eq!(lang("Spanish 5.1 Dolby Digital"), Some(li("spa", "")));
|
||||
assert_eq!(lang("japanese"), Some(li("jpn", "")));
|
||||
assert_eq!(lang("Italian"), Some(li("ita", "")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_compounds_carry_variant() {
|
||||
assert_eq!(
|
||||
lang("Brazilian Portuguese 5.1"),
|
||||
Some(li("por", "Brazilian"))
|
||||
);
|
||||
assert_eq!(lang("Castilian Spanish"), Some(li("spa", "Castilian")));
|
||||
assert_eq!(
|
||||
lang("Canadian French Dolby Digital"),
|
||||
Some(li("fra", "Canadian"))
|
||||
);
|
||||
assert_eq!(
|
||||
lang("Latin American Spanish"),
|
||||
Some(li("spa", "Latin American"))
|
||||
);
|
||||
assert_eq!(lang("Simplified Chinese"), Some(li("zho", "Simplified")));
|
||||
assert_eq!(lang("British English"), Some(li("eng", "British")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_compounds_win_over_bare() {
|
||||
// Brazilian Portuguese must map to (por, Brazilian) via the
|
||||
// compound rule, not be intercepted by bare "portuguese"
|
||||
// (which would yield (por, "") and lose the variant).
|
||||
assert_eq!(lang("Brazilian Portuguese").unwrap().variant, "Brazilian");
|
||||
assert_eq!(lang("Canadian French").unwrap().variant, "Canadian");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_unknown_returns_none() {
|
||||
assert_eq!(lang("Klingon Dolby Atmos"), None);
|
||||
assert_eq!(lang(""), None);
|
||||
assert_eq!(lang("eng"), None); // 3-letter codes are not English names
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_word_boundary_avoids_substring_false_positive() {
|
||||
// No false positive — "engineering" must NOT match "english".
|
||||
assert_eq!(lang("Audio Engineering Demo"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_recognizes_commentary() {
|
||||
assert_eq!(purpose("English Commentary"), LabelPurpose::Commentary);
|
||||
assert_eq!(purpose("Director's Commentary"), LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_recognizes_descriptive() {
|
||||
assert_eq!(
|
||||
purpose("English Descriptive Audio"),
|
||||
LabelPurpose::Descriptive
|
||||
);
|
||||
assert_eq!(purpose("Audio Description"), LabelPurpose::Descriptive);
|
||||
assert_eq!(purpose("Described Video"), LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_descriptive_service_routes_to_descriptive() {
|
||||
// "Descriptive Service" is qualifier territory but the purpose
|
||||
// implication is Descriptive — vocab::purpose treats it as such.
|
||||
assert_eq!(
|
||||
purpose("English Descriptive Service"),
|
||||
LabelPurpose::Descriptive
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_word_boundary_avoids_commenter_false_positive() {
|
||||
// "Commenter Pro audio" does NOT match "commentary" — the
|
||||
// existing dbp/ctrm hand-rolls would have. Vocab is stricter.
|
||||
assert_eq!(purpose("Commenter Pro audio track"), LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_recognizes_score() {
|
||||
assert_eq!(purpose("Music Only"), LabelPurpose::Score);
|
||||
assert_eq!(purpose("Isolated Score"), LabelPurpose::Score);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_unknown_is_normal() {
|
||||
assert_eq!(purpose("English Dolby Atmos"), LabelPurpose::Normal);
|
||||
assert_eq!(purpose(""), LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_recognizes_sdh() {
|
||||
assert_eq!(qualifier("English SDH"), LabelQualifier::Sdh);
|
||||
assert_eq!(qualifier("English Captions"), LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_recognizes_forced() {
|
||||
assert_eq!(qualifier("English Forced"), LabelQualifier::Forced);
|
||||
assert_eq!(qualifier("Forced Narrative"), LabelQualifier::Forced);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_recognizes_descriptive_service() {
|
||||
assert_eq!(
|
||||
qualifier("English RNIB"),
|
||||
LabelQualifier::DescriptiveService
|
||||
);
|
||||
assert_eq!(
|
||||
qualifier("English Descriptive Service"),
|
||||
LabelQualifier::DescriptiveService
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_sdh_wins_over_forced_when_both_present() {
|
||||
// SDH track is its own stream regardless of forced flag.
|
||||
assert_eq!(qualifier("English Forced SDH"), LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_unknown_is_none() {
|
||||
assert_eq!(qualifier("English"), LabelQualifier::None);
|
||||
assert_eq!(qualifier(""), LabelQualifier::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_word_basic() {
|
||||
assert!(has_word("english forced", "english"));
|
||||
assert!(has_word("english forced", "forced"));
|
||||
assert!(has_word("english", "english"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_word_rejects_substring() {
|
||||
assert!(!has_word("engineering", "english"));
|
||||
assert!(!has_word("englishman", "english"));
|
||||
assert!(!has_word("aenglish", "english"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_word_punctuation_boundary() {
|
||||
// "(SDH)" is a valid boundary — parentheses count as non-alphanum.
|
||||
assert!(has_word("english (sdh)", "sdh"));
|
||||
assert!(has_word("commentary,extra,info", "commentary"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,452 @@
|
||||
//! Tolerant XML scraping helpers — promoted from two near-duplicate
|
||||
//! hand-rolls in `paramount.rs` (attribute extraction) and `criterion.rs`
|
||||
//! (tag-text extraction).
|
||||
//!
|
||||
//! These are NOT a full XML parser. They handle the subset of XML the
|
||||
//! BD-J authoring tools we've seen actually emit: ASCII tag/attr
|
||||
//! names, no entity references inside label strings, optional XML
|
||||
//! namespaces. Hardening goals over the prior `find("<tag>")` /
|
||||
//! `find(r#"name=""#)` matchers:
|
||||
//!
|
||||
//! 1. **Case-insensitive** tag and attribute names — vendors casing
|
||||
//! is inconsistent across authoring-tool revisions.
|
||||
//! 2. **Namespace-aware** — strip an optional `ns:` prefix so
|
||||
//! `<ns:playlist>` and `<playlist>` both match.
|
||||
//! 3. **Whitespace-tolerant** — multiple/tab/newline characters
|
||||
//! around `=` between attribute name and value; whitespace inside
|
||||
//! the opening tag.
|
||||
//! 4. **Quote-style tolerant** — both `"value"` and `'value'`.
|
||||
//! 5. **Self-closing tag handling** — `<tag />` and `<tag/>` both
|
||||
//! work; [`text`] returns `Some("")` for empty content.
|
||||
//!
|
||||
//! Out of scope (intentionally simple): XML entity decoding
|
||||
//! (`&`, `<`, etc.), CDATA sections, comments, processing
|
||||
//! instructions, DTD declarations. None of the BD-J authored disc
|
||||
//! data we've observed exercises any of those — labels are plain
|
||||
//! ASCII/Latin-1 in attribute values.
|
||||
|
||||
/// Extract the value of attribute `name` from one XML element
|
||||
/// fragment (e.g. `<playlist name="Feature" id="00222" />`).
|
||||
///
|
||||
/// Returns the raw attribute text (no entity decoding) or `None` if
|
||||
/// the attribute isn't present. Empty string for `name=""` is
|
||||
/// represented as `Some("")`.
|
||||
pub fn attr(element: &str, name: &str) -> Option<String> {
|
||||
let bytes = element.as_bytes();
|
||||
let name_lower = name.to_ascii_lowercase();
|
||||
let name_bytes = name_lower.as_bytes();
|
||||
let mut i = 0;
|
||||
while i + name_bytes.len() < bytes.len() {
|
||||
// Find the next position where `name=` could start. We need
|
||||
// a word boundary before the name (whitespace or `<` or `:`).
|
||||
if i > 0 && is_name_char(bytes[i - 1]) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if !slice_eq_ignore_case(&bytes[i..i + name_bytes.len()], name_bytes) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
let after_name = i + name_bytes.len();
|
||||
// The character immediately after the name must not be a
|
||||
// name-continuation (otherwise we matched a prefix like
|
||||
// `lang_id` when looking for `lang`).
|
||||
if after_name < bytes.len() && is_name_char(bytes[after_name]) {
|
||||
i = after_name;
|
||||
continue;
|
||||
}
|
||||
// Walk past whitespace, then `=`, then more whitespace, then
|
||||
// the opening quote.
|
||||
let mut j = after_name;
|
||||
while j < bytes.len() && is_ws(bytes[j]) {
|
||||
j += 1;
|
||||
}
|
||||
if j >= bytes.len() || bytes[j] != b'=' {
|
||||
i = j.max(i + 1);
|
||||
continue;
|
||||
}
|
||||
j += 1; // past '='
|
||||
while j < bytes.len() && is_ws(bytes[j]) {
|
||||
j += 1;
|
||||
}
|
||||
if j >= bytes.len() {
|
||||
return None;
|
||||
}
|
||||
let quote = bytes[j];
|
||||
if quote != b'"' && quote != b'\'' {
|
||||
// Unquoted attribute values aren't part of well-formed
|
||||
// XML (HTML5 allows them, XML doesn't). Skip.
|
||||
i = j;
|
||||
continue;
|
||||
}
|
||||
let value_start = j + 1;
|
||||
let close = bytes[value_start..].iter().position(|&b| b == quote)?;
|
||||
let value = &element[value_start..value_start + close];
|
||||
return Some(value.to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Extract the trimmed text content of the first occurrence of
|
||||
/// `<tag>...</tag>` in `xml`. Returns `None` if the tag isn't found
|
||||
/// or its opening tag is malformed.
|
||||
///
|
||||
/// Whitespace around the inner text is stripped. Self-closing
|
||||
/// `<tag />` yields `Some("")`. Nested same-name tags are NOT
|
||||
/// handled — the first close encountered wins (this matches the
|
||||
/// prior behavior in criterion.rs).
|
||||
pub fn text(xml: &str, tag: &str) -> Option<String> {
|
||||
let (open_end, body_start) = find_open_tag(xml, tag, 0)?;
|
||||
// Self-closing — already consumed in find_open_tag if `/>`.
|
||||
if open_end == body_start {
|
||||
// Means find_open_tag returned the same offset twice for
|
||||
// self-closing form. (Not currently the case in our impl,
|
||||
// but defensive.)
|
||||
return Some(String::new());
|
||||
}
|
||||
// For self-closing tags, body_start is past `/>` and we have no
|
||||
// content. Detect by checking the char at body_start - 1 was `/`.
|
||||
if body_start >= 2 && &xml[body_start - 2..body_start] == "/>" {
|
||||
return Some(String::new());
|
||||
}
|
||||
// Find the matching close tag. Case-insensitive + namespace-aware.
|
||||
let close_start = find_close_tag(xml, tag, body_start)?;
|
||||
Some(xml[body_start..close_start].trim().to_string())
|
||||
}
|
||||
|
||||
/// Locate the next `<tag>` opening AND its closing `</tag>` in
|
||||
/// `xml`, starting at byte offset `from`. Returns `(element_start,
|
||||
/// element_end)` — `element_start` is the `<` of the opening tag,
|
||||
/// `element_end` is one past the `>` of the closing tag. Useful for
|
||||
/// iterating over repeated elements like `<playlist>` blocks in
|
||||
/// `paramount`.
|
||||
///
|
||||
/// Self-closing elements return the same offset for body_end as the
|
||||
/// element_end (i.e. `element_end - element_start` includes only the
|
||||
/// `<tag .../>` text).
|
||||
pub fn find_element(xml: &str, tag: &str, from: usize) -> Option<(usize, usize)> {
|
||||
let bytes = xml.as_bytes();
|
||||
let mut i = from;
|
||||
while i < bytes.len() {
|
||||
if bytes[i] != b'<' {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
// Try matching tag name at i+1 (after `<`).
|
||||
let after_lt = i + 1;
|
||||
if !matches_tag_name_at(bytes, after_lt, tag) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
// Found an open tag at offset i. Walk to find the closing `>`
|
||||
// of the open tag itself.
|
||||
let mut j = after_lt;
|
||||
// Skip past the tag name (and optional namespace prefix).
|
||||
while j < bytes.len() && (is_name_char(bytes[j]) || bytes[j] == b':') {
|
||||
j += 1;
|
||||
}
|
||||
// Walk attributes — track quoting state.
|
||||
let mut self_closing = false;
|
||||
while j < bytes.len() {
|
||||
match bytes[j] {
|
||||
b'>' => {
|
||||
j += 1;
|
||||
break;
|
||||
}
|
||||
b'/' if j + 1 < bytes.len() && bytes[j + 1] == b'>' => {
|
||||
self_closing = true;
|
||||
j += 2;
|
||||
break;
|
||||
}
|
||||
b'"' | b'\'' => {
|
||||
let q = bytes[j];
|
||||
j += 1;
|
||||
while j < bytes.len() && bytes[j] != q {
|
||||
j += 1;
|
||||
}
|
||||
if j < bytes.len() {
|
||||
j += 1;
|
||||
}
|
||||
}
|
||||
_ => j += 1,
|
||||
}
|
||||
}
|
||||
if self_closing {
|
||||
return Some((i, j));
|
||||
}
|
||||
// Find matching close. Doesn't handle nested same-name; OK
|
||||
// for our authoring-tool subset.
|
||||
let close_start = find_close_tag(xml, tag, j)?;
|
||||
let close_end = find_byte(bytes, b'>', close_start)? + 1;
|
||||
return Some((i, close_end));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
// ── Internal helpers ───────────────────────────────────────────────────────
|
||||
|
||||
/// True if `bytes[start..]` opens a tag named `tag`, allowing an
|
||||
/// optional `ns:` namespace prefix. Comparison is case-insensitive.
|
||||
/// The character after the tag name must not be a name-continuation
|
||||
/// (so `<player>` doesn't match `<play>`).
|
||||
fn matches_tag_name_at(bytes: &[u8], start: usize, tag: &str) -> bool {
|
||||
let tag_lower = tag.to_ascii_lowercase();
|
||||
let tag_bytes = tag_lower.as_bytes();
|
||||
// Skip optional `prefix:` (one or more name chars + `:`).
|
||||
let mut name_start = start;
|
||||
let mut scan = start;
|
||||
while scan < bytes.len() && is_name_char(bytes[scan]) {
|
||||
scan += 1;
|
||||
}
|
||||
if scan < bytes.len() && bytes[scan] == b':' {
|
||||
name_start = scan + 1;
|
||||
}
|
||||
if name_start + tag_bytes.len() > bytes.len() {
|
||||
return false;
|
||||
}
|
||||
if !slice_eq_ignore_case(&bytes[name_start..name_start + tag_bytes.len()], tag_bytes) {
|
||||
return false;
|
||||
}
|
||||
// Boundary: char after the tag name must be `>`, `/`, whitespace.
|
||||
let after = name_start + tag_bytes.len();
|
||||
if after >= bytes.len() {
|
||||
return false;
|
||||
}
|
||||
matches!(bytes[after], b'>' | b'/' | b' ' | b'\t' | b'\n' | b'\r')
|
||||
}
|
||||
|
||||
/// Find the offset of the next `</tag>` (or `</ns:tag>`) in `xml`
|
||||
/// starting at `from`. Case-insensitive; returns the offset of the
|
||||
/// `<`. None if not found.
|
||||
fn find_close_tag(xml: &str, tag: &str, from: usize) -> Option<usize> {
|
||||
let bytes = xml.as_bytes();
|
||||
let mut i = from;
|
||||
while i + 2 < bytes.len() {
|
||||
if bytes[i] == b'<' && bytes[i + 1] == b'/' {
|
||||
// Check tag name (with optional namespace).
|
||||
if matches_tag_name_at(bytes, i + 2, tag) {
|
||||
return Some(i);
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Find the open tag of `<tag>` in `xml` starting at `from`. Returns
|
||||
/// `(after_open_lt, after_open_gt)` — the offsets are: just past
|
||||
/// the `<` of the open tag, and just past the `>` of the open tag.
|
||||
fn find_open_tag(xml: &str, tag: &str, from: usize) -> Option<(usize, usize)> {
|
||||
let bytes = xml.as_bytes();
|
||||
let (elem_start, _) = find_element(xml, tag, from)?;
|
||||
let after_lt = elem_start + 1;
|
||||
// Find the `>` that ends the open tag (handling quoted attrs).
|
||||
let mut j = after_lt;
|
||||
while j < bytes.len() {
|
||||
match bytes[j] {
|
||||
b'>' => return Some((after_lt, j + 1)),
|
||||
b'/' if j + 1 < bytes.len() && bytes[j + 1] == b'>' => {
|
||||
return Some((after_lt, j + 2));
|
||||
}
|
||||
b'"' | b'\'' => {
|
||||
let q = bytes[j];
|
||||
j += 1;
|
||||
while j < bytes.len() && bytes[j] != q {
|
||||
j += 1;
|
||||
}
|
||||
if j < bytes.len() {
|
||||
j += 1;
|
||||
}
|
||||
}
|
||||
_ => j += 1,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn find_byte(bytes: &[u8], target: u8, from: usize) -> Option<usize> {
|
||||
bytes[from..]
|
||||
.iter()
|
||||
.position(|&b| b == target)
|
||||
.map(|p| p + from)
|
||||
}
|
||||
|
||||
/// True if `c` can be part of an XML name token (rough). We accept
|
||||
/// alphanumerics, `_`, `-`, `.`.
|
||||
fn is_name_char(c: u8) -> bool {
|
||||
c.is_ascii_alphanumeric() || c == b'_' || c == b'-' || c == b'.'
|
||||
}
|
||||
|
||||
fn is_ws(c: u8) -> bool {
|
||||
matches!(c, b' ' | b'\t' | b'\n' | b'\r')
|
||||
}
|
||||
|
||||
fn slice_eq_ignore_case(a: &[u8], b_lower: &[u8]) -> bool {
|
||||
if a.len() != b_lower.len() {
|
||||
return false;
|
||||
}
|
||||
a.iter()
|
||||
.zip(b_lower.iter())
|
||||
.all(|(&x, &y)| x.to_ascii_lowercase() == y)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn attr_basic() {
|
||||
assert_eq!(
|
||||
attr(r#"<playlist name="Feature" id="00222" />"#, "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
assert_eq!(
|
||||
attr(r#"<playlist name="Feature" id="00222" />"#, "id"),
|
||||
Some("00222".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_case_insensitive_name() {
|
||||
assert_eq!(
|
||||
attr(r#"<playlist Name="Feature" />"#, "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
assert_eq!(
|
||||
attr(r#"<playlist NAME="Feature" />"#, "Name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_accepts_single_quotes() {
|
||||
assert_eq!(
|
||||
attr(r#"<playlist name='Feature' />"#, "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_whitespace_around_equals() {
|
||||
assert_eq!(
|
||||
attr(r#"<playlist name = "Feature" />"#, "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
assert_eq!(
|
||||
attr("<playlist name\n=\n\"Feature\" />", "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_missing_returns_none() {
|
||||
assert_eq!(attr(r#"<playlist name="X" />"#, "id"), None);
|
||||
assert_eq!(attr("", "name"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_no_substring_false_positive() {
|
||||
// Looking for "lang" should NOT match "lang_id" or
|
||||
// "language" because of the name-char boundary check.
|
||||
assert_eq!(attr(r#"<x lang_id="fra" language="eng" />"#, "lang"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_empty_value() {
|
||||
assert_eq!(attr(r#"<x name="" id="1" />"#, "name"), Some("".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_basic() {
|
||||
assert_eq!(text("<x>hello</x>", "x"), Some("hello".into()));
|
||||
assert_eq!(
|
||||
text("<x> hello world </x>", "x"),
|
||||
Some("hello world".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_case_insensitive_tag() {
|
||||
assert_eq!(text("<X>foo</X>", "x"), Some("foo".into()));
|
||||
assert_eq!(text("<Foo>bar</foo>", "foo"), Some("bar".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_namespace_prefix() {
|
||||
assert_eq!(text("<ns:tag>value</ns:tag>", "tag"), Some("value".into()));
|
||||
assert_eq!(text("<foo:Bar>v</foo:Bar>", "bar"), Some("v".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_self_closing() {
|
||||
assert_eq!(text("<x/>", "x"), Some("".into()));
|
||||
assert_eq!(text("<x />", "x"), Some("".into()));
|
||||
assert_eq!(text("<x attr=\"y\" />", "x"), Some("".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_with_attrs() {
|
||||
assert_eq!(
|
||||
text(r#"<x id="1" name="y">hello</x>"#, "x"),
|
||||
Some("hello".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_missing_close_returns_none() {
|
||||
assert_eq!(text("<x>hello", "x"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_skips_inner_tags_naively() {
|
||||
// Limitation noted: nested same-name tags aren't handled.
|
||||
// Different-name nesting works (we just return everything
|
||||
// between the open and close).
|
||||
assert_eq!(
|
||||
text("<x><y>nested</y></x>", "x"),
|
||||
Some("<y>nested</y>".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_basic() {
|
||||
let xml = r#"<x /> <y attr="1">body</y>"#;
|
||||
let (s, e) = find_element(xml, "y", 0).unwrap();
|
||||
assert_eq!(&xml[s..e], r#"<y attr="1">body</y>"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_self_closing() {
|
||||
let xml = r#"<x />"#;
|
||||
let (s, e) = find_element(xml, "x", 0).unwrap();
|
||||
assert_eq!(&xml[s..e], "<x />");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_handles_quoted_gt_in_attr() {
|
||||
// A `>` inside a quoted attribute value should not terminate
|
||||
// the open tag prematurely.
|
||||
let xml = r#"<x attr="foo>bar">body</x>"#;
|
||||
let (s, e) = find_element(xml, "x", 0).unwrap();
|
||||
assert_eq!(&xml[s..e], r#"<x attr="foo>bar">body</x>"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_iteration() {
|
||||
let xml = "<p>a</p><p>b</p><p>c</p>";
|
||||
let mut positions = Vec::new();
|
||||
let mut from = 0;
|
||||
while let Some((s, e)) = find_element(xml, "p", from) {
|
||||
positions.push(&xml[s..e]);
|
||||
from = e;
|
||||
}
|
||||
assert_eq!(positions, vec!["<p>a</p>", "<p>b</p>", "<p>c</p>"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_with_namespace() {
|
||||
let xml = r#"<root><ns:item id="1" /></root>"#;
|
||||
let (s, e) = find_element(xml, "item", 0).unwrap();
|
||||
assert_eq!(&xml[s..e], r#"<ns:item id="1" />"#);
|
||||
}
|
||||
}
|
||||
+60
-12
@@ -79,21 +79,27 @@ pub mod disc;
|
||||
pub mod drive;
|
||||
pub mod error;
|
||||
pub mod event;
|
||||
pub mod halt;
|
||||
pub(crate) mod identity;
|
||||
pub(crate) mod ifo;
|
||||
pub mod io;
|
||||
pub mod keydb;
|
||||
pub(crate) mod labels;
|
||||
pub mod labels;
|
||||
pub(crate) mod mpls;
|
||||
pub mod mux;
|
||||
pub mod pes;
|
||||
pub(crate) mod platform;
|
||||
pub mod profile;
|
||||
pub mod progress;
|
||||
pub mod scsi;
|
||||
pub mod sector;
|
||||
pub(crate) mod speed;
|
||||
pub(crate) mod udf;
|
||||
pub mod verify;
|
||||
|
||||
// Re-export verify types at the crate root for ergonomic imports.
|
||||
pub use verify::{SectorRange, SectorStatus, VerifyResult, verify_title};
|
||||
|
||||
// ─── Drive lifecycle ────────────────────────────────────────────────────────
|
||||
//
|
||||
// `Drive::open(path)` → `wait_ready()` → `init()` → `Disc::scan()`. `Drive`
|
||||
@@ -102,7 +108,7 @@ pub mod verify;
|
||||
pub use drive::capture::{
|
||||
CapturedFeature, DriveCapture, capture_drive_data, mask_bytes, mask_string,
|
||||
};
|
||||
pub use drive::{Drive, DriveStatus, find_drive, find_drives};
|
||||
pub use drive::{Drive, DriveStatus, find_drive};
|
||||
|
||||
// ─── Errors ─────────────────────────────────────────────────────────────────
|
||||
//
|
||||
@@ -111,6 +117,31 @@ pub use drive::{Drive, DriveStatus, find_drive, find_drives};
|
||||
// codes to localized messages. See `error.rs` for the full taxonomy.
|
||||
pub use error::{Error, Result};
|
||||
|
||||
// ─── 0.18 primitives ────────────────────────────────────────────────────────
|
||||
//
|
||||
// One-bit cooperative cancellation token, shared by every long-running loop
|
||||
// in libfreemkv (sweep, patch, mux). Replaces the ad-hoc `Arc<AtomicBool>`
|
||||
// flags scattered across 0.17 (`DiscStream::set_halt`, autorip's
|
||||
// `HALT_FLAGS` registry). Clone it cheaply; pass it by value into each
|
||||
// component; poll `is_cancelled()` inside the loop body.
|
||||
pub use halt::Halt;
|
||||
|
||||
// Generic bounded producer/consumer primitive used by sweep, patch, and
|
||||
// mux to overlap reads with writes via a dedicated consumer thread.
|
||||
// `Pipeline::spawn(name, depth, sink)` spawns a named consumer; `pipe.send(item)`
|
||||
// pushes one item with back-pressure; `pipe.finish()` joins the
|
||||
// consumer and surfaces its `close()` output. Callers implement `Sink`
|
||||
// to define per-item behaviour and end-of-stream finalisation.
|
||||
//
|
||||
// `DEFAULT_PIPELINE_DEPTH` (=4) is for callers without specific needs;
|
||||
// most should use READ_PIPELINE_DEPTH or WRITE_PIPELINE_DEPTH instead.
|
||||
// Patch uses `WRITE_THROUGH_DEPTH` (=1). Returning `Flow::Stop` from
|
||||
// `apply` ends the consumer cleanly (still calls `close()`).
|
||||
pub use io::pipeline::{
|
||||
DEFAULT_PIPELINE_DEPTH, Flow, Pipeline, READ_PIPELINE_DEPTH, Sink, WRITE_PIPELINE_DEPTH,
|
||||
WRITE_THROUGH_DEPTH,
|
||||
};
|
||||
|
||||
// ─── Drive events (low-level callbacks) ─────────────────────────────────────
|
||||
pub use event::{Event, EventKind};
|
||||
pub use identity::DriveId;
|
||||
@@ -123,7 +154,7 @@ pub use profile::DriveProfile;
|
||||
// don't touch `DecryptKeys` directly — `DiscStream::new(reader, title, keys, …)`
|
||||
// accepts whatever `Disc::decrypt_keys()` returned. `decrypt_sectors()` is
|
||||
// for callers that operate on raw sector buffers (e.g. ISO patching).
|
||||
pub use decrypt::{DecryptKeys, decrypt_sectors};
|
||||
pub use decrypt::{DecryptKeys, decrypt_sectors, decrypt_threads, set_decrypt_threads};
|
||||
|
||||
// ─── Disc structure ─────────────────────────────────────────────────────────
|
||||
//
|
||||
@@ -132,10 +163,15 @@ pub use decrypt::{DecryptKeys, decrypt_sectors};
|
||||
// for displaying disc name + format quickly while a full scan runs in the
|
||||
// background. The codec / channel / resolution enums are the canonical
|
||||
// structured representation; never compare against display strings.
|
||||
// Note: `disc::Stream` here is the codec enum (audio / video / sub kind)
|
||||
// — not the `pes::Stream` trait re-exported below as `PesStream`. Two
|
||||
// different concepts, the same short name; the trait gets the `Pes`
|
||||
// prefix at the crate root to keep both addressable.
|
||||
pub use disc::{
|
||||
AacsState, AudioChannels, AudioStream, Clip, Codec, ColorSpace, ContentFormat, Disc,
|
||||
DiscFormat, DiscId, DiscTitle, Extent, FrameRate, HdrFormat, KeySource, LabelPurpose,
|
||||
LabelQualifier, Resolution, SampleRate, ScanOptions, Stream, SubtitleStream, VideoStream,
|
||||
AacsState, AudioChannels, AudioStream, Clip, Codec, ColorSpace, ContentFormat, DamageSeverity,
|
||||
Disc, DiscFormat, DiscId, DiscTitle, Extent, FrameRate, HdrFormat, KeySource, LabelPurpose,
|
||||
LabelQualifier, PatchOptions, PatchOutcome, Resolution, SampleRate, ScanOptions, Stream,
|
||||
SubtitleStream, SweepOptions, VideoStream, classify_damage,
|
||||
};
|
||||
|
||||
// ─── Streams ────────────────────────────────────────────────────────────────
|
||||
@@ -143,7 +179,7 @@ pub use disc::{
|
||||
// All stream types implement `pes::Stream` — read PES frames from a source,
|
||||
// write PES frames to a sink. Pick the right type at construction:
|
||||
//
|
||||
// - `DiscStream` — physical drive or ISO (any `SectorReader`). Always read.
|
||||
// - `DiscStream` — physical drive or ISO (any `SectorSource`). Read-only.
|
||||
// - `MkvStream` — Matroska container. Read on `open()`, write on `create()`.
|
||||
// - `M2tsStream` — Blu-ray Transport Stream. Read on `open()`, write on `create()`.
|
||||
// - `NetworkStream` — TCP. Read on `listen()`, write on `connect()`.
|
||||
@@ -153,6 +189,12 @@ pub use disc::{
|
||||
// Most consumers use the URL resolvers (`input()` / `output()`) which pick
|
||||
// the right type from a scheme:// URL. Direct construction is for callers
|
||||
// that need to wire custom readers (e.g. autorip's drive-session reuse).
|
||||
// The trait is re-exported as `PesStream` here to disambiguate from
|
||||
// `disc::Stream` (the codec-kind enum re-exported above), which would
|
||||
// otherwise collide at the crate root.
|
||||
pub use pes::PesFrame;
|
||||
pub use pes::Stream as PesStream;
|
||||
|
||||
pub use mux::DiscStream;
|
||||
pub use mux::M2tsStream;
|
||||
pub use mux::MkvStream;
|
||||
@@ -164,10 +206,16 @@ pub use mux::{InputOptions, StreamUrl, input, output, parse_url};
|
||||
// ─── Lower-level surfaces ───────────────────────────────────────────────────
|
||||
//
|
||||
// `ScsiTransport` is the platform-abstraction trait Drive uses; expose for
|
||||
// out-of-tree platform backends. `SectorReader` lets callers feed any byte
|
||||
// source (test harness, network image, SMB share) into the disc scan
|
||||
// pipeline; `FileSectorReader` is the standard ISO-on-disk implementation.
|
||||
pub use scsi::ScsiTransport;
|
||||
pub use sector::{FileSectorReader, SectorReader};
|
||||
// out-of-tree platform backends. `SectorSource` / `SectorSink` are the
|
||||
// direction-typed read/write traits; `FileSectorSource` and `FileSectorSink`
|
||||
// are the ISO-on-disk implementations. [`DecryptingSectorSource`] is the
|
||||
// single decrypt-on-read decorator (AACS / CSS / none) — wrap any
|
||||
// `SectorSource` to get plaintext sectors out.
|
||||
pub use mux::build_iso_pipeline;
|
||||
pub use scsi::{DriveInfo, ScsiSense, ScsiTransport, drive_has_disc, list_drives};
|
||||
pub use sector::{
|
||||
DecryptingSectorSource, FileSectorSink, FileSectorSource, PrefetchedSectorSource, SectorSink,
|
||||
SectorSource,
|
||||
};
|
||||
pub use speed::DriveSpeed;
|
||||
pub use udf::{UdfFs, read_filesystem};
|
||||
|
||||
@@ -76,6 +76,7 @@ impl CodecParser for Ac3Parser {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: data[start..start + frame_size].to_vec(),
|
||||
duration_ns: None,
|
||||
});
|
||||
pos = start + frame_size;
|
||||
}
|
||||
|
||||
@@ -86,6 +86,7 @@ impl CodecParser for DtsParser {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: data[start..start + total_size].to_vec(),
|
||||
duration_ns: None,
|
||||
});
|
||||
pos = start + total_size;
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ impl CodecParser for DvdSubParser {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: pes.data.clone(),
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
|
||||
@@ -85,6 +85,7 @@ impl CodecParser for H264Parser {
|
||||
pts_ns,
|
||||
keyframe,
|
||||
data: frame_data,
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
@@ -176,14 +177,16 @@ impl<'a> Iterator for NalIterator<'a> {
|
||||
}
|
||||
|
||||
/// Find the position of the next start code (00 00 01) at or after `from`.
|
||||
///
|
||||
/// Backed by `memchr::memmem::find` for SIMD-accelerated bytestring
|
||||
/// search. On AVX2-capable x86_64 this runs ~5–10× the byte-by-byte
|
||||
/// scan that preceded it; on a 200 KB UHD HEVC frame the saving is
|
||||
/// in the hundreds of microseconds per call.
|
||||
pub fn find_start_code(data: &[u8], from: usize) -> Option<usize> {
|
||||
if data.len() < from + 3 {
|
||||
return None;
|
||||
}
|
||||
// Range excludes last 2 bytes since we read 3 bytes at each position.
|
||||
// data.len()-2 as exclusive upper bound means last checked index is data.len()-3,
|
||||
// which accesses data[len-3], data[len-2], data[len-1] — all valid.
|
||||
(from..data.len() - 2).find(|&i| data[i] == 0x00 && data[i + 1] == 0x00 && data[i + 2] == 0x01)
|
||||
memchr::memmem::find(&data[from..], b"\x00\x00\x01").map(|rel| from + rel)
|
||||
}
|
||||
|
||||
/// Skip past the start code at position `pos`, returning the first byte after it.
|
||||
|
||||
@@ -52,7 +52,10 @@ impl CodecParser for HevcParser {
|
||||
let pts_ns = pes.dts.or(pes.pts).map(pts_to_ns).unwrap_or(0);
|
||||
let data = &pes.data;
|
||||
let mut keyframe = false;
|
||||
let mut frame_data = Vec::new();
|
||||
// Pre-size: output is ~input bytes with a few 4-byte length
|
||||
// prefixes added. UHD frames are 150-300 KB; the unsized Vec
|
||||
// growth chain otherwise reallocs 5-7× per frame.
|
||||
let mut frame_data = Vec::with_capacity(data.len() + 64);
|
||||
|
||||
// Single-pass NAL scan: extract params, detect keyframes, build length-prefixed output
|
||||
let mut pos = 0;
|
||||
@@ -107,6 +110,7 @@ impl CodecParser for HevcParser {
|
||||
pts_ns,
|
||||
keyframe,
|
||||
data: frame_data,
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@ impl CodecParser for LpcmParser {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: pes.data[BD_LPCM_HEADER_SIZE..].to_vec(),
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,13 @@ pub struct Frame {
|
||||
pub keyframe: bool,
|
||||
/// Frame data (elementary stream bytes).
|
||||
pub data: Vec<u8>,
|
||||
/// Optional duration in nanoseconds — only set by parsers that
|
||||
/// can compute one (currently PGS, which pairs a display PCS
|
||||
/// with the following empty PCS). When `Some`, the MKV muxer
|
||||
/// emits a `BlockGroup` with `BlockDuration` instead of a
|
||||
/// `SimpleBlock`; without it players guess the display interval
|
||||
/// (subtitles linger past their end-time).
|
||||
pub duration_ns: Option<u64>,
|
||||
}
|
||||
|
||||
/// Convert 90kHz PTS to nanoseconds (round to nearest).
|
||||
@@ -71,6 +78,7 @@ impl CodecParser for PassthroughParser {
|
||||
pts_ns,
|
||||
keyframe: self.keyframe,
|
||||
data: pes.data.clone(),
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
|
||||
@@ -194,6 +194,7 @@ impl CodecParser for Mpeg2Parser {
|
||||
pts_ns,
|
||||
keyframe,
|
||||
data: pes.data.clone(),
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
|
||||
+132
-29
@@ -1,12 +1,32 @@
|
||||
//! HDMV PGS (Presentation Graphics Stream) subtitle parser.
|
||||
//!
|
||||
//! PGS segments: PCS, WDS, PDS, ODS, END.
|
||||
//! Each PES packet contains one or more segments.
|
||||
//! All segments are keyframes (no inter-segment dependencies).
|
||||
//! PGS segments: PCS, WDS, PDS, ODS, END. Each PES packet starts with
|
||||
//! one of those (segment_type byte at offset 0).
|
||||
//!
|
||||
//! Subtitle display lifecycle (BD spec):
|
||||
//! - A "display" PCS (number_of_composition_objects > 0) starts a
|
||||
//! visible subtitle. Its WDS/PDS/ODS follow.
|
||||
//! - A later "empty" PCS (number_of_composition_objects == 0) clears
|
||||
//! the screen.
|
||||
//!
|
||||
//! For Matroska output we collapse that pair into one block with
|
||||
//! `BlockDuration` set to (clear_pts - display_pts). Without a
|
||||
//! duration, hardware players linger on the last bitmap until the
|
||||
//! next subtitle replaces it — which can be many seconds, and on a
|
||||
//! disc where the final subtitle has no follower, until end of file.
|
||||
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
pub struct PgsParser;
|
||||
const SEGMENT_PCS: u8 = 0x16;
|
||||
// Offset within the PES payload at which number_of_composition_objects
|
||||
// lives in a PCS: 3-byte segment header + 10 bytes of PCS fields
|
||||
// (video_w/h, frame_rate, comp_num, comp_state, palette_update,
|
||||
// palette_id_ref) = 13.
|
||||
const PCS_NUM_OBJECTS_OFFSET: usize = 13;
|
||||
|
||||
pub struct PgsParser {
|
||||
pending: Option<(i64, Vec<u8>)>,
|
||||
}
|
||||
|
||||
impl Default for PgsParser {
|
||||
fn default() -> Self {
|
||||
@@ -16,7 +36,7 @@ impl Default for PgsParser {
|
||||
|
||||
impl PgsParser {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
Self { pending: None }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,11 +46,63 @@ impl CodecParser for PgsParser {
|
||||
return Vec::new();
|
||||
}
|
||||
let pts_ns = pes.pts.map(pts_to_ns).unwrap_or(0);
|
||||
vec![Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: pes.data.clone(),
|
||||
}]
|
||||
|
||||
let is_pcs = pes.data[0] == SEGMENT_PCS;
|
||||
let pcs_num_objects = if is_pcs && pes.data.len() > PCS_NUM_OBJECTS_OFFSET {
|
||||
Some(pes.data[PCS_NUM_OBJECTS_OFFSET])
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut out = Vec::new();
|
||||
match pcs_num_objects {
|
||||
// Clear/empty PCS — closes any pending display. Drop the
|
||||
// clear segment itself; BlockDuration covers the screen
|
||||
// wipe.
|
||||
Some(0) => {
|
||||
if let Some((start_pts, data)) = self.pending.take() {
|
||||
let duration = pts_ns.saturating_sub(start_pts).max(0) as u64;
|
||||
out.push(Frame {
|
||||
pts_ns: start_pts,
|
||||
keyframe: true,
|
||||
data,
|
||||
duration_ns: Some(duration),
|
||||
});
|
||||
}
|
||||
}
|
||||
// Display PCS — start a new pending. If a prior display
|
||||
// was never explicitly cleared (replace-without-clear),
|
||||
// emit it with the new PCS's PTS as its end.
|
||||
Some(_) => {
|
||||
if let Some((start_pts, data)) = self.pending.take() {
|
||||
let duration = pts_ns.saturating_sub(start_pts).max(0) as u64;
|
||||
out.push(Frame {
|
||||
pts_ns: start_pts,
|
||||
keyframe: true,
|
||||
data,
|
||||
duration_ns: Some(duration),
|
||||
});
|
||||
}
|
||||
self.pending = Some((pts_ns, pes.data.clone()));
|
||||
}
|
||||
// Non-PCS first segment — either a continuation of the
|
||||
// current display set, or non-standard layout. If we have
|
||||
// a pending display, append; otherwise emit as-is.
|
||||
None => {
|
||||
if let Some((_, ref mut buf)) = self.pending {
|
||||
buf.extend_from_slice(&pes.data);
|
||||
} else {
|
||||
out.push(Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: pes.data.clone(),
|
||||
duration_ns: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
fn codec_private(&self) -> Option<Vec<u8>> {
|
||||
@@ -52,29 +124,60 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_basic_segment() {
|
||||
let mut parser = PgsParser::new();
|
||||
// PGS segment data (PCS = presentation composition segment)
|
||||
let data = vec![0x16, 0x00, 0x00, 0x11, 0x01, 0x02, 0x03];
|
||||
let pes = make_pes(data.clone(), Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data, data);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
// Minimum-viable PCS bytes: type 0x16, segment_length (2 bytes),
|
||||
// then 11 bytes of PCS fields ending in number_of_composition_objects.
|
||||
fn pcs_bytes(num_objects: u8) -> Vec<u8> {
|
||||
let mut v = vec![SEGMENT_PCS, 0x00, 0x0B];
|
||||
v.extend_from_slice(&[0x07, 0x80, 0x04, 0x38]); // 1920x1080
|
||||
v.push(0x10); // frame_rate
|
||||
v.extend_from_slice(&[0x00, 0x01]); // composition_number
|
||||
v.push(0x80); // composition_state = EpochStart
|
||||
v.push(0x00); // palette_update + reserved
|
||||
v.push(0x00); // palette_id_ref
|
||||
v.push(num_objects);
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_keyframes() {
|
||||
fn display_then_clear_yields_duration() {
|
||||
let mut parser = PgsParser::new();
|
||||
for i in 0..3 {
|
||||
let data = vec![0x16, 0x00, i];
|
||||
let pes = make_pes(data, Some(90000 * i as i64));
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert!(frames[0].keyframe, "PGS segment should always be keyframe");
|
||||
}
|
||||
|
||||
// Display PCS at PTS 90000 (= 1s)
|
||||
let display = pcs_bytes(1);
|
||||
let frames = parser.parse(&make_pes(display.clone(), Some(90000)));
|
||||
assert!(frames.is_empty(), "display PCS should be pending");
|
||||
|
||||
// Empty PCS at PTS 270000 (= 3s)
|
||||
let clear = pcs_bytes(0);
|
||||
let frames = parser.parse(&make_pes(clear, Some(270000)));
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
assert_eq!(frames[0].duration_ns, Some(2_000_000_000));
|
||||
assert_eq!(frames[0].data, display);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replace_without_clear_still_emits_prior_with_duration() {
|
||||
let mut parser = PgsParser::new();
|
||||
let _ = parser.parse(&make_pes(pcs_bytes(1), Some(90000)));
|
||||
let frames = parser.parse(&make_pes(pcs_bytes(1), Some(180000)));
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
assert_eq!(frames[0].duration_ns, Some(1_000_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_pcs_segment_appends_to_pending() {
|
||||
let mut parser = PgsParser::new();
|
||||
let _ = parser.parse(&make_pes(pcs_bytes(1), Some(90000)));
|
||||
// ODS-like segment (type 0x15)
|
||||
let frames = parser.parse(&make_pes(vec![0x15, 0x00, 0x02, 0xAA, 0xBB], Some(90000)));
|
||||
assert!(frames.is_empty());
|
||||
// Clear closes the set; data should include the appended bytes.
|
||||
let frames = parser.parse(&make_pes(pcs_bytes(0), Some(180000)));
|
||||
assert_eq!(frames.len(), 1);
|
||||
let data = &frames[0].data;
|
||||
assert!(data.windows(5).any(|w| w == [0x15, 0x00, 0x02, 0xAA, 0xBB]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -139,6 +139,7 @@ impl CodecParser for TrueHdParser {
|
||||
pts_ns: self.next_pts_ns,
|
||||
keyframe: is_major_sync,
|
||||
data: self.buf[..unit_bytes].to_vec(),
|
||||
duration_ns: None,
|
||||
});
|
||||
self.buf.drain(..unit_bytes);
|
||||
self.next_pts_ns += AU_DURATION_NS;
|
||||
|
||||
@@ -96,6 +96,7 @@ impl CodecParser for Vc1Parser {
|
||||
pts_ns: ts_ns,
|
||||
keyframe,
|
||||
data: frame_data.to_vec(),
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
//! `DemuxThread` — runs the read+decrypt+demux pipeline on a
|
||||
//! dedicated thread, feeding completed `PesPacket` batches to the
|
||||
//! caller via a bounded channel.
|
||||
//!
|
||||
//! ## Why a second worker thread
|
||||
//!
|
||||
//! With [`crate::sector::PrefetchedSectorSource`] alone, read+decrypt
|
||||
//! already runs on a producer thread; the *consumer* (main) thread
|
||||
//! still serialises `ts_demuxer.feed` (M2TS parsing) with the codec
|
||||
//! parsers. Profiling on the rip1 testbed showed feed at ~37 % and
|
||||
//! codec parse at ~44 % of consumer wall time — i.e. feed is heavy
|
||||
//! enough that pipelining it with parse pays for itself.
|
||||
//!
|
||||
//! Splitting them: feed runs in [`DemuxThread`]; the consumer thread
|
||||
//! receives `Vec<PesPacket>` batches and runs codec parse + frame
|
||||
//! emission only. Total throughput becomes `1/max(feed, parse)`
|
||||
//! instead of `1/(feed + parse)`.
|
||||
//!
|
||||
//! ## Lifecycle
|
||||
//!
|
||||
//! [`DemuxThread::spawn`] takes ownership of the inner reader and the
|
||||
//! demuxer state, returns a handle plus a `Receiver<DemuxBatch>`.
|
||||
//! Dropping the handle closes the channel which signals the thread
|
||||
//! to exit; the join in `Drop::drop` is bounded.
|
||||
|
||||
use crate::halt::Halt;
|
||||
use crate::sector::SectorSource;
|
||||
use crossbeam_channel::{Receiver, Sender, bounded};
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// Output channel depth. Two batches in flight keeps the consumer
|
||||
/// (codec parser) busy without piling up demuxed bytes if it stalls.
|
||||
const DEMUX_CHANNEL_DEPTH: usize = 2;
|
||||
|
||||
/// One demuxed batch flowing from the demux thread to the consumer.
|
||||
pub enum DemuxBatch {
|
||||
/// Successfully demuxed PesPackets — non-empty.
|
||||
Ts(Vec<super::ts::PesPacket>),
|
||||
Ps(Vec<super::ps::PsPacket>),
|
||||
/// Underlying reader returned an error. Terminal.
|
||||
Err(std::io::Error),
|
||||
}
|
||||
|
||||
/// Spawned demux thread. Drop joins.
|
||||
///
|
||||
/// In zero-copy mode the thread also owns an opaque
|
||||
/// `producer_shell: Option<Box<dyn Send>>` — the join handle of the
|
||||
/// upstream producer (sector or byte prefetcher). Dropping the
|
||||
/// `DemuxThread` runs the shell's `Drop`, which joins the producer.
|
||||
/// `Box<dyn Send>` rather than a concrete type so the same demux
|
||||
/// worker can be wired behind either prefetcher kind.
|
||||
pub struct DemuxThread {
|
||||
handle: Option<JoinHandle<()>>,
|
||||
#[allow(dead_code)]
|
||||
producer_shell: Option<Box<dyn Send>>,
|
||||
}
|
||||
|
||||
impl DemuxThread {
|
||||
/// Spawn the demux thread. Returns the thread handle and a
|
||||
/// receiver for [`DemuxBatch`] items.
|
||||
///
|
||||
/// `reader` is the fully-composed read+decrypt stack (e.g.
|
||||
/// [`PrefetchedSectorSource`](crate::sector::PrefetchedSectorSource)
|
||||
/// wrapping
|
||||
/// [`DecryptingSectorSource`](crate::sector::DecryptingSectorSource)).
|
||||
/// `extents` is what the thread walks; it issues one
|
||||
/// `read_sectors` per batch of `batch_sectors` sectors (aligned
|
||||
/// to 3-sector AACS units when possible).
|
||||
pub fn spawn<S: SectorSource + Send + 'static>(
|
||||
mut reader: S,
|
||||
extents: Vec<crate::disc::Extent>,
|
||||
batch_sectors: u16,
|
||||
halt: Option<Halt>,
|
||||
ts: Option<super::ts::TsDemuxer>,
|
||||
ps: Option<super::ps::PsDemuxer>,
|
||||
) -> (Self, Receiver<DemuxBatch>) {
|
||||
let (tx, rx) = bounded::<DemuxBatch>(DEMUX_CHANNEL_DEPTH);
|
||||
let mut ts = ts;
|
||||
let mut ps = ps;
|
||||
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("freemkv-demux".into())
|
||||
.spawn(move || {
|
||||
let mut buf = vec![0u8; batch_sectors as usize * 2048];
|
||||
let mut ext_idx = 0usize;
|
||||
let mut offset: u32 = 0;
|
||||
let prof = std::env::var_os("FREEMKV_PROFILE").is_some();
|
||||
let mut prof_started = std::time::Instant::now();
|
||||
let mut prof_last_dump = prof_started;
|
||||
let mut prof_read_ns: u128 = 0;
|
||||
let mut prof_feed_ns: u128 = 0;
|
||||
let mut prof_send_ns: u128 = 0;
|
||||
let mut prof_bytes: u64 = 0;
|
||||
while ext_idx < extents.len() {
|
||||
if halt.as_ref().map(|h| h.is_cancelled()).unwrap_or(false) {
|
||||
return;
|
||||
}
|
||||
let ext = &extents[ext_idx];
|
||||
let remaining = ext.sector_count.saturating_sub(offset);
|
||||
if remaining == 0 {
|
||||
ext_idx += 1;
|
||||
offset = 0;
|
||||
continue;
|
||||
}
|
||||
let mut sectors = remaining.min(batch_sectors as u32) as u16;
|
||||
if sectors >= 3 {
|
||||
sectors -= sectors % 3;
|
||||
}
|
||||
let bytes = sectors as usize * 2048;
|
||||
if buf.len() < bytes {
|
||||
buf.resize(bytes, 0);
|
||||
}
|
||||
let lba = ext.start_lba + offset;
|
||||
let t0 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let n = match reader.read_sectors(lba, sectors, &mut buf[..bytes], false) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
let _ = tx.send(DemuxBatch::Err(e.into()));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let t1 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
offset += sectors as u32;
|
||||
|
||||
// Demux this batch immediately so the channel
|
||||
// carries already-parsed PesPackets, not raw
|
||||
// sector bytes.
|
||||
if let Some(ref mut d) = ts {
|
||||
let pkts = d.feed(&buf[..n]);
|
||||
let t2 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if !pkts.is_empty() && tx.send(DemuxBatch::Ts(pkts)).is_err() {
|
||||
return; // consumer dropped
|
||||
}
|
||||
let t3 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if prof {
|
||||
prof_read_ns += t1.unwrap().duration_since(t0.unwrap()).as_nanos();
|
||||
prof_feed_ns += t2.unwrap().duration_since(t1.unwrap()).as_nanos();
|
||||
prof_send_ns += t3.unwrap().duration_since(t2.unwrap()).as_nanos();
|
||||
prof_bytes += n as u64;
|
||||
let now = t3.unwrap();
|
||||
if now.duration_since(prof_last_dump)
|
||||
>= std::time::Duration::from_secs(5)
|
||||
{
|
||||
let el = now.duration_since(prof_started).as_millis().max(1);
|
||||
let mbps = prof_bytes as u128 * 1000 / 1_000_000 / el;
|
||||
eprintln!(
|
||||
"[demux] elapsed={}ms in={}MB/s read={}% feed={}% send={}%",
|
||||
el,
|
||||
mbps,
|
||||
prof_read_ns / 10_000 / el,
|
||||
prof_feed_ns / 10_000 / el,
|
||||
prof_send_ns / 10_000 / el,
|
||||
);
|
||||
prof_last_dump = now;
|
||||
prof_started = now;
|
||||
prof_read_ns = 0;
|
||||
prof_feed_ns = 0;
|
||||
prof_send_ns = 0;
|
||||
prof_bytes = 0;
|
||||
}
|
||||
}
|
||||
} else if let Some(ref mut d) = ps {
|
||||
let pkts = d.feed(&buf[..n]);
|
||||
if !pkts.is_empty() && tx.send(DemuxBatch::Ps(pkts)).is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
// EOF — emit any flushed packets too.
|
||||
if let Some(ref mut d) = ts {
|
||||
let tail = d.flush();
|
||||
if !tail.is_empty() {
|
||||
let _ = tx.send(DemuxBatch::Ts(tail));
|
||||
}
|
||||
} else if let Some(ref mut d) = ps {
|
||||
let tail = d.flush();
|
||||
if !tail.is_empty() {
|
||||
let _ = tx.send(DemuxBatch::Ps(tail));
|
||||
}
|
||||
}
|
||||
// Sender drops here -> consumer sees RecvError → EOF.
|
||||
})
|
||||
.expect("freemkv-demux thread spawn failed");
|
||||
|
||||
(
|
||||
Self {
|
||||
handle: Some(handle),
|
||||
producer_shell: None,
|
||||
},
|
||||
rx,
|
||||
)
|
||||
}
|
||||
|
||||
/// Zero-copy variant. Instead of taking a `SectorSource` and
|
||||
/// memcpy-ing through its `read_sectors` API, this constructor
|
||||
/// consumes the prefetch channels directly: filled buffers come
|
||||
/// in via `prefetch_rx`, the demux thread feeds them, then
|
||||
/// returns them to `recycle_tx` for the producer to re-fill.
|
||||
/// Eliminates the 16 MiB memcpy per batch that the SectorSource
|
||||
/// adapter incurred (and, with the producer-side recycling pool,
|
||||
/// also eliminates the per-batch heap alloc / cross-thread free
|
||||
/// that was costing 40 %+ of demux-thread time before).
|
||||
///
|
||||
/// `producer_shell` is an opaque handle whose only purpose is to
|
||||
/// outlive the demux thread and join the upstream producer when
|
||||
/// dropped. Both
|
||||
/// [`crate::sector::PrefetchedSectorSource::into_channels`] and
|
||||
/// [`crate::io::byte_prefetcher::BytePrefetcher::into_channels`]
|
||||
/// hand back a shell that fits — pass either.
|
||||
pub fn spawn_zero_copy<S: Send + 'static>(
|
||||
prefetch_rx: Receiver<std::io::Result<Vec<u8>>>,
|
||||
recycle_tx: Sender<Vec<u8>>,
|
||||
producer_shell: S,
|
||||
halt: Option<Halt>,
|
||||
ts: Option<super::ts::TsDemuxer>,
|
||||
ps: Option<super::ps::PsDemuxer>,
|
||||
) -> (Self, Receiver<DemuxBatch>) {
|
||||
let (tx, rx) = bounded::<DemuxBatch>(DEMUX_CHANNEL_DEPTH);
|
||||
let mut ts = ts;
|
||||
let mut ps = ps;
|
||||
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("freemkv-demux".into())
|
||||
.spawn(move || {
|
||||
let prof = std::env::var_os("FREEMKV_PROFILE").is_some();
|
||||
let mut prof_started = std::time::Instant::now();
|
||||
let mut prof_last_dump = prof_started;
|
||||
let mut prof_read_ns: u128 = 0;
|
||||
let mut prof_feed_ns: u128 = 0;
|
||||
let mut prof_bytes: u64 = 0;
|
||||
loop {
|
||||
if halt.as_ref().map(|h| h.is_cancelled()).unwrap_or(false) {
|
||||
return;
|
||||
}
|
||||
let t0 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let buf = match prefetch_rx.recv() {
|
||||
Ok(Ok(b)) => b,
|
||||
Ok(Err(e)) => {
|
||||
let _ = tx.send(DemuxBatch::Err(e));
|
||||
return;
|
||||
}
|
||||
Err(_) => break, // producer done → EOF
|
||||
};
|
||||
let t1 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let n = buf.len();
|
||||
if let Some(ref mut d) = ts {
|
||||
let pkts = d.feed(&buf);
|
||||
let t2 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Recycle the buffer back to the producer
|
||||
// before pushing the demuxed packets. If the
|
||||
// recycle channel is closed the producer has
|
||||
// exited; we drop the buffer and continue.
|
||||
let _ = recycle_tx.send(buf);
|
||||
if !pkts.is_empty() && tx.send(DemuxBatch::Ts(pkts)).is_err() {
|
||||
return;
|
||||
}
|
||||
if prof {
|
||||
prof_read_ns += t1.unwrap().duration_since(t0.unwrap()).as_nanos();
|
||||
prof_feed_ns += t2.unwrap().duration_since(t1.unwrap()).as_nanos();
|
||||
prof_bytes += n as u64;
|
||||
let now = std::time::Instant::now();
|
||||
if now.duration_since(prof_last_dump)
|
||||
>= std::time::Duration::from_secs(5)
|
||||
{
|
||||
let el = now.duration_since(prof_started).as_millis().max(1);
|
||||
let mbps = prof_bytes as u128 * 1000 / 1_000_000 / el;
|
||||
eprintln!(
|
||||
"[demux] elapsed={}ms in={}MB/s read={}% feed={}%",
|
||||
el,
|
||||
mbps,
|
||||
prof_read_ns / 10_000 / el,
|
||||
prof_feed_ns / 10_000 / el,
|
||||
);
|
||||
prof_last_dump = now;
|
||||
prof_started = now;
|
||||
prof_read_ns = 0;
|
||||
prof_feed_ns = 0;
|
||||
prof_bytes = 0;
|
||||
}
|
||||
}
|
||||
} else if let Some(ref mut d) = ps {
|
||||
let pkts = d.feed(&buf);
|
||||
let _ = recycle_tx.send(buf);
|
||||
if !pkts.is_empty() && tx.send(DemuxBatch::Ps(pkts)).is_err() {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
let _ = recycle_tx.send(buf);
|
||||
}
|
||||
}
|
||||
// Flush tail packets at EOF.
|
||||
if let Some(ref mut d) = ts {
|
||||
let tail = d.flush();
|
||||
if !tail.is_empty() {
|
||||
let _ = tx.send(DemuxBatch::Ts(tail));
|
||||
}
|
||||
} else if let Some(ref mut d) = ps {
|
||||
let tail = d.flush();
|
||||
if !tail.is_empty() {
|
||||
let _ = tx.send(DemuxBatch::Ps(tail));
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("freemkv-demux thread spawn failed");
|
||||
|
||||
(
|
||||
Self {
|
||||
handle: Some(handle),
|
||||
producer_shell: Some(Box::new(producer_shell)),
|
||||
},
|
||||
rx,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DemuxThread {
|
||||
fn drop(&mut self) {
|
||||
if let Some(h) = self.handle.take() {
|
||||
let _ = h.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
+359
-35
@@ -1,16 +1,18 @@
|
||||
//! DiscStream — read any disc (physical drive or ISO file) → PES frames.
|
||||
//!
|
||||
//! One stream type for all disc sources. The source is a SectorReader —
|
||||
//! Drive (hardware) or IsoSectorReader (file). DiscStream doesn't care.
|
||||
//! One stream type for all disc sources. The source is a SectorSource —
|
||||
//! Drive (hardware) or FileSectorSource (file). DiscStream doesn't care.
|
||||
//!
|
||||
//! Read-only. For disc→ISO (raw sector copy), use `Disc::copy()`.
|
||||
|
||||
use crate::disc::{Disc, DiscTitle, Extent};
|
||||
use crate::drive::extract_scsi_context;
|
||||
use crate::event::{BatchSizeReason, Event, EventKind};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::halt::Halt;
|
||||
use crate::sector::{DecryptingSectorSource, SectorSource};
|
||||
use std::io;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
/// Ramp back up to the preferred batch size after this many sectors
|
||||
/// of clean reading at the current (reduced) size. 100 MiB = 51,200 sectors.
|
||||
@@ -95,12 +97,22 @@ impl AdaptiveBatch {
|
||||
|
||||
/// Disc stream. Reads sectors from any source → PES frames.
|
||||
///
|
||||
/// Sources: physical drive, ISO file, or any SectorReader.
|
||||
/// Sources: physical drive, ISO file, or any SectorSource.
|
||||
/// Decrypt, demux, and codec parsing happen internally.
|
||||
pub struct DiscStream {
|
||||
reader: Box<dyn SectorReader>,
|
||||
/// Underlying sector source wrapped in the 0.18
|
||||
/// [`DecryptingSectorSource`] decorator. Every `read_sectors`
|
||||
/// call yields plaintext, so `fill_extents` no longer needs an
|
||||
/// inline `decrypt::decrypt_sectors` step. `DecryptKeys::None`
|
||||
/// (raw / unencrypted disc) makes the decorator a pass-through.
|
||||
reader: DecryptingSectorSource<Box<dyn SectorSource>>,
|
||||
title: DiscTitle,
|
||||
disc: Option<Disc>,
|
||||
/// Mirror of the keys handed in at construction. The decorator
|
||||
/// owns the cryptographic state; this field is kept for
|
||||
/// metadata-side callers (`info()` and friends) that want to
|
||||
/// know whether the disc was encrypted, without reaching through
|
||||
/// the wrapper.
|
||||
decrypt_keys: crate::decrypt::DecryptKeys,
|
||||
|
||||
// Extents to read
|
||||
@@ -119,15 +131,29 @@ pub struct DiscStream {
|
||||
adaptive: AdaptiveBatch,
|
||||
pub errors: u64,
|
||||
pub skip_errors: bool,
|
||||
/// When set and the flag is raised, fill_extents returns Err(Halted) at the
|
||||
/// next retry boundary. Unlike skip_errors, this propagates the error up so
|
||||
/// the rip terminates cleanly. Share the Arc with Drive::halt_flag() to get
|
||||
/// unified Stop behavior across drive reads and sector processing.
|
||||
halt: Option<Arc<AtomicBool>>,
|
||||
/// When set and the token is cancelled, fill_extents returns Err(Halted)
|
||||
/// at the next retry boundary. Unlike skip_errors, this propagates the
|
||||
/// error up so the rip terminates cleanly. Construct with
|
||||
/// [`DiscStream::with_halt`] (preferred) or set post-hoc via the
|
||||
/// deprecated [`DiscStream::set_halt`] bridge — both populate this same
|
||||
/// field and either entry point yields one source of truth.
|
||||
halt: Option<Halt>,
|
||||
event_fn: Option<Box<dyn Fn(Event) + Send>>,
|
||||
eof: bool,
|
||||
|
||||
// PES output
|
||||
// Cumulative bytes successfully read from the source. Drives
|
||||
// EventKind::BytesRead emission and autorip's per-device progress.
|
||||
bytes_read_total: u64,
|
||||
// Pre-computed total of all extents in bytes (or 0 if extents are
|
||||
// empty). Carried in EventKind::BytesRead.total so consumers can show
|
||||
// a percent without a separate API call.
|
||||
bytes_total_extents: u64,
|
||||
|
||||
// PES output — single-threaded inline demux + codec parse. The
|
||||
// pipeline-mode mux (3-stage threaded) lives in
|
||||
// [`super::pipelined_stream::PipelinedPesStream`]; this type is
|
||||
// the legacy in-thread path for live-disc reads where adaptive
|
||||
// batch retry on bad sectors lives in `fill_extents`.
|
||||
ts_demuxer: Option<super::ts::TsDemuxer>,
|
||||
ps_demuxer: Option<super::ps::PsDemuxer>,
|
||||
parsers: Vec<(u16, Box<dyn super::codec::CodecParser>)>,
|
||||
@@ -138,17 +164,25 @@ pub struct DiscStream {
|
||||
impl DiscStream {
|
||||
/// Create a disc stream from any sector reader.
|
||||
///
|
||||
/// Works with physical drives and ISO files — both implement SectorReader.
|
||||
/// Works with physical drives and ISO files — both implement SectorSource.
|
||||
/// The caller opens the source, scans for titles/keys, and passes them in.
|
||||
/// The stream handles demuxing, decryption, and codec parsing internally.
|
||||
pub fn new(
|
||||
reader: Box<dyn SectorReader>,
|
||||
reader: Box<dyn SectorSource>,
|
||||
title: DiscTitle,
|
||||
decrypt_keys: crate::decrypt::DecryptKeys,
|
||||
batch_sectors: u16,
|
||||
content_format: crate::disc::ContentFormat,
|
||||
) -> Self {
|
||||
let extents = title.extents.clone();
|
||||
let bytes_total_extents: u64 = extents.iter().map(|e| e.sector_count as u64 * 2048).sum();
|
||||
|
||||
// Debug log reader type at construction — critical for diagnosing mux reading from drive instead of ISO
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"DiscStream constructed with reader type: {}",
|
||||
std::any::type_name::<dyn SectorSource>()
|
||||
);
|
||||
|
||||
let mut pids = Vec::new();
|
||||
let mut parsers = Vec::new();
|
||||
@@ -179,7 +213,11 @@ impl DiscStream {
|
||||
}
|
||||
|
||||
Self {
|
||||
reader,
|
||||
// Wrap the input reader in DecryptingSectorSource so the
|
||||
// internal fill_extents path sees plaintext bytes. For
|
||||
// DecryptKeys::None (unencrypted / raw / test fixtures)
|
||||
// the decorator is a pass-through.
|
||||
reader: DecryptingSectorSource::new(reader, decrypt_keys.clone()),
|
||||
title,
|
||||
disc: None,
|
||||
decrypt_keys,
|
||||
@@ -194,6 +232,8 @@ impl DiscStream {
|
||||
halt: None,
|
||||
event_fn: None,
|
||||
eof: false,
|
||||
bytes_read_total: 0,
|
||||
bytes_total_extents,
|
||||
ts_demuxer,
|
||||
ps_demuxer,
|
||||
parsers,
|
||||
@@ -207,19 +247,39 @@ impl DiscStream {
|
||||
self.event_fn = Some(Box::new(f));
|
||||
}
|
||||
|
||||
/// Share a halt flag — typically from `Drive::halt_flag()`. When raised,
|
||||
/// the next read-retry boundary inside fill_extents returns Err(Halted)
|
||||
/// instead of continuing. Required for Stop to work during dense bad-sector
|
||||
/// regions (where read() loops internally waiting for enough clean data to
|
||||
/// emit a frame and would otherwise never check an external stop signal).
|
||||
/// Constructor-time builder: attach a [`Halt`] token so that when
|
||||
/// any clone is cancelled, the next read-retry boundary inside
|
||||
/// `fill_extents` returns `Err(Halted)`. Required for Stop to work
|
||||
/// during dense bad-sector regions (where the outer PES read() loop
|
||||
/// can spend minutes inside fill_extents before emitting a frame).
|
||||
///
|
||||
/// Preferred over the post-hoc [`DiscStream::set_halt`] bridge —
|
||||
/// pass the same `Halt` clone you hand to sweep / patch / mux so
|
||||
/// every phase observes a single Stop signal.
|
||||
pub fn with_halt(mut self, halt: Halt) -> Self {
|
||||
self.halt = Some(halt);
|
||||
self
|
||||
}
|
||||
|
||||
/// Bridge for callers that haven't migrated to the
|
||||
/// [`DiscStream::with_halt`] constructor-time path yet. Wraps the
|
||||
/// supplied `Arc<AtomicBool>` as a [`Halt`] (`Halt::from_arc`) and
|
||||
/// stores it in the same internal slot, so a halt installed via
|
||||
/// either entry point goes through one halt-check inside
|
||||
/// `fill_extents`. Calling `set_halt` after `with_halt` (or vice
|
||||
/// versa) replaces the previous token with the new one.
|
||||
#[deprecated(
|
||||
since = "0.18.0",
|
||||
note = "use `DiscStream::with_halt(Halt)` at construction instead"
|
||||
)]
|
||||
pub fn set_halt(&mut self, flag: Arc<AtomicBool>) {
|
||||
self.halt = Some(flag);
|
||||
self.halt = Some(Halt::from_arc(flag));
|
||||
}
|
||||
|
||||
fn is_halted(&self) -> bool {
|
||||
self.halt
|
||||
.as_ref()
|
||||
.map(|h| h.load(Ordering::Relaxed))
|
||||
.map(|h| h.is_cancelled())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
@@ -229,9 +289,12 @@ impl DiscStream {
|
||||
}
|
||||
}
|
||||
|
||||
/// Skip decryption — return raw encrypted bytes.
|
||||
/// Skip decryption — return raw encrypted bytes. Updates both
|
||||
/// the metadata-side key field and the wrapped reader's keys so
|
||||
/// subsequent `read_sectors` calls become a pass-through.
|
||||
pub fn set_raw(&mut self) {
|
||||
self.decrypt_keys = crate::decrypt::DecryptKeys::None;
|
||||
self.reader.set_keys(crate::decrypt::DecryptKeys::None);
|
||||
}
|
||||
|
||||
/// Get the scanned Disc (for listing all titles).
|
||||
@@ -263,10 +326,19 @@ impl DiscStream {
|
||||
// this loop can spend minutes shrinking and skipping sectors; without
|
||||
// the check, Stop wouldn't take effect until the outer PES read() loop
|
||||
// finally emits a frame, which may never happen.
|
||||
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
loop {
|
||||
if self.is_halted() {
|
||||
return Err(crate::error::Error::Halted.into());
|
||||
}
|
||||
|
||||
// Debug: log slow reads during mux — helps diagnose stalls
|
||||
if cfg!(debug_assertions) && start_time.elapsed().as_secs() > 5 {
|
||||
tracing::debug!(target: "mux", "fill_extents waiting at LBA {} ({}s elapsed, sectors={})", lba, start_time.elapsed().as_secs(), remaining);
|
||||
}
|
||||
|
||||
let mut sectors = remaining.min(self.adaptive.current() as u32) as u16;
|
||||
// Align to 3-sector AACS units when possible. Partial units at
|
||||
// extent boundaries are safely handled by decrypt_sectors().
|
||||
@@ -287,6 +359,11 @@ impl DiscStream {
|
||||
}
|
||||
self.buf_valid = bytes;
|
||||
self.current_offset += sectors as u32;
|
||||
self.bytes_read_total = self.bytes_read_total.saturating_add(bytes as u64);
|
||||
self.emit(EventKind::BytesRead {
|
||||
bytes: self.bytes_read_total,
|
||||
total: self.bytes_total_extents,
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -301,7 +378,18 @@ impl DiscStream {
|
||||
self.current_offset += 1;
|
||||
break;
|
||||
} else {
|
||||
return Err(crate::error::Error::DiscRead { sector: lba as u64 }.into());
|
||||
let err = self
|
||||
.reader
|
||||
.read_sectors(lba, sectors, &mut self.read_buf[..2048], false)
|
||||
.err();
|
||||
let (status, sense) =
|
||||
err.as_ref().map(extract_scsi_context).unwrap_or((0, None));
|
||||
return Err(crate::error::Error::DiscRead {
|
||||
sector: lba as u64,
|
||||
status: Some(status),
|
||||
sense,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -319,6 +407,71 @@ impl DiscStream {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-stage profiling state — populated only when `FREEMKV_PROFILE`
|
||||
/// is set. Dumps a percentage breakdown to stderr every
|
||||
/// [`PROFILE_INTERVAL`]. Zero overhead in normal runs (Option check
|
||||
/// is the only added cost).
|
||||
struct StageProf {
|
||||
started: std::time::Instant,
|
||||
last_dump: std::time::Instant,
|
||||
fill_ns: u128,
|
||||
feed_ns: u128,
|
||||
consume_ns: u128,
|
||||
bytes_in: u64,
|
||||
}
|
||||
|
||||
const PROFILE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
thread_local! {
|
||||
static STAGE_PROF: std::cell::RefCell<Option<StageProf>> = const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
fn prof_active() -> bool {
|
||||
std::env::var_os("FREEMKV_PROFILE").is_some()
|
||||
}
|
||||
|
||||
fn prof_tick(stage: &str, ns: u128, bytes: u64) {
|
||||
STAGE_PROF.with(|cell| {
|
||||
let mut slot = cell.borrow_mut();
|
||||
if slot.is_none() {
|
||||
if !prof_active() {
|
||||
return;
|
||||
}
|
||||
let now = std::time::Instant::now();
|
||||
*slot = Some(StageProf {
|
||||
started: now,
|
||||
last_dump: now,
|
||||
fill_ns: 0,
|
||||
feed_ns: 0,
|
||||
consume_ns: 0,
|
||||
bytes_in: 0,
|
||||
});
|
||||
}
|
||||
let p = slot.as_mut().unwrap();
|
||||
match stage {
|
||||
"fill" => p.fill_ns += ns,
|
||||
"feed" => p.feed_ns += ns,
|
||||
"consume" => p.consume_ns += ns,
|
||||
_ => {}
|
||||
}
|
||||
p.bytes_in += bytes;
|
||||
let now = std::time::Instant::now();
|
||||
if now.duration_since(p.last_dump) < PROFILE_INTERVAL {
|
||||
return;
|
||||
}
|
||||
let elapsed_ms = now.duration_since(p.started).as_millis().max(1);
|
||||
let fill_pct = p.fill_ns / 10_000 / elapsed_ms;
|
||||
let feed_pct = p.feed_ns / 10_000 / elapsed_ms;
|
||||
let consume_pct = p.consume_ns / 10_000 / elapsed_ms;
|
||||
let mbps = p.bytes_in as u128 * 1000 / 1_000_000 / elapsed_ms;
|
||||
eprintln!(
|
||||
"[profile] elapsed={}ms in={}MB/s fill={}% feed={}% consume={}%",
|
||||
elapsed_ms, mbps, fill_pct, feed_pct, consume_pct,
|
||||
);
|
||||
p.last_dump = now;
|
||||
});
|
||||
}
|
||||
|
||||
impl crate::pes::Stream for DiscStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
if let Some(frame) = self.pending_frames.pop_front() {
|
||||
@@ -330,6 +483,7 @@ impl crate::pes::Stream for DiscStream {
|
||||
}
|
||||
|
||||
loop {
|
||||
let t0 = std::time::Instant::now();
|
||||
if !self.fill_extents()? {
|
||||
self.eof = true;
|
||||
// Flush demuxer — last PES packet may still be in the assembler
|
||||
@@ -391,29 +545,51 @@ impl crate::pes::Stream for DiscStream {
|
||||
}
|
||||
|
||||
let bytes = self.buf_valid;
|
||||
if let Err(e) =
|
||||
crate::decrypt::decrypt_sectors(&mut self.read_buf[..bytes], &self.decrypt_keys, 0)
|
||||
{
|
||||
return Err(e.into());
|
||||
}
|
||||
let t1 = std::time::Instant::now();
|
||||
prof_tick("fill", t1.duration_since(t0).as_nanos(), bytes as u64);
|
||||
// Plaintext: the wrapped reader (DecryptingSectorSource)
|
||||
// applied AACS / CSS in-place during fill_extents'
|
||||
// read_sectors call. The pre-0.18 inline decrypt step
|
||||
// lived here.
|
||||
|
||||
if let Some(ref mut demuxer) = self.ts_demuxer {
|
||||
let packets = demuxer.feed(&self.read_buf[..bytes]);
|
||||
for pes in &packets {
|
||||
if let Some((_, track)) =
|
||||
self.pid_to_track.iter().find(|(pid, _)| *pid == pes.pid)
|
||||
let t2 = std::time::Instant::now();
|
||||
prof_tick("feed", t2.duration_since(t1).as_nanos(), 0);
|
||||
let skip_parse = std::env::var_os("FREEMKV_SKIP_PARSE").is_some();
|
||||
for pes in packets {
|
||||
if let Some((_, track)) = self
|
||||
.pid_to_track
|
||||
.iter()
|
||||
.find(|(pid, _)| *pid == pes.pid)
|
||||
.copied()
|
||||
{
|
||||
if let Some((_, parser)) =
|
||||
if skip_parse {
|
||||
// Profiling escape hatch — bypass the codec
|
||||
// parser and pass the raw PES bytes straight
|
||||
// through as a single PesFrame. Lets us
|
||||
// attribute consumer-thread time to
|
||||
// "demux + framing" vs "codec parse".
|
||||
self.pending_frames.push_back(crate::pes::PesFrame {
|
||||
track,
|
||||
pts: pes.pts.map(super::codec::pts_to_ns).unwrap_or(0),
|
||||
keyframe: false,
|
||||
data: pes.data,
|
||||
duration_ns: None,
|
||||
});
|
||||
} else if let Some((_, parser)) =
|
||||
self.parsers.iter_mut().find(|(pid, _)| *pid == pes.pid)
|
||||
{
|
||||
for frame in parser.parse(pes) {
|
||||
for frame in parser.parse(&pes) {
|
||||
self.pending_frames.push_back(
|
||||
crate::pes::PesFrame::from_codec_frame(*track, frame),
|
||||
crate::pes::PesFrame::from_codec_frame(track, frame),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let t3 = std::time::Instant::now();
|
||||
prof_tick("consume", t3.duration_since(t2).as_nanos(), 0);
|
||||
} else if let Some(ref mut demuxer) = self.ps_demuxer {
|
||||
let packets = demuxer.feed(&self.read_buf[..bytes]);
|
||||
for ps in &packets {
|
||||
@@ -487,6 +663,13 @@ impl crate::pes::Stream for DiscStream {
|
||||
}
|
||||
|
||||
fn headers_ready(&self) -> bool {
|
||||
// FREEMKV_SKIP_PARSE bypasses codec parsers entirely for
|
||||
// bottleneck profiling, so codec_private is never populated.
|
||||
// Pretend headers are ready immediately in that mode so the
|
||||
// CLI loop doesn't hang waiting for them.
|
||||
if std::env::var_os("FREEMKV_SKIP_PARSE").is_some() {
|
||||
return true;
|
||||
}
|
||||
for (idx, s) in self.title.streams.iter().enumerate() {
|
||||
if let crate::disc::Stream::Video(v) = s {
|
||||
if !v.secondary && self.codec_private(idx).is_none() {
|
||||
@@ -496,4 +679,145 @@ impl crate::pes::Stream for DiscStream {
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn errors(&self) -> u64 {
|
||||
self.errors
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
//! `DiscStream` is the only read-only `Stream` impl in tree (every
|
||||
//! other concrete impl in `mux/*` is bidirectional or write-only).
|
||||
//! These tests lock down a static `Send` assertion plus a
|
||||
//! `Box<dyn Stream>` round trip exercising every method through the
|
||||
//! trait object, so future Send-breaking edits to `DiscStream`'s
|
||||
//! interior types fail at compile time.
|
||||
use super::*;
|
||||
use crate::disc::{ContentFormat, DiscTitle};
|
||||
use crate::pes::Stream;
|
||||
|
||||
/// Static-assert `DiscStream: Send`. The `Stream` trait has `Send` as a
|
||||
/// supertrait — if a future field on `DiscStream` is non-`Send` (e.g.
|
||||
/// a `Box<dyn Read>` instead of `Box<dyn SectorSource>`), this fails
|
||||
/// at compile time, before the runtime trait-object test below.
|
||||
fn _assert_disc_stream_is_send() {
|
||||
fn requires_send<T: Send>() {}
|
||||
requires_send::<DiscStream>();
|
||||
}
|
||||
|
||||
/// Trivial `SectorSource` that yields zeroed sectors. Empty title means
|
||||
/// the demuxer produces no PES frames, so `read()` walks the extents to
|
||||
/// EOF and returns `Ok(None)`. That's enough to exercise the trait-object
|
||||
/// dispatch — the goal here is the bridge, not the demuxer.
|
||||
struct ZeroReader {
|
||||
capacity: u32,
|
||||
}
|
||||
|
||||
impl crate::sector::SectorSource for ZeroReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
_lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> crate::error::Result<usize> {
|
||||
let bytes = count as usize * 2048;
|
||||
buf[..bytes].fill(0);
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
|
||||
fn synthetic_title(sector_count: u32) -> DiscTitle {
|
||||
DiscTitle {
|
||||
extents: vec![crate::disc::Extent {
|
||||
start_lba: 0,
|
||||
sector_count,
|
||||
}],
|
||||
..DiscTitle::empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Smallest credible witness that `DiscStream` flows through `dyn Stream`:
|
||||
/// build a `Box<dyn Stream>`, drive `read()` to EOF, exercise `info()` /
|
||||
/// `headers_ready()` / `codec_private()` through the trait object.
|
||||
#[test]
|
||||
fn stream_via_dyn_object() {
|
||||
let reader = ZeroReader { capacity: 8 };
|
||||
let title = synthetic_title(8);
|
||||
let stream = DiscStream::new(
|
||||
Box::new(reader),
|
||||
title,
|
||||
crate::decrypt::DecryptKeys::None,
|
||||
8,
|
||||
ContentFormat::BdTs,
|
||||
);
|
||||
|
||||
let mut src: Box<dyn Stream> = Box::new(stream);
|
||||
|
||||
// Empty-title fixture has no streams configured, so headers are
|
||||
// trivially ready and codec_private() yields nothing on track 0.
|
||||
assert!(src.headers_ready());
|
||||
assert!(src.codec_private(0).is_none());
|
||||
let _ = src.info();
|
||||
|
||||
// Drive read() to EOF through the trait object — empty-title fixture
|
||||
// produces no frames, but the call still routes through the blanket
|
||||
// dispatch into Stream::read.
|
||||
let mut frames = 0usize;
|
||||
while src.read().expect("read").is_some() {
|
||||
frames += 1;
|
||||
if frames > 1024 {
|
||||
panic!("unexpected unbounded frame stream from empty title");
|
||||
}
|
||||
}
|
||||
assert_eq!(frames, 0);
|
||||
}
|
||||
|
||||
/// `is_halted()` must observe a cancellation signal regardless of
|
||||
/// which entry point installed the token. The deprecated
|
||||
/// `set_halt(Arc<AtomicBool>)` and the new `with_halt(Halt)` are
|
||||
/// two views over one slot — flipping either bit must cause the
|
||||
/// next `fill_extents` retry boundary to bail.
|
||||
#[test]
|
||||
fn halt_via_with_halt_observed_by_is_halted() {
|
||||
let halt = Halt::new();
|
||||
let stream = DiscStream::new(
|
||||
Box::new(ZeroReader { capacity: 8 }),
|
||||
synthetic_title(8),
|
||||
crate::decrypt::DecryptKeys::None,
|
||||
8,
|
||||
crate::disc::ContentFormat::BdTs,
|
||||
)
|
||||
.with_halt(halt.clone());
|
||||
assert!(!stream.is_halted());
|
||||
halt.cancel();
|
||||
assert!(
|
||||
stream.is_halted(),
|
||||
"with_halt token cancellation must be observed by is_halted()"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_via_set_halt_bridge_observed_by_is_halted() {
|
||||
let arc = Arc::new(AtomicBool::new(false));
|
||||
let mut stream = DiscStream::new(
|
||||
Box::new(ZeroReader { capacity: 8 }),
|
||||
synthetic_title(8),
|
||||
crate::decrypt::DecryptKeys::None,
|
||||
8,
|
||||
crate::disc::ContentFormat::BdTs,
|
||||
);
|
||||
stream.set_halt(arc.clone());
|
||||
assert!(!stream.is_halted());
|
||||
arc.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(
|
||||
stream.is_halted(),
|
||||
"set_halt(Arc<AtomicBool>) bridge must observe Arc-side flips"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -321,6 +321,12 @@ pub const EBML_DOC_TYPE_READ_VERSION: u32 = 0x4285;
|
||||
// Segment
|
||||
pub const SEGMENT: u32 = 0x1853_8067;
|
||||
|
||||
// SeekHead
|
||||
pub const SEEK_HEAD: u32 = 0x114D_9B74;
|
||||
pub const SEEK: u32 = 0x4DBB;
|
||||
pub const SEEK_ID: u32 = 0x53AB;
|
||||
pub const SEEK_POSITION: u32 = 0x53AC;
|
||||
|
||||
// Segment Info
|
||||
pub const INFO: u32 = 0x1549_A966;
|
||||
pub const TIMESTAMP_SCALE: u32 = 0x2A_D7B1;
|
||||
@@ -366,6 +372,9 @@ pub const BIT_DEPTH: u32 = 0x6264;
|
||||
pub const CLUSTER: u32 = 0x1F43_B675;
|
||||
pub const CLUSTER_TIMESTAMP: u32 = 0xE7;
|
||||
pub const SIMPLE_BLOCK: u32 = 0xA3;
|
||||
pub const BLOCK_GROUP: u32 = 0xA0;
|
||||
pub const BLOCK: u32 = 0xA1;
|
||||
pub const BLOCK_DURATION: u32 = 0x9B;
|
||||
|
||||
// Cues
|
||||
pub const CUES: u32 = 0x1C53_BB6B;
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
//! Fragmented MP4 muxer — **stub** for Phase 3.
|
||||
//!
|
||||
//! Goal: ISO/IEC 14496-12 fragmented MP4 (`ftyp` + `moov` init segment,
|
||||
//! then a sequence of `moof+mdat` media fragments) targeting a
|
||||
//! [`SequentialSink`](crate::io::sink::SequentialSink). DASH-friendly,
|
||||
//! no Cues backpatch.
|
||||
//!
|
||||
//! Status (v0.21.0 Phase 3): **STUB**. We ship the init segment
|
||||
//! (`ftyp` + a minimal HEVC `moov` skeleton with one video track) so
|
||||
//! the muxer's shape and call site are validated, but media fragments
|
||||
//! are NOT yet emitted — calls to [`Fmp4Mux::write_video`] currently
|
||||
//! accumulate frames into an internal buffer and discard them on
|
||||
//! [`Fmp4Mux::finish`].
|
||||
//!
|
||||
//! ## What's TODO (tracked in Phase 4 / v0.22.0 scope)
|
||||
//!
|
||||
//! - `moof` box: `mfhd` (sequence_number) + `traf` (`tfhd` + `tfdt`
|
||||
//! + `trun` with sample sizes, durations, flags, composition offsets).
|
||||
//! - `mdat` box: concatenated sample data.
|
||||
//! - Fragment cadence: one fragment per GOP or every N seconds,
|
||||
//! whichever comes first.
|
||||
//! - HEVC `hvcC` box inside `moov.trak.mdia.minf.stbl.stsd` so the
|
||||
//! init segment is self-describing.
|
||||
//! - Sample-flags computation (sync vs. delta, depends_on, etc.).
|
||||
//! - Edit lists / fragment_duration for accurate seeking.
|
||||
//!
|
||||
//! Reference: ISO/IEC 14496-12 §8 (Movie Fragments).
|
||||
|
||||
use std::io::{self, Write};
|
||||
|
||||
// Box type literals — four-character codes per ISO/IEC 14496-12 §4.2.
|
||||
|
||||
const FTYP: [u8; 4] = *b"ftyp";
|
||||
const MOOV: [u8; 4] = *b"moov";
|
||||
const MVHD: [u8; 4] = *b"mvhd";
|
||||
const TRAK: [u8; 4] = *b"trak";
|
||||
const TKHD: [u8; 4] = *b"tkhd";
|
||||
const MDIA: [u8; 4] = *b"mdia";
|
||||
const MDHD: [u8; 4] = *b"mdhd";
|
||||
const HDLR: [u8; 4] = *b"hdlr";
|
||||
const MINF: [u8; 4] = *b"minf";
|
||||
const VMHD: [u8; 4] = *b"vmhd";
|
||||
const DINF: [u8; 4] = *b"dinf";
|
||||
const DREF: [u8; 4] = *b"dref";
|
||||
const URL_: [u8; 4] = *b"url ";
|
||||
const STBL: [u8; 4] = *b"stbl";
|
||||
const STSD: [u8; 4] = *b"stsd";
|
||||
const STTS: [u8; 4] = *b"stts";
|
||||
const STSC: [u8; 4] = *b"stsc";
|
||||
const STSZ: [u8; 4] = *b"stsz";
|
||||
const STCO: [u8; 4] = *b"stco";
|
||||
const MVEX: [u8; 4] = *b"mvex";
|
||||
const TREX: [u8; 4] = *b"trex";
|
||||
|
||||
/// Default movie timescale — 90 kHz lines up with MPEG-TS PTS and the
|
||||
/// HEVC SPS `vui_time_scale` for film content, simplifying the math
|
||||
/// when fragment emission lands.
|
||||
const MOVIE_TIMESCALE: u32 = 90_000;
|
||||
/// Video track ID. fMP4 init segments conventionally use track_ID=1
|
||||
/// for the primary video track; a single-track DASH representation has
|
||||
/// no reason to deviate.
|
||||
const VIDEO_TRACK_ID: u32 = 1;
|
||||
|
||||
/// Fragmented MP4 muxer — stub.
|
||||
///
|
||||
/// See the module-level doc comment for what is and isn't shipped in
|
||||
/// this stub.
|
||||
pub struct Fmp4Mux<W: Write> {
|
||||
writer: W,
|
||||
header_written: bool,
|
||||
/// Pending frames — held for the future fragment-emit path. The
|
||||
/// stub drops these on `finish` but keeping them around lets the
|
||||
/// post-stub work re-attach without changing the public API.
|
||||
pending: Vec<PendingSample>,
|
||||
/// hvcC bytes, if provided. Embedded in the `moov.…stsd.hvc1.hvcC`
|
||||
/// box once that path lands.
|
||||
#[allow(dead_code)]
|
||||
codec_private: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
struct PendingSample {
|
||||
#[allow(dead_code)]
|
||||
pts_ns: i64,
|
||||
#[allow(dead_code)]
|
||||
keyframe: bool,
|
||||
#[allow(dead_code)]
|
||||
data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl<W: Write> Fmp4Mux<W> {
|
||||
pub fn new(writer: W) -> Self {
|
||||
Self {
|
||||
writer,
|
||||
header_written: false,
|
||||
pending: Vec::new(),
|
||||
codec_private: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Provide the `HEVCDecoderConfigurationRecord` for the video track.
|
||||
/// The stub stores it but doesn't yet embed it in `moov` — that's
|
||||
/// part of the post-stub work.
|
||||
pub fn set_video_codec_private(&mut self, hvcc: Vec<u8>) {
|
||||
self.codec_private = Some(hvcc);
|
||||
}
|
||||
|
||||
/// Write one video PES frame.
|
||||
///
|
||||
/// **Stub behaviour:** the first call emits the init segment
|
||||
/// (`ftyp` + `moov`) so any consumer that just wants the shape can
|
||||
/// receive it. Subsequent calls accumulate frames in memory for
|
||||
/// the future fragmenting path; **no media bytes are written yet**.
|
||||
pub fn write_video(&mut self, pts_ns: i64, keyframe: bool, data: &[u8]) -> io::Result<()> {
|
||||
if !self.header_written {
|
||||
self.write_init_segment()?;
|
||||
self.header_written = true;
|
||||
}
|
||||
// TODO(0.22.0): emit one `moof+mdat` per GOP. For now stash the
|
||||
// frame so the future patch can hot-wire emission without API
|
||||
// churn.
|
||||
self.pending.push(PendingSample {
|
||||
pts_ns,
|
||||
keyframe,
|
||||
data: data.to_vec(),
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush. The stub additionally drops accumulated `pending` frames.
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
// TODO(0.22.0): emit final fragment from pending; today the
|
||||
// stub just clears the buffer to release memory.
|
||||
self.pending.clear();
|
||||
self.writer.flush()
|
||||
}
|
||||
|
||||
fn write_init_segment(&mut self) -> io::Result<()> {
|
||||
let ftyp = build_ftyp();
|
||||
let moov = build_moov();
|
||||
self.writer.write_all(&ftyp)?;
|
||||
self.writer.write_all(&moov)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the `ftyp` box. `major_brand = "iso6"`, `minor_version = 1`,
|
||||
/// compatible brands `iso6 dash msdh hvc1` — the same conservative set
|
||||
/// shaka-packager uses for HEVC-in-fMP4 outputs.
|
||||
fn build_ftyp() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(b"iso6");
|
||||
body.extend_from_slice(&1u32.to_be_bytes());
|
||||
body.extend_from_slice(b"iso6");
|
||||
body.extend_from_slice(b"dash");
|
||||
body.extend_from_slice(b"msdh");
|
||||
body.extend_from_slice(b"hvc1");
|
||||
wrap_box(&FTYP, &body)
|
||||
}
|
||||
|
||||
/// Build the `moov` box — minimal skeleton. Single video trak, no
|
||||
/// hvcC inside stsd yet (TODO: full hvc1 sample entry).
|
||||
fn build_moov() -> Vec<u8> {
|
||||
let mvhd = build_mvhd();
|
||||
let trak = build_video_trak();
|
||||
let mvex = build_mvex();
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&mvhd);
|
||||
body.extend_from_slice(&trak);
|
||||
body.extend_from_slice(&mvex);
|
||||
wrap_box(&MOOV, &body)
|
||||
}
|
||||
|
||||
fn build_mvhd() -> Vec<u8> {
|
||||
// Version 0, 100 bytes total body. Fields per ISO/IEC 14496-12 §8.2.2.
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0, 0, 0, 0]); // version + flags
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // creation_time
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // modification_time
|
||||
body.extend_from_slice(&MOVIE_TIMESCALE.to_be_bytes());
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // duration = 0 (fragmented)
|
||||
body.extend_from_slice(&0x0001_0000u32.to_be_bytes()); // rate 1.0
|
||||
body.extend_from_slice(&0x0100u16.to_be_bytes()); // volume 1.0
|
||||
body.extend_from_slice(&[0u8; 2]); // reserved
|
||||
body.extend_from_slice(&[0u8; 8]); // reserved
|
||||
// 3x3 identity transformation matrix in 16.16 fixed point.
|
||||
for v in [0x1_0000u32, 0, 0, 0, 0x1_0000, 0, 0, 0, 0x4000_0000] {
|
||||
body.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
body.extend_from_slice(&[0u8; 24]); // pre_defined[6]
|
||||
body.extend_from_slice(&2u32.to_be_bytes()); // next_track_ID (1 reserved for video)
|
||||
wrap_box(&MVHD, &body)
|
||||
}
|
||||
|
||||
fn build_video_trak() -> Vec<u8> {
|
||||
let tkhd = build_tkhd();
|
||||
let mdia = build_mdia();
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&tkhd);
|
||||
body.extend_from_slice(&mdia);
|
||||
wrap_box(&TRAK, &body)
|
||||
}
|
||||
|
||||
fn build_tkhd() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
// version=0 | flags=0x000007 (track_enabled | in_movie | in_preview)
|
||||
body.extend_from_slice(&[0, 0, 0, 7]);
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // creation_time
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // modification_time
|
||||
body.extend_from_slice(&VIDEO_TRACK_ID.to_be_bytes());
|
||||
body.extend_from_slice(&[0u8; 4]); // reserved
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // duration
|
||||
body.extend_from_slice(&[0u8; 8]); // reserved
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // layer
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // alternate_group
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // volume (video=0)
|
||||
body.extend_from_slice(&[0u8; 2]); // reserved
|
||||
// 3x3 identity matrix.
|
||||
for v in [0x1_0000u32, 0, 0, 0, 0x1_0000, 0, 0, 0, 0x4000_0000] {
|
||||
body.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
// width / height in 16.16 fixed point — placeholder 1920x1080.
|
||||
body.extend_from_slice(&(1920u32 << 16).to_be_bytes());
|
||||
body.extend_from_slice(&(1080u32 << 16).to_be_bytes());
|
||||
wrap_box(&TKHD, &body)
|
||||
}
|
||||
|
||||
fn build_mdia() -> Vec<u8> {
|
||||
let mdhd = build_mdhd();
|
||||
let hdlr = build_hdlr_vide();
|
||||
let minf = build_minf();
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&mdhd);
|
||||
body.extend_from_slice(&hdlr);
|
||||
body.extend_from_slice(&minf);
|
||||
wrap_box(&MDIA, &body)
|
||||
}
|
||||
|
||||
fn build_mdhd() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0, 0, 0, 0]); // version + flags
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // creation_time
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // modification_time
|
||||
body.extend_from_slice(&MOVIE_TIMESCALE.to_be_bytes());
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // duration
|
||||
// language: 'und' in 5-bit-per-char ISO 639-2 packed (bit 15 = 0).
|
||||
body.extend_from_slice(&[0x55, 0xC4]);
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // pre_defined
|
||||
wrap_box(&MDHD, &body)
|
||||
}
|
||||
|
||||
fn build_hdlr_vide() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0, 0, 0, 0]); // version + flags
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // pre_defined
|
||||
body.extend_from_slice(b"vide");
|
||||
body.extend_from_slice(&[0u8; 12]); // reserved
|
||||
body.extend_from_slice(b"VideoHandler\0");
|
||||
wrap_box(&HDLR, &body)
|
||||
}
|
||||
|
||||
fn build_minf() -> Vec<u8> {
|
||||
let vmhd = build_vmhd();
|
||||
let dinf = build_dinf();
|
||||
let stbl = build_stbl();
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&vmhd);
|
||||
body.extend_from_slice(&dinf);
|
||||
body.extend_from_slice(&stbl);
|
||||
wrap_box(&MINF, &body)
|
||||
}
|
||||
|
||||
fn build_vmhd() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0, 0, 0, 1]); // version + flags=1
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // graphicsmode
|
||||
body.extend_from_slice(&[0u8; 6]); // opcolor
|
||||
wrap_box(&VMHD, &body)
|
||||
}
|
||||
|
||||
fn build_dinf() -> Vec<u8> {
|
||||
let mut dref_body = Vec::new();
|
||||
dref_body.extend_from_slice(&[0, 0, 0, 0]);
|
||||
dref_body.extend_from_slice(&1u32.to_be_bytes()); // entry_count
|
||||
// url with flags=1 (self-contained) and zero name.
|
||||
let url_body = [0u8, 0, 0, 1];
|
||||
dref_body.extend_from_slice(&wrap_box(&URL_, &url_body));
|
||||
let dref = wrap_box(&DREF, &dref_body);
|
||||
wrap_box(&DINF, &dref)
|
||||
}
|
||||
|
||||
fn build_stbl() -> Vec<u8> {
|
||||
// Stub stsd: empty sample description (zero entries). Replace with
|
||||
// hvc1+hvcC once the fragmenting path lands so the init segment is
|
||||
// actually decodable.
|
||||
let mut stsd_body = Vec::new();
|
||||
stsd_body.extend_from_slice(&[0, 0, 0, 0]);
|
||||
stsd_body.extend_from_slice(&0u32.to_be_bytes()); // entry_count
|
||||
let stsd = wrap_box(&STSD, &stsd_body);
|
||||
|
||||
// Empty stts/stsc/stsz/stco — fragmented init has no samples here.
|
||||
let stts = wrap_box(&STTS, &[0, 0, 0, 0, 0, 0, 0, 0]); // version+flags, count=0
|
||||
let stsc = wrap_box(&STSC, &[0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
let stsz = wrap_box(
|
||||
&STSZ,
|
||||
&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // version+flags, sample_size=0, count=0
|
||||
);
|
||||
let stco = wrap_box(&STCO, &[0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&stsd);
|
||||
body.extend_from_slice(&stts);
|
||||
body.extend_from_slice(&stsc);
|
||||
body.extend_from_slice(&stsz);
|
||||
body.extend_from_slice(&stco);
|
||||
wrap_box(&STBL, &body)
|
||||
}
|
||||
|
||||
fn build_mvex() -> Vec<u8> {
|
||||
// trex: track_ID=1, default_sample_description_index=1, others=0.
|
||||
let mut trex_body = Vec::new();
|
||||
trex_body.extend_from_slice(&[0, 0, 0, 0]); // version + flags
|
||||
trex_body.extend_from_slice(&VIDEO_TRACK_ID.to_be_bytes());
|
||||
trex_body.extend_from_slice(&1u32.to_be_bytes()); // default_sample_description_index
|
||||
trex_body.extend_from_slice(&0u32.to_be_bytes()); // default_sample_duration
|
||||
trex_body.extend_from_slice(&0u32.to_be_bytes()); // default_sample_size
|
||||
trex_body.extend_from_slice(&0u32.to_be_bytes()); // default_sample_flags
|
||||
let trex = wrap_box(&TREX, &trex_body);
|
||||
wrap_box(&MVEX, &trex)
|
||||
}
|
||||
|
||||
/// Wrap a box body in `[size:u32-BE][type:4]`. Suitable for any body
|
||||
/// that fits in u32; oversized boxes (size > 4 GiB) need the 64-bit
|
||||
/// large-size extension which we don't generate in the stub.
|
||||
fn wrap_box(box_type: &[u8; 4], body: &[u8]) -> Vec<u8> {
|
||||
let size = (body.len() + 8) as u32;
|
||||
let mut out = Vec::with_capacity(body.len() + 8);
|
||||
out.extend_from_slice(&size.to_be_bytes());
|
||||
out.extend_from_slice(box_type);
|
||||
out.extend_from_slice(body);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Decode the first box's size + type from `buf`.
|
||||
fn read_box_header(buf: &[u8]) -> (u32, [u8; 4]) {
|
||||
let size = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]);
|
||||
let bt = [buf[4], buf[5], buf[6], buf[7]];
|
||||
(size, bt)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn init_segment_starts_with_ftyp_then_moov() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = Fmp4Mux::new(&mut sink);
|
||||
// Trigger init emission via a single (stubbed) write.
|
||||
mux.write_video(0, true, &[0x00, 0x00, 0x00, 0x01, 0x40])
|
||||
.unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
let (ftyp_size, ftyp_type) = read_box_header(&sink);
|
||||
assert_eq!(&ftyp_type, b"ftyp");
|
||||
assert!(ftyp_size >= 24, "ftyp too small: {ftyp_size}");
|
||||
|
||||
let (moov_size, moov_type) = read_box_header(&sink[ftyp_size as usize..]);
|
||||
assert_eq!(&moov_type, b"moov");
|
||||
assert!(moov_size > 100, "moov skeleton too small: {moov_size}");
|
||||
|
||||
// Stub guarantee: no media bytes after the init segment.
|
||||
let total = ftyp_size as usize + moov_size as usize;
|
||||
assert_eq!(sink.len(), total, "stub leaked media bytes past moov");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn moov_contains_trak_mvex() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = Fmp4Mux::new(&mut sink);
|
||||
mux.write_video(0, true, &[]).unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(sink);
|
||||
|
||||
// Re-emit into a fresh buffer for parsing.
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
let mut mux2 = Fmp4Mux::new(&mut buf);
|
||||
mux2.write_video(0, true, &[]).unwrap();
|
||||
mux2.finish().unwrap();
|
||||
drop(mux2);
|
||||
|
||||
// Find moov payload start.
|
||||
let (ftyp_size, _) = read_box_header(&buf);
|
||||
let moov_start = ftyp_size as usize;
|
||||
let (moov_size, _) = read_box_header(&buf[moov_start..]);
|
||||
let moov_payload = &buf[moov_start + 8..moov_start + moov_size as usize];
|
||||
|
||||
// Scan for the trak and mvex four-CC anywhere in the moov payload.
|
||||
let has_trak = moov_payload.windows(4).any(|w| w == b"trak");
|
||||
let has_mvex = moov_payload.windows(4).any(|w| w == b"mvex");
|
||||
assert!(has_trak, "moov missing trak");
|
||||
assert!(has_mvex, "moov missing mvex");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
//! HEVC (H.265) elementary stream muxer — Annex B byte stream.
|
||||
//!
|
||||
//! Consumes [`PesFrame`](crate::pes::PesFrame)s for a single video track
|
||||
//! and writes them as a raw `.hevc` / `.h265` Annex B byte stream:
|
||||
//! `00 00 00 01 | NAL_unit | 00 00 00 01 | NAL_unit | …` with no
|
||||
//! container framing.
|
||||
//!
|
||||
//! On the first frame the muxer emits the codec_private's VPS, SPS, PPS
|
||||
//! (parsed from a `HEVCDecoderConfigurationRecord` in
|
||||
//! `length-prefixed-in-hvcC` form), then converts each PES frame's
|
||||
//! length-prefixed NAL units to Annex B and writes them.
|
||||
//!
|
||||
//! Sequential-only — no Cues, no backpatch. Target sink is any
|
||||
//! [`SequentialSink`](crate::io::sink::SequentialSink): file, socket,
|
||||
//! pipe, anything `Write + Send`.
|
||||
|
||||
use std::io::{self, Write};
|
||||
|
||||
/// Annex B 4-byte start code.
|
||||
pub(crate) const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01];
|
||||
|
||||
/// HEVC NAL unit type bits live in `(byte0 >> 1) & 0x3F` in Annex B.
|
||||
/// We don't filter NAL types here — the muxer is format-only — but we
|
||||
/// keep the constant as documentation of the field layout.
|
||||
#[allow(dead_code)]
|
||||
const HEVC_NAL_TYPE_MASK: u8 = 0x3F;
|
||||
|
||||
/// Streaming HEVC Annex B muxer.
|
||||
///
|
||||
/// One instance per output stream. Tracks whether parameter sets have
|
||||
/// already been emitted so they're written exactly once at the head of
|
||||
/// the stream, mirroring the convention used by `ffmpeg -c:v copy -f
|
||||
/// hevc`.
|
||||
pub struct HevcMux<W: Write> {
|
||||
writer: W,
|
||||
/// `HEVCDecoderConfigurationRecord` payload (hvcC). Parsed lazily
|
||||
/// on the first `write_frame` so callers can set it after
|
||||
/// construction but before the first frame.
|
||||
codec_private: Option<Vec<u8>>,
|
||||
/// Set once VPS/SPS/PPS have been written to the stream. Subsequent
|
||||
/// frames write only their own NAL units.
|
||||
params_written: bool,
|
||||
}
|
||||
|
||||
impl<W: Write> HevcMux<W> {
|
||||
/// Construct over `writer`. The muxer does not impose any extra
|
||||
/// buffering of its own — the sink owns its write buffering policy
|
||||
/// (see [`LocalFileSink`](crate::io::sink::LocalFileSink) and
|
||||
/// [`SocketSink`](crate::io::sink::SocketSink)).
|
||||
pub fn new(writer: W) -> Self {
|
||||
Self {
|
||||
writer,
|
||||
codec_private: None,
|
||||
params_written: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Provide the `HEVCDecoderConfigurationRecord` (hvcC) so the muxer
|
||||
/// can prepend VPS/SPS/PPS Annex B NALs at stream start. Optional —
|
||||
/// if the PES frames already carry inline parameter sets (some
|
||||
/// upstream demuxers do this), skipping this call is fine.
|
||||
pub fn set_codec_private(&mut self, data: Vec<u8>) {
|
||||
self.codec_private = Some(data);
|
||||
}
|
||||
|
||||
/// Write one PES frame (= one access unit) as Annex B NAL units.
|
||||
///
|
||||
/// Input may be either:
|
||||
/// - Length-prefixed: `[u32-BE len][NAL bytes]` repeated. This is
|
||||
/// the form emitted by libfreemkv's HEVC parser (the MKV-native
|
||||
/// layout). Converted to Annex B.
|
||||
/// - Already Annex B: bytes containing `00 00 00 01` start codes
|
||||
/// anywhere in the buffer. Passed through unchanged.
|
||||
///
|
||||
/// `_pts_ns` is accepted for symmetry with other muxers but ignored
|
||||
/// — Annex B has no timing layer.
|
||||
pub fn write_frame(&mut self, _pts_ns: i64, data: &[u8]) -> io::Result<()> {
|
||||
if !self.params_written {
|
||||
if let Some(cp) = &self.codec_private {
|
||||
if let Some(params) = hvcc_to_annex_b(cp) {
|
||||
self.writer.write_all(¶ms)?;
|
||||
}
|
||||
}
|
||||
self.params_written = true;
|
||||
}
|
||||
let annex_b = length_prefixed_to_annex_b(data);
|
||||
self.writer.write_all(&annex_b)
|
||||
}
|
||||
|
||||
/// Flush the underlying writer. No trailer NAL is needed — an Annex
|
||||
/// B stream ends whenever the file/socket ends.
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
self.writer.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a `HEVCDecoderConfigurationRecord` (hvcC) into Annex B NAL
|
||||
/// units. Returns `Some(bytes)` if at least one NAL was extracted, else
|
||||
/// `None`.
|
||||
///
|
||||
/// Layout (per ISO/IEC 14496-15 §8.3.3.1.2):
|
||||
/// - 22-byte fixed header
|
||||
/// - byte 22 = `numOfArrays`
|
||||
/// - each array: `array_completeness:1 | reserved:1 | NAL_unit_type:6`,
|
||||
/// `numNalus:u16-BE`, then `numNalus` × `(nalUnitLength:u16-BE +
|
||||
/// NAL bytes)`.
|
||||
///
|
||||
/// We don't filter on NAL type — VPS (32), SPS (33), PPS (34), and any
|
||||
/// SEI arrays included in hvcC all get the same Annex B treatment.
|
||||
fn hvcc_to_annex_b(hvcc: &[u8]) -> Option<Vec<u8>> {
|
||||
if hvcc.len() < 23 {
|
||||
return None;
|
||||
}
|
||||
let num_arrays = hvcc[22] as usize;
|
||||
let mut out = Vec::new();
|
||||
let mut offset = 23;
|
||||
for _ in 0..num_arrays {
|
||||
if offset + 3 > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
offset += 1; // array_completeness + nal_type byte
|
||||
let num_nalus = u16::from_be_bytes([hvcc[offset], hvcc[offset + 1]]) as usize;
|
||||
offset += 2;
|
||||
for _ in 0..num_nalus {
|
||||
if offset + 2 > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
let nal_len = u16::from_be_bytes([hvcc[offset], hvcc[offset + 1]]) as usize;
|
||||
offset += 2;
|
||||
if offset + nal_len > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
out.extend_from_slice(&START_CODE);
|
||||
out.extend_from_slice(&hvcc[offset..offset + nal_len]);
|
||||
offset += nal_len;
|
||||
}
|
||||
}
|
||||
if out.is_empty() { None } else { Some(out) }
|
||||
}
|
||||
|
||||
/// Convert length-prefixed NAL units (`[u32-BE len][NAL]` repeated) to
|
||||
/// Annex B (`00 00 00 01 [NAL]` repeated).
|
||||
///
|
||||
/// If the input doesn't parse as length-prefixed (no valid lengths
|
||||
/// extracted), it's returned unchanged on the assumption that it's
|
||||
/// already Annex B — some upstream paths (raw HEVC ES from disc) pass
|
||||
/// Annex B straight through the PES layer.
|
||||
pub(crate) fn length_prefixed_to_annex_b(data: &[u8]) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(data.len() + (data.len() / 32));
|
||||
let mut offset = 0;
|
||||
while offset + 4 <= data.len() {
|
||||
let len = u32::from_be_bytes([
|
||||
data[offset],
|
||||
data[offset + 1],
|
||||
data[offset + 2],
|
||||
data[offset + 3],
|
||||
]) as usize;
|
||||
offset += 4;
|
||||
if offset + len > data.len() {
|
||||
// Mid-NAL truncation — fall through to the pass-through path
|
||||
// rather than emitting a half-NAL.
|
||||
return data.to_vec();
|
||||
}
|
||||
out.extend_from_slice(&START_CODE);
|
||||
out.extend_from_slice(&data[offset..offset + len]);
|
||||
offset += len;
|
||||
}
|
||||
if out.is_empty() && !data.is_empty() {
|
||||
// No length prefixes found — input is likely already Annex B.
|
||||
return data.to_vec();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn length_prefixed_converts_to_annex_b() {
|
||||
// Two NALs: [3-byte payload AA BB CC] and [2-byte payload DD EE].
|
||||
let mut buf = Vec::new();
|
||||
buf.extend_from_slice(&3u32.to_be_bytes());
|
||||
buf.extend_from_slice(&[0xAA, 0xBB, 0xCC]);
|
||||
buf.extend_from_slice(&2u32.to_be_bytes());
|
||||
buf.extend_from_slice(&[0xDD, 0xEE]);
|
||||
|
||||
let got = length_prefixed_to_annex_b(&buf);
|
||||
let want = [
|
||||
0x00, 0x00, 0x00, 0x01, 0xAA, 0xBB, 0xCC, // first NAL
|
||||
0x00, 0x00, 0x00, 0x01, 0xDD, 0xEE, // second NAL
|
||||
];
|
||||
assert_eq!(&got[..], &want[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn already_annex_b_passes_through_when_no_lengths_match() {
|
||||
// A buffer < 4 bytes can't parse a length prefix at all →
|
||||
// pass-through path triggers.
|
||||
let raw = [0xAA, 0xBB, 0xCC];
|
||||
let got = length_prefixed_to_annex_b(&raw);
|
||||
assert_eq!(&got[..], &raw[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mid_nal_truncation_returns_original() {
|
||||
// `[u32-BE 100][only 3 bytes]` — length prefix claims 100 bytes
|
||||
// but the input only has 3 after the prefix. We treat that as
|
||||
// malformed and pass the original buffer through so receivers
|
||||
// can attempt their own recovery.
|
||||
let mut raw = Vec::new();
|
||||
raw.extend_from_slice(&100u32.to_be_bytes());
|
||||
raw.extend_from_slice(&[0xAA, 0xBB, 0xCC]);
|
||||
let got = length_prefixed_to_annex_b(&raw);
|
||||
assert_eq!(&got[..], &raw[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hvcc_extracts_vps_sps_pps() {
|
||||
// Build a minimal-but-valid hvcC: 22-byte header, then 3 arrays
|
||||
// (VPS / SPS / PPS), each with 1 NAL of a 4-byte payload that
|
||||
// we can spot in the output.
|
||||
let mut hvcc = vec![0u8; 22];
|
||||
hvcc.push(3); // numOfArrays
|
||||
for (nal_type, payload) in [
|
||||
(32u8, [0x40, 0x01, 0x0C, 0x01]),
|
||||
(33, [0x42, 0x01, 0x01, 0x01]),
|
||||
(34, [0x44, 0x01, 0xC1, 0x72]),
|
||||
] {
|
||||
hvcc.push(nal_type & 0x3F);
|
||||
hvcc.extend_from_slice(&1u16.to_be_bytes()); // numNalus
|
||||
hvcc.extend_from_slice(&(payload.len() as u16).to_be_bytes());
|
||||
hvcc.extend_from_slice(&payload);
|
||||
}
|
||||
|
||||
let annex_b = hvcc_to_annex_b(&hvcc).expect("at least one NAL");
|
||||
// Three NALs × (4-byte start + 4-byte payload) = 24 bytes.
|
||||
assert_eq!(annex_b.len(), 24);
|
||||
assert_eq!(&annex_b[..4], &START_CODE);
|
||||
assert_eq!(&annex_b[8..12], &START_CODE);
|
||||
assert_eq!(&annex_b[16..20], &START_CODE);
|
||||
assert_eq!(annex_b[4], 0x40); // VPS first byte
|
||||
assert_eq!(annex_b[12], 0x42); // SPS first byte
|
||||
assert_eq!(annex_b[20], 0x44); // PPS first byte
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mux_writes_params_then_frames() {
|
||||
// Build hvcC with one SPS to verify params-once semantics.
|
||||
let mut hvcc = vec![0u8; 22];
|
||||
hvcc.push(1);
|
||||
hvcc.push(33);
|
||||
hvcc.extend_from_slice(&1u16.to_be_bytes());
|
||||
hvcc.extend_from_slice(&3u16.to_be_bytes());
|
||||
hvcc.extend_from_slice(&[0x42, 0x01, 0x01]);
|
||||
|
||||
let mut frame_data = Vec::new();
|
||||
frame_data.extend_from_slice(&2u32.to_be_bytes());
|
||||
frame_data.extend_from_slice(&[0xAA, 0xBB]);
|
||||
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = HevcMux::new(&mut sink);
|
||||
mux.set_codec_private(hvcc);
|
||||
mux.write_frame(0, &frame_data).unwrap();
|
||||
// Second frame — no SPS re-emission.
|
||||
mux.write_frame(40_000_000, &frame_data).unwrap();
|
||||
mux.finish().unwrap();
|
||||
|
||||
// SPS NAL (7 bytes) + 2× frame NAL (6 bytes) = 19 bytes.
|
||||
assert_eq!(sink.len(), 7 + 6 + 6);
|
||||
// Start codes at offsets 0 (SPS), 7 (frame1), 13 (frame2).
|
||||
assert_eq!(&sink[0..4], &START_CODE);
|
||||
assert_eq!(&sink[7..11], &START_CODE);
|
||||
assert_eq!(&sink[13..17], &START_CODE);
|
||||
assert_eq!(sink[4], 0x42);
|
||||
assert_eq!(sink[11], 0xAA);
|
||||
assert_eq!(sink[17], 0xAA);
|
||||
}
|
||||
}
|
||||
-101
@@ -1,101 +0,0 @@
|
||||
//! ISO sector reader — file-backed SectorReader for Blu-ray ISO images.
|
||||
//!
|
||||
//! An ISO is a flat image of 2048-byte sectors. Sector N starts at byte offset N * 2048.
|
||||
//! Used by DiscStream::open_iso() and Disc::scan_image().
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorReader;
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::Path;
|
||||
|
||||
const SECTOR_SIZE: u64 = 2048;
|
||||
|
||||
/// File-backed sector reader for ISO images.
|
||||
pub struct IsoSectorReader {
|
||||
file: File,
|
||||
capacity: u32,
|
||||
}
|
||||
|
||||
impl IsoSectorReader {
|
||||
pub fn open(path: &str) -> std::io::Result<Self> {
|
||||
let file = File::open(Path::new(path))?;
|
||||
let size = file.metadata()?.len();
|
||||
let sectors = size / SECTOR_SIZE;
|
||||
if sectors > u32::MAX as u64 {
|
||||
return Err(crate::error::Error::IsoTooLarge {
|
||||
path: path.to_string(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let capacity = sectors as u32;
|
||||
Ok(Self { file, capacity })
|
||||
}
|
||||
|
||||
pub fn capacity(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for IsoSectorReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
let bytes = count as usize * SECTOR_SIZE as usize;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(lba as u64 * SECTOR_SIZE))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.read_exact(&mut buf[..bytes])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn iso_reader_read_sectors() {
|
||||
let mut data = vec![0u8; 4 * SECTOR_SIZE as usize];
|
||||
for i in 0..4u8 {
|
||||
let offset = i as usize * SECTOR_SIZE as usize;
|
||||
data[offset] = i + 1;
|
||||
data[offset + 2047] = i + 100;
|
||||
}
|
||||
|
||||
let dir = std::env::temp_dir().join("freemkv_test_iso_read");
|
||||
std::fs::write(&dir, &data).unwrap();
|
||||
|
||||
let mut reader = IsoSectorReader::open(dir.to_str().unwrap()).unwrap();
|
||||
assert_eq!(reader.capacity(), 4);
|
||||
|
||||
let mut buf = [0u8; 2048];
|
||||
reader.read_sectors(0, 1, &mut buf, true).unwrap();
|
||||
assert_eq!(buf[0], 1);
|
||||
assert_eq!(buf[2047], 100);
|
||||
|
||||
reader.read_sectors(2, 1, &mut buf, true).unwrap();
|
||||
assert_eq!(buf[0], 3);
|
||||
assert_eq!(buf[2047], 102);
|
||||
|
||||
std::fs::remove_file(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn iso_reader_capacity() {
|
||||
let data = vec![0u8; 10 * SECTOR_SIZE as usize];
|
||||
let dir = std::env::temp_dir().join("freemkv_test_iso_cap");
|
||||
std::fs::write(&dir, &data).unwrap();
|
||||
|
||||
let reader = IsoSectorReader::open(dir.to_str().unwrap()).unwrap();
|
||||
assert_eq!(reader.capacity(), 10);
|
||||
|
||||
std::fs::remove_file(&dir).ok();
|
||||
}
|
||||
}
|
||||
+140
-228
@@ -1,63 +1,27 @@
|
||||
//! M2tsStream — BD transport stream with embedded metadata header.
|
||||
//! M2tsStream — BD transport stream write sink.
|
||||
//!
|
||||
//! Write: prepends FMKV metadata header, then muxes PES frames into BD-TS.
|
||||
//! Read: extracts metadata header (or scans PMT), then demuxes BD-TS into PES frames.
|
||||
//! Write: prepends FMKV metadata header, then muxes PES frames into
|
||||
//! BD-TS. The read direction lives on the pipeline highway —
|
||||
//! `m2ts://` URLs route through
|
||||
//! [`super::resolve::input`] → `build_m2ts_pipeline` →
|
||||
//! [`super::pipelined_stream::PipelinedPesStream`], so this type is
|
||||
//! write-only.
|
||||
|
||||
use super::{meta, ts};
|
||||
use super::meta;
|
||||
use crate::disc::{DiscTitle, Stream as DiscStream};
|
||||
use std::io::{self, Read, Write};
|
||||
use std::io::{self, Write};
|
||||
|
||||
type PesSetup = (
|
||||
Vec<u16>,
|
||||
Vec<(u16, Box<dyn super::codec::CodecParser>)>,
|
||||
Vec<(u16, usize)>,
|
||||
);
|
||||
|
||||
/// Size of initial scan buffer for PMT/stream detection.
|
||||
const SCAN_SIZE: usize = 1024 * 1024;
|
||||
|
||||
enum Mode {
|
||||
Write {
|
||||
muxer: super::tsmux::TsMuxer<Box<dyn Write>>,
|
||||
},
|
||||
Read {
|
||||
reader: Box<dyn Read>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Read as many bytes as possible into buf (multiple read calls if needed).
|
||||
/// Bounded by buf.len() — caller controls max bytes read.
|
||||
fn read_fill(r: &mut impl Read, buf: &mut [u8]) -> io::Result<usize> {
|
||||
let mut total = 0;
|
||||
while total < buf.len() {
|
||||
match r.read(&mut buf[total..]) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => total += n,
|
||||
Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
/// BD transport stream with embedded metadata.
|
||||
/// BD transport stream write sink with embedded FMKV metadata
|
||||
/// header.
|
||||
pub struct M2tsStream {
|
||||
disc_title: DiscTitle,
|
||||
mode: Mode,
|
||||
// PES support
|
||||
demuxer: Option<ts::TsDemuxer>,
|
||||
parsers: Vec<(u16, Box<dyn super::codec::CodecParser>)>,
|
||||
pending_frames: std::collections::VecDeque<crate::pes::PesFrame>,
|
||||
pid_to_track: Vec<(u16, usize)>,
|
||||
pes_eof: bool,
|
||||
/// Codec private data per stream (from FMKV header).
|
||||
stored_codec_privates: Vec<Option<Vec<u8>>>,
|
||||
muxer: super::tsmux::TsMuxer<Box<dyn Write + Send>>,
|
||||
}
|
||||
|
||||
impl M2tsStream {
|
||||
/// Create for writing PES frames → BD-TS output.
|
||||
/// Writes FMKV metadata header, then muxes PES frames into BD transport stream.
|
||||
pub fn create(mut writer: impl Write + 'static, title: &DiscTitle) -> io::Result<Self> {
|
||||
pub fn create(mut writer: impl Write + Send + 'static, title: &DiscTitle) -> io::Result<Self> {
|
||||
// Write FMKV metadata header
|
||||
if !title.streams.is_empty() {
|
||||
let m = meta::M2tsMeta::from_title(title);
|
||||
@@ -72,7 +36,7 @@ impl M2tsStream {
|
||||
DiscStream::Subtitle(s) => s.pid,
|
||||
})
|
||||
.collect();
|
||||
let boxed: Box<dyn Write> = Box::new(writer);
|
||||
let boxed: Box<dyn Write + Send> = Box::new(writer);
|
||||
let mut muxer = super::tsmux::TsMuxer::new(boxed, &pids);
|
||||
for (i, cp) in title.codec_privates.iter().enumerate() {
|
||||
if let Some(data) = cp {
|
||||
@@ -81,205 +45,153 @@ impl M2tsStream {
|
||||
}
|
||||
Ok(Self {
|
||||
disc_title: title.clone(),
|
||||
mode: Mode::Write { muxer },
|
||||
demuxer: None,
|
||||
parsers: Vec::new(),
|
||||
pending_frames: std::collections::VecDeque::new(),
|
||||
pid_to_track: Vec::new(),
|
||||
pes_eof: false,
|
||||
stored_codec_privates: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn setup_pes(streams: &[DiscStream]) -> PesSetup {
|
||||
let mut pids = Vec::new();
|
||||
let mut parsers: Vec<(u16, Box<dyn super::codec::CodecParser>)> = Vec::new();
|
||||
let mut pid_to_track = Vec::new();
|
||||
for (i, s) in streams.iter().enumerate() {
|
||||
let (pid, codec) = match s {
|
||||
DiscStream::Video(v) => (v.pid, v.codec),
|
||||
DiscStream::Audio(a) => (a.pid, a.codec),
|
||||
DiscStream::Subtitle(s) => (s.pid, s.codec),
|
||||
};
|
||||
pids.push(pid);
|
||||
pid_to_track.push((pid, i));
|
||||
parsers.push((pid, super::codec::parser_for_codec(codec, None)));
|
||||
}
|
||||
(pids, parsers, pid_to_track)
|
||||
}
|
||||
|
||||
/// Open an M2TS stream for reading. Takes any Read source — file, pipe, socket.
|
||||
///
|
||||
/// Tries FMKV metadata header first. Falls back to PMT scan of first 1 MB.
|
||||
pub fn open(mut reader: impl Read + 'static) -> io::Result<Self> {
|
||||
// Read first chunk — enough for FMKV header or PMT scan
|
||||
let mut head = vec![0u8; SCAN_SIZE];
|
||||
let head_len = read_fill(&mut reader, &mut head)?;
|
||||
head.truncate(head_len);
|
||||
|
||||
// Try FMKV metadata header from the buffered head
|
||||
let mut cursor = io::Cursor::new(&head);
|
||||
if let Ok(Some(m)) = meta::read_header(&mut cursor) {
|
||||
let header_end = cursor.position() as usize;
|
||||
let title = m.to_title();
|
||||
let (pids, parsers, pid_to_track) = Self::setup_pes(&title.streams);
|
||||
// Chain: remaining head bytes + rest of reader
|
||||
let remaining_head = &head[header_end..];
|
||||
let chain: Box<dyn Read> =
|
||||
Box::new(io::Cursor::new(remaining_head.to_vec()).chain(reader));
|
||||
return Ok(Self {
|
||||
disc_title: title.clone(),
|
||||
mode: Mode::Read { reader: chain },
|
||||
demuxer: if pids.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(ts::TsDemuxer::new(&pids))
|
||||
},
|
||||
parsers,
|
||||
pending_frames: std::collections::VecDeque::new(),
|
||||
pid_to_track,
|
||||
pes_eof: false,
|
||||
stored_codec_privates: title.codec_privates,
|
||||
});
|
||||
}
|
||||
|
||||
// No FMKV header — scan head for PMT
|
||||
let streams = ts::scan_streams(&head)
|
||||
.ok_or_else(|| -> io::Error { crate::error::Error::NoStreams.into() })?;
|
||||
|
||||
let (pids, parsers, pid_to_track) = Self::setup_pes(&streams);
|
||||
|
||||
// Chain: full head (it's all TS data) + rest of reader
|
||||
let chain: Box<dyn Read> = Box::new(io::Cursor::new(head).chain(reader));
|
||||
|
||||
Ok(Self {
|
||||
disc_title: DiscTitle {
|
||||
duration_secs: 0.0, // unknown without seeking
|
||||
streams,
|
||||
..DiscTitle::empty()
|
||||
},
|
||||
mode: Mode::Read { reader: chain },
|
||||
demuxer: if pids.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(ts::TsDemuxer::new(&pids))
|
||||
},
|
||||
parsers,
|
||||
pending_frames: std::collections::VecDeque::new(),
|
||||
pid_to_track,
|
||||
pes_eof: false,
|
||||
stored_codec_privates: Vec::new(),
|
||||
muxer,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::pes::Stream for M2tsStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
if let Some(frame) = self.pending_frames.pop_front() {
|
||||
return Ok(Some(frame));
|
||||
}
|
||||
if self.pes_eof {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
loop {
|
||||
let reader = match &mut self.mode {
|
||||
Mode::Read { reader } => reader,
|
||||
_ => return Err(crate::error::Error::StreamWriteOnly.into()),
|
||||
};
|
||||
let mut buf = vec![0u8; 192 * 1024];
|
||||
let n = reader.read(&mut buf)?;
|
||||
if n == 0 {
|
||||
self.pes_eof = true;
|
||||
// Flush demuxer — last PES packet may still be in the assembler
|
||||
if let Some(ref mut demuxer) = self.demuxer {
|
||||
for pes in &demuxer.flush() {
|
||||
if let Some((_, track)) =
|
||||
self.pid_to_track.iter().find(|(pid, _)| *pid == pes.pid)
|
||||
{
|
||||
if let Some((_, parser)) =
|
||||
self.parsers.iter_mut().find(|(pid, _)| *pid == pes.pid)
|
||||
{
|
||||
for frame in parser.parse(pes) {
|
||||
self.pending_frames.push_back(
|
||||
crate::pes::PesFrame::from_codec_frame(*track, frame),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return Ok(self.pending_frames.pop_front());
|
||||
}
|
||||
|
||||
if let Some(ref mut demuxer) = self.demuxer {
|
||||
let packets = demuxer.feed(&buf[..n]);
|
||||
for pes in &packets {
|
||||
if let Some((_, track)) =
|
||||
self.pid_to_track.iter().find(|(pid, _)| *pid == pes.pid)
|
||||
{
|
||||
if let Some((_, parser)) =
|
||||
self.parsers.iter_mut().find(|(pid, _)| *pid == pes.pid)
|
||||
{
|
||||
for frame in parser.parse(pes) {
|
||||
self.pending_frames.push_back(
|
||||
crate::pes::PesFrame::from_codec_frame(*track, frame),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(frame) = self.pending_frames.pop_front() {
|
||||
return Ok(Some(frame));
|
||||
}
|
||||
}
|
||||
// Write-only sink. The m2ts:// read direction is served by
|
||||
// `super::resolve::build_m2ts_pipeline` →
|
||||
// `PipelinedPesStream`; routing through this type for reads
|
||||
// was removed when the highway became the only ingress.
|
||||
Err(crate::error::Error::StreamWriteOnly.into())
|
||||
}
|
||||
|
||||
fn write(&mut self, frame: &crate::pes::PesFrame) -> io::Result<()> {
|
||||
match &mut self.mode {
|
||||
Mode::Write { muxer } => muxer.write_frame(frame.track, frame.pts, &frame.data),
|
||||
Mode::Read { .. } => Err(crate::error::Error::StreamReadOnly.into()),
|
||||
}
|
||||
self.muxer
|
||||
.write_frame(frame.track, frame.pts, frame.keyframe, &frame.data)
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> io::Result<()> {
|
||||
match &mut self.mode {
|
||||
Mode::Write { muxer } => muxer.finish(),
|
||||
Mode::Read { .. } => Ok(()),
|
||||
}
|
||||
self.muxer.finish()
|
||||
}
|
||||
|
||||
fn info(&self) -> &crate::disc::DiscTitle {
|
||||
&self.disc_title
|
||||
}
|
||||
|
||||
fn codec_private(&self, track: usize) -> Option<Vec<u8>> {
|
||||
// First check stored codec_privates from FMKV header
|
||||
if let Some(Some(cp)) = self.stored_codec_privates.get(track) {
|
||||
return Some(cp.clone());
|
||||
fn codec_private(&self, _track: usize) -> Option<Vec<u8>> {
|
||||
// Write side doesn't have parsers; codec_private flows in
|
||||
// via the title metadata at `create` time and gets baked
|
||||
// into the FMKV header. Nothing to surface back here.
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::disc::{
|
||||
Codec, ColorSpace, ContentFormat, DiscTitle, FrameRate, HdrFormat, Resolution,
|
||||
Stream as DiscStream, VideoStream,
|
||||
};
|
||||
use crate::pes::{PesFrame, Stream as PesStreamTrait};
|
||||
|
||||
const VIDEO_PID: u16 = 0x1011;
|
||||
|
||||
fn make_title() -> DiscTitle {
|
||||
DiscTitle {
|
||||
playlist: String::new(),
|
||||
playlist_id: 0,
|
||||
duration_secs: 0.0,
|
||||
size_bytes: 0,
|
||||
clips: Vec::new(),
|
||||
streams: vec![DiscStream::Video(VideoStream {
|
||||
pid: VIDEO_PID,
|
||||
codec: Codec::Hevc,
|
||||
resolution: Resolution::R1080p,
|
||||
frame_rate: FrameRate::F24,
|
||||
hdr: HdrFormat::Sdr,
|
||||
color_space: ColorSpace::Bt709,
|
||||
secondary: false,
|
||||
label: String::new(),
|
||||
})],
|
||||
chapters: Vec::new(),
|
||||
extents: Vec::new(),
|
||||
content_format: ContentFormat::BdTs,
|
||||
codec_privates: vec![Some({
|
||||
// Minimal hvcC with one VPS-like array entry.
|
||||
let marker: &[u8] = &[0x40, 0x01, 0x0C, 0x01];
|
||||
let mut hvcc = vec![0u8; 22];
|
||||
hvcc.push(1); // numArrays
|
||||
hvcc.push(32);
|
||||
hvcc.extend_from_slice(&1u16.to_be_bytes()); // numNalus
|
||||
hvcc.extend_from_slice(&(marker.len() as u16).to_be_bytes());
|
||||
hvcc.extend_from_slice(marker);
|
||||
hvcc
|
||||
})],
|
||||
}
|
||||
}
|
||||
|
||||
fn fake_idr_pes_data() -> Vec<u8> {
|
||||
// 4-byte length prefix + NAL: type 19 (IDR_W_RADL).
|
||||
let mut nal = vec![(19u8 << 1) & 0x7E, 0x01];
|
||||
for i in 0..200 {
|
||||
nal.push((i & 0xFF) as u8);
|
||||
}
|
||||
// Fall back to parser-extracted codec_private
|
||||
let pid = self
|
||||
.pid_to_track
|
||||
.iter()
|
||||
.find(|(_, idx)| *idx == track)
|
||||
.map(|(pid, _)| *pid)?;
|
||||
self.parsers
|
||||
.iter()
|
||||
.find(|(p, _)| *p == pid)
|
||||
.and_then(|(_, parser)| parser.codec_private())
|
||||
let mut out = Vec::with_capacity(4 + nal.len());
|
||||
out.extend_from_slice(&(nal.len() as u32).to_be_bytes());
|
||||
out.extend_from_slice(&nal);
|
||||
out
|
||||
}
|
||||
|
||||
fn headers_ready(&self) -> bool {
|
||||
for (idx, s) in self.disc_title.streams.iter().enumerate() {
|
||||
if let crate::disc::Stream::Video(v) = s {
|
||||
if !v.secondary && self.codec_private(idx).is_none() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
/// Writer wrapper that shares an Arc<Mutex<Vec<u8>>> so the test can
|
||||
/// inspect the bytes after the muxer drops.
|
||||
struct SharedSink(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
|
||||
impl Write for SharedSink {
|
||||
fn write(&mut self, b: &[u8]) -> io::Result<usize> {
|
||||
self.0.lock().unwrap().extend_from_slice(b);
|
||||
Ok(b.len())
|
||||
}
|
||||
true
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn m2ts_stream_forwards_keyframe_to_rai() {
|
||||
let title = make_title();
|
||||
let shared = std::sync::Arc::new(std::sync::Mutex::new(Vec::<u8>::new()));
|
||||
let sink = SharedSink(shared.clone());
|
||||
let mut stream = M2tsStream::create(sink, &title).unwrap();
|
||||
let frame = PesFrame {
|
||||
track: 0,
|
||||
pts: 0,
|
||||
keyframe: true,
|
||||
data: fake_idr_pes_data(),
|
||||
duration_ns: None,
|
||||
};
|
||||
stream.write(&frame).unwrap();
|
||||
stream.finish().unwrap();
|
||||
drop(stream);
|
||||
|
||||
let buf = shared.lock().unwrap().clone();
|
||||
|
||||
// Skip FMKV metadata header via meta::read_header.
|
||||
let mut cursor = std::io::Cursor::new(&buf);
|
||||
let _meta = super::meta::read_header(&mut cursor)
|
||||
.unwrap()
|
||||
.expect("FMKV header present");
|
||||
let header_end = cursor.position() as usize;
|
||||
let ts_bytes = &buf[header_end..];
|
||||
|
||||
// Find first PUSI packet on VIDEO_PID; verify RAI in AF flags.
|
||||
let pkt = ts_bytes
|
||||
.chunks(192)
|
||||
.find(|p| {
|
||||
let h = &p[4..];
|
||||
let pid = (((h[1] & 0x1F) as u16) << 8) | h[2] as u16;
|
||||
pid == VIDEO_PID && (h[1] & 0x40) != 0
|
||||
})
|
||||
.expect("video PUSI packet present");
|
||||
let h = &pkt[4..];
|
||||
let afc = (h[3] >> 4) & 0x03;
|
||||
assert!(afc & 0b10 != 0, "AF must be present");
|
||||
let af_len = h[4] as usize;
|
||||
assert!(af_len >= 1, "AF length must include flags byte");
|
||||
let flags = h[5];
|
||||
assert_eq!(flags & 0x40, 0x40, "RAI bit set");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,828 @@
|
||||
//! Standard MPEG-TS (188-byte packets) muxer — sequential-only.
|
||||
//!
|
||||
//! Distinct from `super::tsmux::TsMuxer` (BD-TS with 192-byte packets
|
||||
//! and the 4-byte TP_extra_header). This muxer emits the IETF / ISO/IEC
|
||||
//! 13818-1 wire format that ffmpeg, VLC, and `m2tsindex` consume
|
||||
//! out of the box. Use it for plain `.ts` / `.m2ts` files over a
|
||||
//! [`SequentialSink`](crate::io::sink::SequentialSink), and for
|
||||
//! MPEG-TS-over-UDP via [`UdpSocketSink`](crate::io::sink::UdpSocketSink).
|
||||
//!
|
||||
//! ## Wire format
|
||||
//!
|
||||
//! Every output packet is exactly 188 bytes:
|
||||
//!
|
||||
//! ```text
|
||||
//! sync_byte:8 = 0x47
|
||||
//! transport_error_indicator:1 = 0
|
||||
//! payload_unit_start_indicator:1
|
||||
//! transport_priority:1 = 0
|
||||
//! PID:13
|
||||
//! transport_scrambling_control:2 = 0
|
||||
//! adaptation_field_control:2
|
||||
//! continuity_counter:4
|
||||
//! [adaptation field, if signalled]
|
||||
//! [payload, if signalled]
|
||||
//! ```
|
||||
//!
|
||||
//! ## Wiring
|
||||
//!
|
||||
//! Single program (PMT PID `0x1000`, program number `1`):
|
||||
//! - PAT on PID `0x0000`
|
||||
//! - PMT on PID `0x1000`
|
||||
//! - Video (HEVC, stream_type `0x24`) on PID `0x0100`
|
||||
//! - First audio (AC3, stream_type `0x81`) on PID `0x0101`
|
||||
//! (or TrueHD, stream_type `0x83`, if hinted)
|
||||
//!
|
||||
//! ## Scope vs. full TS
|
||||
//!
|
||||
//! This is a deliberately minimal viable muxer:
|
||||
//! - One program, one video track, optionally one audio track.
|
||||
//! - PAT + PMT re-emitted every `PSI_INTERVAL_PACKETS` packets so a
|
||||
//! mid-stream receiver can lock on within ~100 ms at typical
|
||||
//! UHD bitrates.
|
||||
//! - PCR clock derived from video PTS (`pts - PCR_LEAD_90KHZ`),
|
||||
//! attached to the video PID's adaptation field every
|
||||
//! `PCR_INTERVAL_PACKETS` packets.
|
||||
//! - No language / descriptor tags, no SCTE-35 markers, no per-PID
|
||||
//! PMT version bumps, no SDT/EIT. Sufficient for "ffmpeg can play
|
||||
//! this back", not for full broadcast deployment.
|
||||
|
||||
use std::io::{self, Write};
|
||||
|
||||
mod packet;
|
||||
|
||||
use packet::{Packet, PacketWriter};
|
||||
|
||||
/// PID `0x0000` — PAT, mandated by spec.
|
||||
const PID_PAT: u16 = 0x0000;
|
||||
/// PID for the single program's PMT. `0x1000` is the conventional
|
||||
/// choice; anything outside reserved/null ranges works.
|
||||
const PID_PMT: u16 = 0x1000;
|
||||
/// PID for the video elementary stream.
|
||||
const PID_VIDEO: u16 = 0x0100;
|
||||
/// PID for the (optional) first audio elementary stream.
|
||||
const PID_AUDIO: u16 = 0x0101;
|
||||
/// Null PID — reserved by spec; never used here.
|
||||
#[allow(dead_code)]
|
||||
const PID_NULL: u16 = 0x1FFF;
|
||||
|
||||
/// Re-emit PAT/PMT every N TS packets. ~250 × 188 B = 47 KB; at
|
||||
/// 80 Mb/s UHD that's ~5 ms — well under typical receiver lock budget.
|
||||
const PSI_INTERVAL_PACKETS: u64 = 250;
|
||||
/// Re-stamp PCR every N TS packets carrying the video PID. Spec MAX
|
||||
/// is 100 ms; 40 packets × 188 B at moderate bitrate keeps us under.
|
||||
const PCR_INTERVAL_PACKETS: u64 = 40;
|
||||
/// PCR lead time — the PCR must precede the PTS of the first byte of
|
||||
/// the picture it timestamps. 200 ms in 90 kHz ticks.
|
||||
const PCR_LEAD_90KHZ: u64 = 90_000 / 5;
|
||||
|
||||
/// Stream-type codes from ISO/IEC 13818-1 Table 2-29 + later amendments.
|
||||
const STREAM_TYPE_HEVC: u8 = 0x24;
|
||||
const STREAM_TYPE_AC3: u8 = 0x81;
|
||||
const STREAM_TYPE_TRUEHD: u8 = 0x83;
|
||||
|
||||
/// Audio codec hint for [`M2tsMux::new`] / [`M2tsMux::set_audio`]. The
|
||||
/// muxer needs to know the codec to pick the right PMT `stream_type`
|
||||
/// and the right PES stream_id; it doesn't decode the audio.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum AudioCodec {
|
||||
/// AC-3 / E-AC-3, stream_type `0x81`, PES stream_id `0xBD`.
|
||||
Ac3,
|
||||
/// Dolby TrueHD, stream_type `0x83`, PES stream_id `0xBD`.
|
||||
TrueHd,
|
||||
}
|
||||
|
||||
impl AudioCodec {
|
||||
fn stream_type(self) -> u8 {
|
||||
match self {
|
||||
AudioCodec::Ac3 => STREAM_TYPE_AC3,
|
||||
AudioCodec::TrueHd => STREAM_TYPE_TRUEHD,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sequential MPEG-TS muxer with one video and optional one audio track.
|
||||
pub struct M2tsMux<W: Write> {
|
||||
out: PacketWriter<W>,
|
||||
/// hvcC parameter set bytes to prepend to the first video PES.
|
||||
/// Optional — if the upstream frames already carry inline params,
|
||||
/// callers can omit this.
|
||||
video_codec_private: Option<Vec<u8>>,
|
||||
/// Set on first video frame: have we emitted VPS/SPS/PPS?
|
||||
params_written: bool,
|
||||
/// Audio codec, if an audio track is configured. `None` = video-only.
|
||||
audio: Option<AudioCodec>,
|
||||
/// First seen PTS (90 kHz). All subsequent PTS / PCR values are
|
||||
/// relative to this so output streams start near t=0 and don't
|
||||
/// confuse downstream parsers that don't tolerate huge starting
|
||||
/// timestamps.
|
||||
base_pts_90k: Option<u64>,
|
||||
/// Per-PID continuity counter, 4 bits, monotonically increasing.
|
||||
cc_video: u8,
|
||||
cc_audio: u8,
|
||||
cc_pat: u8,
|
||||
cc_pmt: u8,
|
||||
/// Total packets written, used to gate PSI / PCR cadence.
|
||||
packets_written: u64,
|
||||
/// Video packets written since last PCR, used to gate PCR cadence.
|
||||
video_packets_since_pcr: u64,
|
||||
}
|
||||
|
||||
impl<W: Write> M2tsMux<W> {
|
||||
/// Construct a muxer wrapping `writer`. By default audio is
|
||||
/// disabled; call [`set_audio`](Self::set_audio) before the first
|
||||
/// frame to enable.
|
||||
pub fn new(writer: W) -> Self {
|
||||
Self {
|
||||
out: PacketWriter::new(writer),
|
||||
video_codec_private: None,
|
||||
params_written: false,
|
||||
audio: None,
|
||||
base_pts_90k: None,
|
||||
cc_video: 0,
|
||||
cc_audio: 0,
|
||||
cc_pat: 0,
|
||||
cc_pmt: 0,
|
||||
packets_written: 0,
|
||||
video_packets_since_pcr: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Provide the `HEVCDecoderConfigurationRecord` for video so the
|
||||
/// muxer prepends VPS/SPS/PPS Annex B NALs at stream start.
|
||||
pub fn set_video_codec_private(&mut self, hvcc: Vec<u8>) {
|
||||
self.video_codec_private = Some(hvcc);
|
||||
}
|
||||
|
||||
/// Enable a single audio track. Must be called before
|
||||
/// [`write_audio`](Self::write_audio).
|
||||
pub fn set_audio(&mut self, codec: AudioCodec) {
|
||||
self.audio = Some(codec);
|
||||
}
|
||||
|
||||
/// Write one video PES frame. `data` is either length-prefixed
|
||||
/// NALUs (MKV-style) or already Annex B; both are accepted. `keyframe`
|
||||
/// drives the random_access_indicator bit on the first packet of this
|
||||
/// PES (and gates codec_private NAL prepending — those only attach to
|
||||
/// the first keyframe).
|
||||
pub fn write_video(&mut self, pts_ns: i64, keyframe: bool, data: &[u8]) -> io::Result<()> {
|
||||
let pts_90k = self.base_relative_pts(pts_ns);
|
||||
// PCR comes "before" the PTS it timestamps; clamp at 0 for the
|
||||
// first frame so we don't underflow.
|
||||
let pcr = pts_90k.saturating_sub(PCR_LEAD_90KHZ);
|
||||
|
||||
// Annex-B-ify the frame and prepend VPS/SPS/PPS once, on the
|
||||
// FIRST keyframe (not first frame — non-key frames before the
|
||||
// first keyframe can't carry params usefully).
|
||||
let mut es = Vec::with_capacity(data.len() + 64);
|
||||
if keyframe && !self.params_written {
|
||||
if let Some(cp) = &self.video_codec_private {
|
||||
let payload = hvcc_payload(cp);
|
||||
if !payload.is_empty() {
|
||||
let params = super::hevc::length_prefixed_to_annex_b(&payload);
|
||||
es.extend_from_slice(¶ms);
|
||||
}
|
||||
}
|
||||
self.params_written = true;
|
||||
}
|
||||
let annex_b = super::hevc::length_prefixed_to_annex_b(data);
|
||||
es.extend_from_slice(&annex_b);
|
||||
|
||||
let pes = build_video_pes(pts_90k, &es);
|
||||
self.write_pes(PID_VIDEO, &pes, Some(pcr), keyframe)
|
||||
}
|
||||
|
||||
/// Write one audio PES frame. Returns `Ok(())` and silently drops
|
||||
/// the frame if no audio track was configured — the design assumes
|
||||
/// the upstream picks tracks and won't ship audio to a video-only
|
||||
/// muxer, but defending against it keeps the API a single shape.
|
||||
pub fn write_audio(&mut self, pts_ns: i64, data: &[u8]) -> io::Result<()> {
|
||||
if self.audio.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
let pts_90k = self.base_relative_pts(pts_ns);
|
||||
let pes = build_audio_pes(pts_90k, data);
|
||||
self.write_pes(PID_AUDIO, &pes, None, false)
|
||||
}
|
||||
|
||||
/// Drain the underlying writer. No TS-level trailer is mandatory —
|
||||
/// receivers detect end-of-stream from socket close or file EOF.
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
self.out.flush()
|
||||
}
|
||||
|
||||
/// Convert input PTS (nanoseconds) to 90 kHz ticks rebased on the
|
||||
/// first frame's PTS. Saturating at 0 keeps the math friendly when
|
||||
/// frames arrive slightly out of decode order.
|
||||
fn base_relative_pts(&mut self, pts_ns: i64) -> u64 {
|
||||
let raw_90k = if pts_ns > 0 {
|
||||
(pts_ns as u64) * 9 / 100_000
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let base = *self.base_pts_90k.get_or_insert(raw_90k);
|
||||
raw_90k.saturating_sub(base)
|
||||
}
|
||||
|
||||
/// Emit one PES payload as a chain of TS packets on `pid`. If `pcr`
|
||||
/// is provided the first packet carries an adaptation field with
|
||||
/// the PCR. PAT/PMT are re-emitted every `PSI_INTERVAL_PACKETS`.
|
||||
///
|
||||
/// Packet-size math (TS = 188 bytes, header = 4 bytes):
|
||||
/// - 184 B remain after the header for the adaptation-field area
|
||||
/// plus the payload area.
|
||||
/// - With AF body of `b` bytes and `s` stuffing bytes: AF total =
|
||||
/// `1 + b + s` (the leading `1` is the `adaptation_field_length`
|
||||
/// byte itself). Payload = `184 - (1 + b + s)`.
|
||||
/// - With no AF area at all: payload = `184`.
|
||||
///
|
||||
/// The fit-the-tail logic on the last packet of the PES uses
|
||||
/// stuffing rather than a separate small packet, which is the
|
||||
/// standard MPEG-TS convention.
|
||||
fn write_pes(
|
||||
&mut self,
|
||||
pid: u16,
|
||||
pes: &[u8],
|
||||
pcr: Option<u64>,
|
||||
is_keyframe_video: bool,
|
||||
) -> io::Result<()> {
|
||||
// PSI cadence is enforced per TS packet — interleave a fresh
|
||||
// PAT+PMT into the packet stream every PSI_INTERVAL_PACKETS so
|
||||
// long single-PES emissions (e.g. one 60 KB video frame) don't
|
||||
// starve receivers tuning in mid-stream.
|
||||
let mut offset = 0;
|
||||
let mut first = true;
|
||||
while offset < pes.len() {
|
||||
self.maybe_emit_psi()?;
|
||||
|
||||
let attach_pcr = first
|
||||
&& (pid == PID_VIDEO)
|
||||
&& (pcr.is_some())
|
||||
&& (self.packets_written == 0
|
||||
|| self.video_packets_since_pcr >= PCR_INTERVAL_PACKETS);
|
||||
|
||||
// RAI rides only the FIRST packet of a keyframe video PES.
|
||||
let attach_rai = first && is_keyframe_video && pid == PID_VIDEO;
|
||||
|
||||
let mut af_body: Vec<u8> = if attach_pcr {
|
||||
build_pcr_adaptation(pcr.unwrap_or(0))
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
if attach_rai {
|
||||
if af_body.is_empty() {
|
||||
af_body.push(0x40); // flags: RAI only
|
||||
} else {
|
||||
af_body[0] |= 0x40; // OR RAI into existing PCR flags
|
||||
}
|
||||
}
|
||||
|
||||
let remaining = pes.len() - offset;
|
||||
// Capacity for payload given AF body and 1-byte AF length.
|
||||
// When AF body is empty we can still skip the AF entirely
|
||||
// and get the full 184 B; only invoke the AF when we'd
|
||||
// otherwise need stuffing.
|
||||
let (af_present, payload_len, stuffing): (bool, usize, usize) = if !af_body.is_empty() {
|
||||
// AF is mandatory (PCR). 1 byte length + body + stuffing
|
||||
// + payload = 184.
|
||||
let max_payload = 184 - 1 - af_body.len();
|
||||
let p = remaining.min(max_payload);
|
||||
let s = max_payload - p;
|
||||
(true, p, s)
|
||||
} else if remaining >= 184 {
|
||||
// Full payload packet — no AF at all.
|
||||
(false, 184, 0)
|
||||
} else {
|
||||
// Last (small) packet — stuff via empty AF.
|
||||
// 1 byte length + 0 body + stuffing + payload = 184.
|
||||
let max_payload = 183;
|
||||
let p = remaining.min(max_payload);
|
||||
let s = max_payload - p;
|
||||
(true, p, s)
|
||||
};
|
||||
|
||||
let cc = self.advance_cc(pid);
|
||||
let mut packet = Packet::new();
|
||||
packet.set_header(pid, first, true, af_present, cc);
|
||||
if af_present {
|
||||
packet.append_adaptation(&af_body, stuffing);
|
||||
}
|
||||
packet.append_payload(&pes[offset..offset + payload_len]);
|
||||
debug_assert_eq!(packet.len(), 188, "packet not 188 bytes");
|
||||
self.out.write_packet(&packet)?;
|
||||
|
||||
self.packets_written += 1;
|
||||
if pid == PID_VIDEO {
|
||||
if attach_pcr {
|
||||
self.video_packets_since_pcr = 0;
|
||||
} else {
|
||||
self.video_packets_since_pcr += 1;
|
||||
}
|
||||
}
|
||||
offset += payload_len;
|
||||
first = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn advance_cc(&mut self, pid: u16) -> u8 {
|
||||
let slot = match pid {
|
||||
PID_VIDEO => &mut self.cc_video,
|
||||
PID_AUDIO => &mut self.cc_audio,
|
||||
PID_PAT => &mut self.cc_pat,
|
||||
PID_PMT => &mut self.cc_pmt,
|
||||
_ => return 0,
|
||||
};
|
||||
let cc = *slot;
|
||||
*slot = (*slot + 1) & 0x0F;
|
||||
cc
|
||||
}
|
||||
|
||||
fn maybe_emit_psi(&mut self) -> io::Result<()> {
|
||||
if self.packets_written == 0 || self.packets_written % PSI_INTERVAL_PACKETS == 0 {
|
||||
self.emit_pat()?;
|
||||
self.emit_pmt()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn emit_pat(&mut self) -> io::Result<()> {
|
||||
let payload = build_pat(PID_PMT);
|
||||
let cc = self.advance_cc(PID_PAT);
|
||||
let mut packet = Packet::new();
|
||||
packet.set_header(PID_PAT, true, true, false, cc);
|
||||
packet.append_payload(&payload);
|
||||
packet.pad_to_188();
|
||||
self.out.write_packet(&packet)?;
|
||||
self.packets_written += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn emit_pmt(&mut self) -> io::Result<()> {
|
||||
let payload = build_pmt(self.audio);
|
||||
let cc = self.advance_cc(PID_PMT);
|
||||
let mut packet = Packet::new();
|
||||
packet.set_header(PID_PMT, true, true, false, cc);
|
||||
packet.append_payload(&payload);
|
||||
packet.pad_to_188();
|
||||
self.out.write_packet(&packet)?;
|
||||
self.packets_written += 1;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the raw hvcC bytes for handoff to `length_prefixed_to_annex_b`.
|
||||
/// hvcC layout: 22-byte fixed header, then `numOfArrays` arrays of
|
||||
/// `(nalType, numNalus, [nalLength:u16, NAL bytes]…)`. We convert this
|
||||
/// directly to a length-prefixed byte stream (NAL length is u16 in
|
||||
/// hvcC; widen to u32 for the standard length-prefixed encoding).
|
||||
fn hvcc_payload(hvcc: &[u8]) -> Vec<u8> {
|
||||
if hvcc.len() < 23 {
|
||||
return Vec::new();
|
||||
}
|
||||
let num_arrays = hvcc[22] as usize;
|
||||
let mut out = Vec::new();
|
||||
let mut offset = 23;
|
||||
for _ in 0..num_arrays {
|
||||
if offset + 3 > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
offset += 1;
|
||||
let num_nalus = u16::from_be_bytes([hvcc[offset], hvcc[offset + 1]]) as usize;
|
||||
offset += 2;
|
||||
for _ in 0..num_nalus {
|
||||
if offset + 2 > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
let nal_len = u16::from_be_bytes([hvcc[offset], hvcc[offset + 1]]) as usize;
|
||||
offset += 2;
|
||||
if offset + nal_len > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
out.extend_from_slice(&(nal_len as u32).to_be_bytes());
|
||||
out.extend_from_slice(&hvcc[offset..offset + nal_len]);
|
||||
offset += nal_len;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Build a PES packet for a video access unit.
|
||||
fn build_video_pes(pts_90k: u64, es: &[u8]) -> Vec<u8> {
|
||||
build_pes_packet(0xE0, pts_90k, es, /* length_in_header */ false)
|
||||
}
|
||||
|
||||
/// Build a PES packet for an audio access unit.
|
||||
fn build_audio_pes(pts_90k: u64, es: &[u8]) -> Vec<u8> {
|
||||
// Audio PES: length is fillable when it fits in u16. We always
|
||||
// write the length so receivers don't have to scan for the next
|
||||
// start code.
|
||||
build_pes_packet(0xBD, pts_90k, es, /* length_in_header */ true)
|
||||
}
|
||||
|
||||
fn build_pes_packet(stream_id: u8, pts_90k: u64, es: &[u8], length_in_header: bool) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(es.len() + 14);
|
||||
out.extend_from_slice(&[0x00, 0x00, 0x01, stream_id]);
|
||||
// PES_packet_length: total bytes after this field. 3 flag bytes + 5
|
||||
// PTS bytes + es.len(). Zero means "unbounded" — used for video
|
||||
// where PES can exceed u16.
|
||||
let pes_len = 8 + es.len();
|
||||
if length_in_header && pes_len <= u16::MAX as usize {
|
||||
out.extend_from_slice(&(pes_len as u16).to_be_bytes());
|
||||
} else {
|
||||
out.extend_from_slice(&[0x00, 0x00]);
|
||||
}
|
||||
// Flags byte 1: 10 = MPEG-2 marker, then scrambling/priority/etc 0.
|
||||
out.push(0x80);
|
||||
// Flags byte 2: PTS flag (bit 7).
|
||||
out.push(0x80);
|
||||
// PES_header_data_length = 5 (just PTS).
|
||||
out.push(5);
|
||||
// PTS bytes — 33-bit timestamp split across 5 bytes with marker bits.
|
||||
let pts = pts_90k & 0x1_FFFF_FFFF;
|
||||
out.push(0x21 | (((pts >> 29) & 0x0E) as u8));
|
||||
out.push(((pts >> 22) & 0xFF) as u8);
|
||||
out.push(0x01 | (((pts >> 14) & 0xFE) as u8));
|
||||
out.push(((pts >> 7) & 0xFF) as u8);
|
||||
out.push(0x01 | (((pts << 1) & 0xFE) as u8));
|
||||
out.extend_from_slice(es);
|
||||
out
|
||||
}
|
||||
|
||||
/// Build the PAT payload (section, with pointer_field).
|
||||
fn build_pat(pmt_pid: u16) -> Vec<u8> {
|
||||
let mut section = Vec::new();
|
||||
section.push(0x00); // table_id = PAT
|
||||
// section_syntax_indicator(1) | '0'(1) | reserved(2) | section_length(12)
|
||||
// section_length covers from end of this field through CRC.
|
||||
// Body: transport_stream_id(2) + version/cni(1) + section/last_section(2) + program(4) = 9 bytes,
|
||||
// plus CRC(4) = 13. Encoded big-endian.
|
||||
section.extend_from_slice(&[0xB0, 13]);
|
||||
section.extend_from_slice(&[0x00, 0x01]); // transport_stream_id = 1
|
||||
section.push(0xC1); // reserved | version=0 | current_next=1
|
||||
section.push(0x00); // section_number
|
||||
section.push(0x00); // last_section_number
|
||||
section.extend_from_slice(&[0x00, 0x01]); // program_number = 1
|
||||
// reserved(3) | network_PID/program_map_PID(13)
|
||||
let pid_bytes = (0xE000u16 | (pmt_pid & 0x1FFF)).to_be_bytes();
|
||||
section.extend_from_slice(&pid_bytes);
|
||||
let crc = mpegts_crc32(§ion);
|
||||
section.extend_from_slice(&crc.to_be_bytes());
|
||||
// Prepend pointer_field=0 (section starts immediately).
|
||||
let mut payload = Vec::with_capacity(section.len() + 1);
|
||||
payload.push(0x00);
|
||||
payload.extend_from_slice(§ion);
|
||||
payload
|
||||
}
|
||||
|
||||
/// Build the PMT payload (section, with pointer_field).
|
||||
fn build_pmt(audio: Option<AudioCodec>) -> Vec<u8> {
|
||||
let mut section = Vec::new();
|
||||
section.push(0x02); // table_id = PMT
|
||||
// section_length filled in after we know the body size.
|
||||
let len_placeholder = section.len();
|
||||
section.extend_from_slice(&[0xB0, 0x00]);
|
||||
|
||||
section.extend_from_slice(&1u16.to_be_bytes()); // program_number
|
||||
section.push(0xC1); // reserved | version=0 | current_next=1
|
||||
section.push(0x00); // section_number
|
||||
section.push(0x00); // last_section_number
|
||||
// reserved(3) | PCR_PID(13)
|
||||
let pcr_pid = (0xE000u16 | (PID_VIDEO & 0x1FFF)).to_be_bytes();
|
||||
section.extend_from_slice(&pcr_pid);
|
||||
// program_info_length = 0
|
||||
section.extend_from_slice(&[0xF0, 0x00]);
|
||||
|
||||
// Video elementary stream entry.
|
||||
section.push(STREAM_TYPE_HEVC);
|
||||
let v_pid = (0xE000u16 | (PID_VIDEO & 0x1FFF)).to_be_bytes();
|
||||
section.extend_from_slice(&v_pid);
|
||||
section.extend_from_slice(&[0xF0, 0x00]); // ES_info_length = 0
|
||||
|
||||
if let Some(codec) = audio {
|
||||
section.push(codec.stream_type());
|
||||
let a_pid = (0xE000u16 | (PID_AUDIO & 0x1FFF)).to_be_bytes();
|
||||
section.extend_from_slice(&a_pid);
|
||||
section.extend_from_slice(&[0xF0, 0x00]);
|
||||
}
|
||||
|
||||
// Now patch section_length: covers everything after the length field
|
||||
// through the CRC, so (current body size - 3 bytes consumed by
|
||||
// table_id + 2 length bytes) + 4 (CRC).
|
||||
let section_len = section.len() - 3 + 4;
|
||||
section[len_placeholder] = 0xB0 | ((section_len >> 8) as u8 & 0x0F);
|
||||
section[len_placeholder + 1] = section_len as u8;
|
||||
|
||||
let crc = mpegts_crc32(§ion);
|
||||
section.extend_from_slice(&crc.to_be_bytes());
|
||||
|
||||
let mut payload = Vec::with_capacity(section.len() + 1);
|
||||
payload.push(0x00);
|
||||
payload.extend_from_slice(§ion);
|
||||
payload
|
||||
}
|
||||
|
||||
/// Build the adaptation field carrying a PCR (no other flags).
|
||||
fn build_pcr_adaptation(pcr_90k: u64) -> Vec<u8> {
|
||||
// adaptation_field_length is set by `Packet::append_adaptation`
|
||||
// — this function returns just the field body.
|
||||
//
|
||||
// Layout: discontinuity_indicator(1) | random_access(1) |
|
||||
// elementary_stream_priority(1) | PCR_flag(1) | OPCR_flag(1) |
|
||||
// splicing_point_flag(1) | transport_private_data_flag(1) |
|
||||
// adaptation_field_extension_flag(1) | PCR(48b).
|
||||
let mut af = vec![0x10]; // PCR_flag=1; RAI is OR'd in by the caller when applicable
|
||||
let pcr_base = pcr_90k & 0x1_FFFF_FFFF; // 33-bit
|
||||
let pcr_ext: u16 = 0; // 9-bit, we keep it zero (no sub-tick precision)
|
||||
// Encode PCR: 33b base | 6b reserved | 9b extension = 48b
|
||||
af.push((pcr_base >> 25) as u8);
|
||||
af.push((pcr_base >> 17) as u8);
|
||||
af.push((pcr_base >> 9) as u8);
|
||||
af.push((pcr_base >> 1) as u8);
|
||||
af.push(((pcr_base << 7) as u8 & 0x80) | 0x7E | ((pcr_ext >> 8) as u8 & 0x01));
|
||||
af.push(pcr_ext as u8);
|
||||
af
|
||||
}
|
||||
|
||||
/// MPEG-TS CRC-32 (poly 0x04C11DB7, init 0xFFFFFFFF, no reflection, no
|
||||
/// final XOR). Implementation: bitwise so we don't need a table —
|
||||
/// PSI sections are tiny, the cost is negligible.
|
||||
fn mpegts_crc32(data: &[u8]) -> u32 {
|
||||
let mut crc: u32 = 0xFFFF_FFFF;
|
||||
for &b in data {
|
||||
crc ^= (b as u32) << 24;
|
||||
for _ in 0..8 {
|
||||
if crc & 0x8000_0000 != 0 {
|
||||
crc = (crc << 1) ^ 0x04C1_1DB7;
|
||||
} else {
|
||||
crc <<= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
crc
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// All emitted bytes must align to 188-byte packet boundaries and
|
||||
/// every packet must start with `0x47`.
|
||||
fn assert_ts_well_formed(buf: &[u8]) {
|
||||
assert_eq!(
|
||||
buf.len() % 188,
|
||||
0,
|
||||
"stream not packet-aligned: {} bytes",
|
||||
buf.len()
|
||||
);
|
||||
for (i, chunk) in buf.chunks(188).enumerate() {
|
||||
assert_eq!(chunk[0], 0x47, "packet {} missing sync byte", i);
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_pids(buf: &[u8]) -> Vec<u16> {
|
||||
buf.chunks(188)
|
||||
.map(|p| u16::from_be_bytes([p[1] & 0x1F, p[2]]))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crc32_is_self_validating() {
|
||||
// The MPEG-TS CRC has the property that prepending a single
|
||||
// bit-flip changes the output; running it over its own input +
|
||||
// CRC yields a fixed magic constant (the CRC residue). Rather
|
||||
// than hardcoding sample bytes, verify the underlying algorithm
|
||||
// by checking that two distinct inputs produce distinct CRCs
|
||||
// and that the same input is deterministic.
|
||||
let a = [
|
||||
0u8, 0xB0, 0x0D, 0x00, 0x01, 0xC1, 0x00, 0x00, 0x00, 0x01, 0xE1, 0x00,
|
||||
];
|
||||
let mut b = a;
|
||||
b[5] ^= 0x01; // flip one bit
|
||||
let crc_a = mpegts_crc32(&a);
|
||||
let crc_b = mpegts_crc32(&b);
|
||||
assert_ne!(crc_a, crc_b);
|
||||
assert_eq!(crc_a, mpegts_crc32(&a)); // deterministic
|
||||
// Sanity: all-zero input ⇒ CRC = 0 (init XORs but the
|
||||
// shift/feedback cancels for zero data after init drains).
|
||||
// We don't assert exact value — that depends on poly choice —
|
||||
// but check it's not the same as for non-zero data.
|
||||
let crc_zero = mpegts_crc32(&[0u8; 12]);
|
||||
assert_ne!(crc_zero, crc_a);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_only_mux_emits_pat_pmt_then_video() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = M2tsMux::new(&mut sink);
|
||||
// One small video frame, no codec_private (so no params NAL inline).
|
||||
let mut frame = Vec::new();
|
||||
frame.extend_from_slice(&4u32.to_be_bytes());
|
||||
frame.extend_from_slice(&[0x40, 0x01, 0x0C, 0x01]);
|
||||
mux.write_video(0, true, &frame).unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
assert_ts_well_formed(&sink);
|
||||
let pids = extract_pids(&sink);
|
||||
// First two packets: PAT, PMT. At least one video packet after.
|
||||
assert_eq!(pids[0], PID_PAT);
|
||||
assert_eq!(pids[1], PID_PMT);
|
||||
assert!(pids.iter().any(|p| *p == PID_VIDEO));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_track_appears_in_pmt_and_stream() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = M2tsMux::new(&mut sink);
|
||||
mux.set_audio(AudioCodec::Ac3);
|
||||
// Video + audio frame pair.
|
||||
let mut frame = Vec::new();
|
||||
frame.extend_from_slice(&3u32.to_be_bytes());
|
||||
frame.extend_from_slice(&[0x40, 0x01, 0x0C]);
|
||||
mux.write_video(0, true, &frame).unwrap();
|
||||
mux.write_audio(20_000_000, &[0x0B, 0x77, 0x12, 0x34])
|
||||
.unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
assert_ts_well_formed(&sink);
|
||||
let pids = extract_pids(&sink);
|
||||
assert!(pids.iter().any(|p| *p == PID_VIDEO));
|
||||
assert!(pids.iter().any(|p| *p == PID_AUDIO));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn psi_re_emits_at_interval() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = M2tsMux::new(&mut sink);
|
||||
// Build a frame large enough to span > PSI_INTERVAL_PACKETS TS packets.
|
||||
// 184 B payload per packet ⇒ ~250 packets = ~46 KB elementary stream.
|
||||
let big: Vec<u8> = (0..(60 * 1024)).map(|i| (i & 0xff) as u8).collect();
|
||||
let mut frame = Vec::new();
|
||||
frame.extend_from_slice(&(big.len() as u32).to_be_bytes());
|
||||
frame.extend_from_slice(&big);
|
||||
mux.write_video(0, true, &frame).unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
assert_ts_well_formed(&sink);
|
||||
let pids = extract_pids(&sink);
|
||||
// Count PAT/PMT pairs — must be at least 2 given the input size.
|
||||
let pat_count = pids.iter().filter(|p| **p == PID_PAT).count();
|
||||
let pmt_count = pids.iter().filter(|p| **p == PID_PMT).count();
|
||||
assert!(pat_count >= 2, "expected ≥2 PAT, got {}", pat_count);
|
||||
assert!(pmt_count >= 2, "expected ≥2 PMT, got {}", pmt_count);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn continuity_counter_increments_per_pid() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = M2tsMux::new(&mut sink);
|
||||
// Three small video frames to get a sequence of video TS packets.
|
||||
for pts in [0i64, 40_000_000, 80_000_000] {
|
||||
let mut frame = Vec::new();
|
||||
frame.extend_from_slice(&3u32.to_be_bytes());
|
||||
frame.extend_from_slice(&[0xAA, 0xBB, 0xCC]);
|
||||
mux.write_video(pts, pts == 0, &frame).unwrap();
|
||||
}
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
// Collect CCs for video packets in order.
|
||||
let ccs: Vec<u8> = sink
|
||||
.chunks(188)
|
||||
.filter(|p| u16::from_be_bytes([p[1] & 0x1F, p[2]]) == PID_VIDEO)
|
||||
.map(|p| p[3] & 0x0F)
|
||||
.collect();
|
||||
for w in ccs.windows(2) {
|
||||
assert_eq!(w[1], (w[0] + 1) & 0x0F);
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the adaptation field body (length byte stripped) for one
|
||||
/// 188-byte TS packet, or None when the packet has no AF.
|
||||
fn af_body(packet: &[u8]) -> Option<Vec<u8>> {
|
||||
let afc = (packet[3] >> 4) & 0x03;
|
||||
if afc & 0b10 == 0 {
|
||||
return None;
|
||||
}
|
||||
let af_len = packet[4] as usize;
|
||||
if af_len == 0 {
|
||||
return Some(Vec::new());
|
||||
}
|
||||
Some(packet[5..5 + af_len].to_vec())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rai_set_on_keyframe_pes_packet() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = M2tsMux::new(&mut sink);
|
||||
let mut frame = Vec::new();
|
||||
frame.extend_from_slice(&4u32.to_be_bytes());
|
||||
frame.extend_from_slice(&[0x40, 0x01, 0x0C, 0x01]);
|
||||
mux.write_video(0, true, &frame).unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
// Find the first PUSI packet on PID_VIDEO.
|
||||
let pkt = sink
|
||||
.chunks(188)
|
||||
.find(|p| u16::from_be_bytes([p[1] & 0x1F, p[2]]) == PID_VIDEO && (p[1] & 0x40) != 0)
|
||||
.expect("video PUSI packet exists");
|
||||
let af = af_body(pkt).expect("AF present on first packet of keyframe video PES");
|
||||
assert!(!af.is_empty(), "AF flags byte present");
|
||||
assert_eq!(af[0] & 0x40, 0x40, "RAI bit set");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pcr_packet_without_keyframe_has_rai_clear() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = M2tsMux::new(&mut sink);
|
||||
// First frame is the keyframe (gates codec_private; also gets PCR).
|
||||
let mut frame0 = Vec::new();
|
||||
frame0.extend_from_slice(&4u32.to_be_bytes());
|
||||
frame0.extend_from_slice(&[0x40, 0x01, 0x0C, 0x01]);
|
||||
mux.write_video(0, true, &frame0).unwrap();
|
||||
// Push enough non-key video frames to cross PCR_INTERVAL_PACKETS
|
||||
// video packets so a later PCR-bearing packet exists.
|
||||
// Each frame is ~50 KB → ~270 packets, well over 40.
|
||||
let big: Vec<u8> = (0..(50 * 1024)).map(|i| (i & 0xff) as u8).collect();
|
||||
for i in 1..3 {
|
||||
let mut frame = Vec::new();
|
||||
frame.extend_from_slice(&(big.len() as u32).to_be_bytes());
|
||||
frame.extend_from_slice(&big);
|
||||
mux.write_video((i as i64) * 40_000_000, false, &frame)
|
||||
.unwrap();
|
||||
}
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
// The first PUSI video packet carries PCR + RAI (keyframe).
|
||||
// A later video PUSI packet with AF + PCR but NOT keyframe must
|
||||
// have RAI clear.
|
||||
let video_pusi: Vec<&[u8]> = sink
|
||||
.chunks(188)
|
||||
.filter(|p| u16::from_be_bytes([p[1] & 0x1F, p[2]]) == PID_VIDEO && (p[1] & 0x40) != 0)
|
||||
.collect();
|
||||
assert!(
|
||||
video_pusi.len() >= 2,
|
||||
"expected ≥2 video PES starts, got {}",
|
||||
video_pusi.len()
|
||||
);
|
||||
// Find a later one with AF that carries PCR (flags & 0x10 set).
|
||||
let later_pcr = video_pusi
|
||||
.iter()
|
||||
.skip(1)
|
||||
.find_map(|p| {
|
||||
let af = af_body(p)?;
|
||||
if !af.is_empty() && (af[0] & 0x10) != 0 {
|
||||
Some(af)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.expect("later PCR-bearing PUSI exists");
|
||||
assert_eq!(
|
||||
later_pcr[0] & 0x40,
|
||||
0,
|
||||
"RAI must be clear on non-keyframe PCR packet"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keyframe_video_with_pcr_combines_flags() {
|
||||
// PCR attaches only when video_packets_since_pcr >=
|
||||
// PCR_INTERVAL_PACKETS (40). The very first video packet emits a
|
||||
// PAT+PMT first, so packets_written != 0 and attach_pcr is false on
|
||||
// frame 0. We push: keyframe (no PCR) → many non-key (drives the
|
||||
// PCR counter past the interval) → second keyframe (PCR + RAI).
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = M2tsMux::new(&mut sink);
|
||||
let mut small = Vec::new();
|
||||
small.extend_from_slice(&4u32.to_be_bytes());
|
||||
small.extend_from_slice(&[0x40, 0x01, 0x0C, 0x01]);
|
||||
mux.write_video(0, true, &small).unwrap();
|
||||
// ~50 KB ≈ 270 packets — well over PCR_INTERVAL_PACKETS.
|
||||
let big: Vec<u8> = (0..(50 * 1024)).map(|i| (i & 0xff) as u8).collect();
|
||||
let mut big_frame = Vec::new();
|
||||
big_frame.extend_from_slice(&(big.len() as u32).to_be_bytes());
|
||||
big_frame.extend_from_slice(&big);
|
||||
mux.write_video(40_000_000, false, &big_frame).unwrap();
|
||||
// Now a second keyframe — must combine RAI (keyframe) and PCR
|
||||
// (counter exceeded).
|
||||
mux.write_video(80_000_000, true, &small).unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
// Collect video PUSI packets and find the third (second keyframe).
|
||||
let video_pusi: Vec<&[u8]> = sink
|
||||
.chunks(188)
|
||||
.filter(|p| u16::from_be_bytes([p[1] & 0x1F, p[2]]) == PID_VIDEO && (p[1] & 0x40) != 0)
|
||||
.collect();
|
||||
assert!(video_pusi.len() >= 3, "three video PES starts expected");
|
||||
let af = af_body(video_pusi[2]).expect("AF present");
|
||||
assert!(!af.is_empty(), "AF flags byte present");
|
||||
assert_eq!(af[0], 0x50, "flags == RAI | PCR");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
//! 188-byte MPEG-TS packet builder + writer.
|
||||
//!
|
||||
//! Internal helper for `super::M2tsMux`. Not public API — exposes raw
|
||||
//! byte layout so the parent module can compose PSI / PCR / PES bytes
|
||||
//! without each caller re-implementing the 188-byte boundary math.
|
||||
|
||||
use std::io::{self, Write};
|
||||
|
||||
const TS_PACKET_SIZE: usize = 188;
|
||||
const SYNC_BYTE: u8 = 0x47;
|
||||
const STUFF_BYTE: u8 = 0xFF;
|
||||
|
||||
/// One TS packet under construction. Always emits 188 bytes when
|
||||
/// [`pad_to_188`](Self::pad_to_188) is called; if it's not called the
|
||||
/// caller is responsible for filling the packet exactly.
|
||||
pub(super) struct Packet {
|
||||
buf: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Packet {
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
buf: Vec::with_capacity(TS_PACKET_SIZE),
|
||||
}
|
||||
}
|
||||
|
||||
/// Write the 4-byte TS packet header.
|
||||
///
|
||||
/// * `pid` — 13-bit PID
|
||||
/// * `payload_unit_start` — first packet of a PES / PSI section
|
||||
/// * `has_payload` — packet carries any payload bytes
|
||||
/// * `has_adaptation` — packet carries an adaptation field
|
||||
/// * `cc` — 4-bit continuity counter
|
||||
pub(super) fn set_header(
|
||||
&mut self,
|
||||
pid: u16,
|
||||
payload_unit_start: bool,
|
||||
has_payload: bool,
|
||||
has_adaptation: bool,
|
||||
cc: u8,
|
||||
) {
|
||||
self.buf.clear();
|
||||
self.buf.push(SYNC_BYTE);
|
||||
let pus_bit = if payload_unit_start { 0x40 } else { 0 };
|
||||
// transport_error_indicator(1)=0 | payload_unit_start(1) | transport_priority(1)=0 | PID(5 high)
|
||||
self.buf.push(pus_bit | ((pid >> 8) as u8 & 0x1F));
|
||||
self.buf.push(pid as u8);
|
||||
// transport_scrambling_control(2)=0 | adaptation_field_control(2) | continuity_counter(4)
|
||||
let afc = match (has_adaptation, has_payload) {
|
||||
(false, false) => 0b00, // reserved — should not happen
|
||||
(false, true) => 0b01, // payload only
|
||||
(true, false) => 0b10, // adaptation only
|
||||
(true, true) => 0b11, // both
|
||||
};
|
||||
self.buf.push((afc << 4) | (cc & 0x0F));
|
||||
}
|
||||
|
||||
/// Append the adaptation field after the header.
|
||||
///
|
||||
/// `body` is the adaptation field body (flags byte plus optional PCR
|
||||
/// and so on). `stuffing` is the number of `0xFF` stuffing bytes to
|
||||
/// append after the body. The first byte of the field
|
||||
/// (`adaptation_field_length`) is computed here from
|
||||
/// `body.len() + stuffing`.
|
||||
pub(super) fn append_adaptation(&mut self, body: &[u8], stuffing: usize) {
|
||||
let af_len = body.len() + stuffing;
|
||||
debug_assert!(af_len <= 183, "adaptation field overflow");
|
||||
self.buf.push(af_len as u8);
|
||||
self.buf.extend_from_slice(body);
|
||||
for _ in 0..stuffing {
|
||||
self.buf.push(STUFF_BYTE);
|
||||
}
|
||||
}
|
||||
|
||||
/// Append payload bytes.
|
||||
pub(super) fn append_payload(&mut self, payload: &[u8]) {
|
||||
self.buf.extend_from_slice(payload);
|
||||
debug_assert!(self.buf.len() <= TS_PACKET_SIZE, "packet overflow");
|
||||
}
|
||||
|
||||
/// Pad the packet to exactly 188 bytes with `0xFF` bytes — used by
|
||||
/// PSI emit paths where the section is much smaller than 184 bytes.
|
||||
/// For PSI packets only — payload-carrying packets reserve room for
|
||||
/// stuffing via `append_adaptation`.
|
||||
pub(super) fn pad_to_188(&mut self) {
|
||||
while self.buf.len() < TS_PACKET_SIZE {
|
||||
self.buf.push(STUFF_BYTE);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn bytes(&self) -> &[u8] {
|
||||
&self.buf
|
||||
}
|
||||
|
||||
pub(super) fn len(&self) -> usize {
|
||||
self.buf.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Buffered writer for assembled TS packets. Owns the underlying sink.
|
||||
pub(super) struct PacketWriter<W: Write> {
|
||||
inner: W,
|
||||
}
|
||||
|
||||
impl<W: Write> PacketWriter<W> {
|
||||
pub(super) fn new(inner: W) -> Self {
|
||||
Self { inner }
|
||||
}
|
||||
|
||||
pub(super) fn write_packet(&mut self, packet: &Packet) -> io::Result<()> {
|
||||
let bytes = packet.bytes();
|
||||
debug_assert_eq!(bytes.len(), TS_PACKET_SIZE);
|
||||
self.inner.write_all(bytes)
|
||||
}
|
||||
|
||||
pub(super) fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn pad_fills_to_188() {
|
||||
let mut p = Packet::new();
|
||||
p.set_header(0x100, true, true, false, 0);
|
||||
p.append_payload(&[1, 2, 3]);
|
||||
p.pad_to_188();
|
||||
assert_eq!(p.bytes().len(), 188);
|
||||
assert_eq!(p.bytes()[0], SYNC_BYTE);
|
||||
// After 4-byte header + 3 payload, byte 7 starts stuffing.
|
||||
assert_eq!(p.bytes()[7], STUFF_BYTE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_pid_round_trips() {
|
||||
let mut p = Packet::new();
|
||||
p.set_header(0x1ABC, false, true, false, 0xA);
|
||||
let pid = u16::from_be_bytes([p.bytes()[1] & 0x1F, p.bytes()[2]]);
|
||||
assert_eq!(pid, 0x1ABC);
|
||||
assert_eq!(p.bytes()[3] & 0x0F, 0xA);
|
||||
}
|
||||
}
|
||||
+564
-25
@@ -87,11 +87,19 @@ impl MkvTrack {
|
||||
}
|
||||
|
||||
pub fn audio(a: &AudioStream) -> Self {
|
||||
// Codec ID strings must distinguish the DTS family — strict
|
||||
// players (Plex transcoder, some hardware decoders, some AV
|
||||
// receivers) reject lossless DTS-HD MA payload when the
|
||||
// track advertises plain `A_DTS` because it implies the
|
||||
// bitstream is the 1.5 Mbps "core" only. Fix is just to
|
||||
// emit the right ID per BD-STN codec field.
|
||||
let codec_id = match a.codec {
|
||||
Codec::Ac3 => "A_AC3",
|
||||
Codec::Ac3Plus => "A_EAC3",
|
||||
Codec::TrueHd => "A_TRUEHD",
|
||||
Codec::DtsHdMa | Codec::DtsHdHr | Codec::Dts => "A_DTS",
|
||||
Codec::DtsHdMa => "A_DTS/MA",
|
||||
Codec::DtsHdHr => "A_DTS/HR",
|
||||
Codec::Dts => "A_DTS",
|
||||
Codec::Lpcm => "A_PCM/INT/BIG",
|
||||
_ => "A_AC3",
|
||||
};
|
||||
@@ -159,6 +167,12 @@ struct CuePoint {
|
||||
cluster_pos: u64, // relative to Segment start
|
||||
}
|
||||
|
||||
/// SeekHead entry that needs its 8-byte SeekPosition back-patched after Cues are written.
|
||||
struct SeekPositionFixup {
|
||||
target_id: u32,
|
||||
value_offset: u64, // absolute file offset of the 8-byte SeekPosition value
|
||||
}
|
||||
|
||||
/// MKV muxer. Call write_frame() for each frame, then finish() at the end.
|
||||
pub struct MkvMuxer<W: Write + Seek> {
|
||||
writer: W,
|
||||
@@ -170,6 +184,10 @@ pub struct MkvMuxer<W: Write + Seek> {
|
||||
base_pts_ms: Option<i64>,
|
||||
cues: Vec<CuePoint>,
|
||||
frame_count: u64,
|
||||
seek_fixups: Vec<SeekPositionFixup>,
|
||||
info_offset: u64,
|
||||
tracks_offset: u64,
|
||||
chapters_offset: Option<u64>,
|
||||
}
|
||||
|
||||
/// New cluster every 5 seconds.
|
||||
@@ -200,7 +218,34 @@ impl<W: Write + Seek> MkvMuxer<W> {
|
||||
ebml::write_unknown_size(&mut writer)?;
|
||||
let segment_start = writer.stream_position()?;
|
||||
|
||||
// SeekHead with fixed-width SeekPosition placeholders. Order: Info, Tracks, [Chapters], Cues.
|
||||
let mut seek_fixups: Vec<SeekPositionFixup> = Vec::new();
|
||||
let seekhead_pos = ebml::start_master(&mut writer, ebml::SEEK_HEAD)?;
|
||||
let mut targets: Vec<u32> = vec![ebml::INFO, ebml::TRACKS];
|
||||
if !chapters.is_empty() {
|
||||
targets.push(ebml::CHAPTERS);
|
||||
}
|
||||
targets.push(ebml::CUES);
|
||||
let seek_id_be = (ebml::SEEK as u16).to_be_bytes();
|
||||
let seek_inner_id_be = (ebml::SEEK_ID as u16).to_be_bytes();
|
||||
let seek_pos_id_be = (ebml::SEEK_POSITION as u16).to_be_bytes();
|
||||
for target_id in &targets {
|
||||
writer.write_all(&[seek_id_be[0], seek_id_be[1], 0x92])?;
|
||||
writer.write_all(&[seek_inner_id_be[0], seek_inner_id_be[1], 0x84])?;
|
||||
writer.write_all(&target_id.to_be_bytes())?;
|
||||
writer.write_all(&[seek_pos_id_be[0], seek_pos_id_be[1], 0x88])?;
|
||||
let value_offset = writer.stream_position()?;
|
||||
writer.write_all(&[0u8; 8])?;
|
||||
seek_fixups.push(SeekPositionFixup {
|
||||
target_id: *target_id,
|
||||
value_offset,
|
||||
});
|
||||
}
|
||||
ebml::end_master(&mut writer, seekhead_pos)?;
|
||||
|
||||
// Info
|
||||
let info_start = writer.stream_position()?;
|
||||
let info_offset = info_start - segment_start;
|
||||
let info_pos = ebml::start_master(&mut writer, ebml::INFO)?;
|
||||
ebml::write_uint(&mut writer, ebml::TIMESTAMP_SCALE, 1_000_000)?; // 1ms precision
|
||||
if duration_secs > 0.0 {
|
||||
@@ -215,6 +260,8 @@ impl<W: Write + Seek> MkvMuxer<W> {
|
||||
ebml::end_master(&mut writer, info_pos)?;
|
||||
|
||||
// Tracks
|
||||
let tracks_start = writer.stream_position()?;
|
||||
let tracks_offset = tracks_start - segment_start;
|
||||
let tracks_pos = ebml::start_master(&mut writer, ebml::TRACKS)?;
|
||||
for (i, track) in tracks.iter().enumerate() {
|
||||
let entry_pos = ebml::start_master(&mut writer, ebml::TRACK_ENTRY)?;
|
||||
@@ -302,7 +349,10 @@ impl<W: Write + Seek> MkvMuxer<W> {
|
||||
ebml::end_master(&mut writer, tracks_pos)?;
|
||||
|
||||
// Chapters
|
||||
let mut chapters_offset: Option<u64> = None;
|
||||
if !chapters.is_empty() {
|
||||
let chapters_start = writer.stream_position()?;
|
||||
chapters_offset = Some(chapters_start - segment_start);
|
||||
let chapters_pos = ebml::start_master(&mut writer, ebml::CHAPTERS)?;
|
||||
let edition_pos = ebml::start_master(&mut writer, ebml::EDITION_ENTRY)?;
|
||||
for (i, ch) in chapters.iter().enumerate() {
|
||||
@@ -330,42 +380,60 @@ impl<W: Write + Seek> MkvMuxer<W> {
|
||||
base_pts_ms: None,
|
||||
cues: Vec::new(),
|
||||
frame_count: 0,
|
||||
seek_fixups,
|
||||
info_offset,
|
||||
tracks_offset,
|
||||
chapters_offset,
|
||||
})
|
||||
}
|
||||
|
||||
/// Write a single frame.
|
||||
///
|
||||
/// When `duration_ns` is `Some`, the frame is emitted as a
|
||||
/// `BlockGroup` with `BlockDuration` so the player knows exactly
|
||||
/// when to remove the on-screen artifact (the practical case is
|
||||
/// PGS subtitles — without it, the last bitmap lingers until the
|
||||
/// next display set replaces it). Otherwise a plain `SimpleBlock`.
|
||||
pub fn write_frame(
|
||||
&mut self,
|
||||
track_idx: usize,
|
||||
pts_ns: i64,
|
||||
keyframe: bool,
|
||||
data: &[u8],
|
||||
duration_ns: Option<u64>,
|
||||
) -> io::Result<()> {
|
||||
let raw_ms = pts_ns / 1_000_000;
|
||||
let base = *self.base_pts_ms.get_or_insert(raw_ms);
|
||||
let pts_ms = raw_ms - base;
|
||||
|
||||
// Start new cluster if needed
|
||||
if !self.cluster_open || (pts_ms - self.cluster_ts_ms) >= CLUSTER_DURATION_MS {
|
||||
if self.cluster_open {
|
||||
// Close current cluster (it's a master with unknown size — we use known size)
|
||||
// Actually, for streaming we keep clusters open-ended. Just start a new one.
|
||||
// Cluster boundaries must coincide with a video keyframe so every
|
||||
// Cues entry resolves to a seekable IDR at the cluster start.
|
||||
let is_video_key = keyframe && track_idx == 0;
|
||||
let needs_new_cluster = !self.cluster_open
|
||||
|| (is_video_key && (pts_ms - self.cluster_ts_ms) >= CLUSTER_DURATION_MS);
|
||||
|
||||
if needs_new_cluster {
|
||||
if !is_video_key {
|
||||
return Ok(());
|
||||
}
|
||||
self.start_cluster(pts_ms)?;
|
||||
|
||||
// Add cue point at cluster start for keyframes (video track 0)
|
||||
if keyframe && track_idx == 0 {
|
||||
self.cues.push(CuePoint {
|
||||
timestamp_ms: pts_ms,
|
||||
track: track_idx + 1,
|
||||
cluster_pos: self.cluster_pos - self.segment_start,
|
||||
});
|
||||
}
|
||||
self.cues.push(CuePoint {
|
||||
timestamp_ms: pts_ms,
|
||||
track: track_idx + 1,
|
||||
cluster_pos: self.cluster_pos - self.segment_start,
|
||||
});
|
||||
}
|
||||
|
||||
// Write SimpleBlock
|
||||
let relative_ts = (pts_ms - self.cluster_ts_ms) as i16;
|
||||
self.write_simple_block(track_idx + 1, relative_ts, keyframe, data)?;
|
||||
match duration_ns {
|
||||
Some(dur_ns) => {
|
||||
let duration_ms = (dur_ns / 1_000_000).max(1);
|
||||
self.write_block_group(track_idx + 1, relative_ts, keyframe, data, duration_ms)?;
|
||||
}
|
||||
None => {
|
||||
self.write_simple_block(track_idx + 1, relative_ts, keyframe, data)?;
|
||||
}
|
||||
}
|
||||
self.frame_count += 1;
|
||||
|
||||
Ok(())
|
||||
@@ -377,6 +445,8 @@ impl<W: Write + Seek> MkvMuxer<W> {
|
||||
self.end_cluster()?;
|
||||
|
||||
// Write Cues
|
||||
let cues_start = self.writer.stream_position()?;
|
||||
let cues_offset = cues_start - self.segment_start;
|
||||
if !self.cues.is_empty() {
|
||||
let cues_pos = ebml::start_master(&mut self.writer, ebml::CUES)?;
|
||||
for cue in &self.cues {
|
||||
@@ -395,6 +465,21 @@ impl<W: Write + Seek> MkvMuxer<W> {
|
||||
ebml::end_master(&mut self.writer, cues_pos)?;
|
||||
}
|
||||
|
||||
// Back-patch SeekHead SeekPosition values now that all element offsets are known.
|
||||
for fixup in &self.seek_fixups {
|
||||
let offset = match fixup.target_id {
|
||||
ebml::INFO => self.info_offset,
|
||||
ebml::TRACKS => self.tracks_offset,
|
||||
ebml::CHAPTERS => self.chapters_offset.unwrap_or(0),
|
||||
ebml::CUES => cues_offset,
|
||||
_ => 0,
|
||||
};
|
||||
self.writer
|
||||
.seek(std::io::SeekFrom::Start(fixup.value_offset))?;
|
||||
self.writer.write_all(&offset.to_be_bytes())?;
|
||||
}
|
||||
self.writer.seek(std::io::SeekFrom::End(0))?;
|
||||
|
||||
self.writer.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -447,6 +532,34 @@ impl<W: Write + Seek> MkvMuxer<W> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_block_group(
|
||||
&mut self,
|
||||
track_num: usize,
|
||||
relative_ts: i16,
|
||||
keyframe: bool,
|
||||
data: &[u8],
|
||||
duration_ms: u64,
|
||||
) -> io::Result<()> {
|
||||
let track_vint = if track_num < 0x80 {
|
||||
vec![(track_num as u8) | 0x80]
|
||||
} else {
|
||||
vec![0x40 | ((track_num >> 8) as u8), track_num as u8]
|
||||
};
|
||||
let flags: u8 = if keyframe { 0x80 } else { 0x00 };
|
||||
let block_size = track_vint.len() + 2 + 1 + data.len();
|
||||
|
||||
let bg_pos = ebml::start_master(&mut self.writer, ebml::BLOCK_GROUP)?;
|
||||
ebml::write_id(&mut self.writer, ebml::BLOCK)?;
|
||||
ebml::write_size(&mut self.writer, block_size as u64)?;
|
||||
self.writer.write_all(&track_vint)?;
|
||||
self.writer.write_all(&relative_ts.to_be_bytes())?;
|
||||
self.writer.write_all(&[flags])?;
|
||||
self.writer.write_all(data)?;
|
||||
ebml::write_uint(&mut self.writer, ebml::BLOCK_DURATION, duration_ms)?;
|
||||
ebml::end_master(&mut self.writer, bg_pos)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
@@ -556,7 +669,7 @@ mod tests {
|
||||
let tracks = [make_video_track()];
|
||||
let mut muxer = MkvMuxer::new(buf, &tracks, None, 60.0, &[]).unwrap();
|
||||
muxer
|
||||
.write_frame(0, 0, true, &[0xDE, 0xAD, 0xBE, 0xEF])
|
||||
.write_frame(0, 0, true, &[0xDE, 0xAD, 0xBE, 0xEF], None)
|
||||
.unwrap();
|
||||
let data = muxer.writer.into_inner();
|
||||
assert!(
|
||||
@@ -594,7 +707,9 @@ mod tests {
|
||||
let writer = SharedWriter(shared.clone());
|
||||
let tracks = [make_video_track()];
|
||||
let mut muxer = MkvMuxer::new(writer, &tracks, Some("Cue Test"), 60.0, &[]).unwrap();
|
||||
muxer.write_frame(0, 0, true, &[0x01, 0x02, 0x03]).unwrap();
|
||||
muxer
|
||||
.write_frame(0, 0, true, &[0x01, 0x02, 0x03], None)
|
||||
.unwrap();
|
||||
muxer.finish().unwrap();
|
||||
|
||||
let data = shared.lock().unwrap().clone().into_inner();
|
||||
@@ -610,13 +725,17 @@ mod tests {
|
||||
let tracks = [make_video_track(), make_audio_track()];
|
||||
let mut muxer = MkvMuxer::new(buf, &tracks, Some("Multi"), 120.0, &[]).unwrap();
|
||||
// Write frames to both tracks
|
||||
muxer.write_frame(0, 0, true, &[0x00, 0x00, 0x01]).unwrap();
|
||||
muxer.write_frame(1, 0, false, &[0x0B, 0x77, 0x00]).unwrap();
|
||||
muxer
|
||||
.write_frame(0, 40_000_000, false, &[0x00, 0x00, 0x01])
|
||||
.write_frame(0, 0, true, &[0x00, 0x00, 0x01], None)
|
||||
.unwrap();
|
||||
muxer
|
||||
.write_frame(1, 32_000_000, false, &[0x0B, 0x77, 0x01])
|
||||
.write_frame(1, 0, false, &[0x0B, 0x77, 0x00], None)
|
||||
.unwrap();
|
||||
muxer
|
||||
.write_frame(0, 40_000_000, false, &[0x00, 0x00, 0x01], None)
|
||||
.unwrap();
|
||||
muxer
|
||||
.write_frame(1, 32_000_000, false, &[0x0B, 0x77, 0x01], None)
|
||||
.unwrap();
|
||||
// Should not panic
|
||||
let data = muxer.writer.into_inner();
|
||||
@@ -631,10 +750,12 @@ mod tests {
|
||||
|
||||
// Record position before first frame
|
||||
let pos_before_kf = muxer.writer.position();
|
||||
muxer.write_frame(0, 0, true, &[0xAA]).unwrap();
|
||||
muxer.write_frame(0, 0, true, &[0xAA], None).unwrap();
|
||||
let pos_after_kf = muxer.writer.position();
|
||||
|
||||
muxer.write_frame(0, 1_000_000, false, &[0xBB]).unwrap();
|
||||
muxer
|
||||
.write_frame(0, 1_000_000, false, &[0xBB], None)
|
||||
.unwrap();
|
||||
let pos_after_nkf = muxer.writer.position();
|
||||
|
||||
let data = muxer.writer.into_inner();
|
||||
@@ -812,4 +933,422 @@ mod tests {
|
||||
"FlagForced element should not be present for non-forced subtitle"
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Seekability tests: SeekHead, keyframe-aligned clusters, Cues
|
||||
// ============================================================
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
/// Writer that lets the test inspect the buffer after `finish()` consumes the muxer.
|
||||
struct SharedWriter(Arc<Mutex<Cursor<Vec<u8>>>>);
|
||||
impl Write for SharedWriter {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.0.lock().unwrap().write(buf)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.0.lock().unwrap().flush()
|
||||
}
|
||||
}
|
||||
impl Seek for SharedWriter {
|
||||
fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
|
||||
self.0.lock().unwrap().seek(pos)
|
||||
}
|
||||
}
|
||||
|
||||
/// Build interleaved frames at 24 fps video (IDR every gop_secs) + 48 kHz audio (1024 samples per frame).
|
||||
fn frames_for(duration_secs: f64, gop_secs: f64) -> Vec<(usize, i64, bool, Vec<u8>)> {
|
||||
let video_interval_ns: i64 = 1_000_000_000 / 24;
|
||||
let audio_interval_ns: i64 = (1024i64 * 1_000_000_000) / 48_000;
|
||||
let gop_frames = (gop_secs * 24.0).round() as i64;
|
||||
|
||||
let mut out: Vec<(usize, i64, bool, Vec<u8>)> = Vec::new();
|
||||
let total_ns = (duration_secs * 1_000_000_000.0) as i64;
|
||||
|
||||
let mut vi: i64 = 0;
|
||||
loop {
|
||||
let pts = vi * video_interval_ns;
|
||||
if pts >= total_ns {
|
||||
break;
|
||||
}
|
||||
let keyframe = vi % gop_frames == 0;
|
||||
out.push((0, pts, keyframe, vec![0xAB; 64]));
|
||||
vi += 1;
|
||||
}
|
||||
|
||||
let mut ai: i64 = 0;
|
||||
loop {
|
||||
let pts = ai * audio_interval_ns;
|
||||
if pts >= total_ns {
|
||||
break;
|
||||
}
|
||||
out.push((1, pts, true, vec![0xCD; 32]));
|
||||
ai += 1;
|
||||
}
|
||||
|
||||
out.sort_by_key(|f| f.1);
|
||||
out
|
||||
}
|
||||
|
||||
/// Mux frames through a SharedWriter and return the final buffer.
|
||||
fn mux_to_bytes(
|
||||
tracks: &[MkvTrack],
|
||||
chapters: &[Chapter],
|
||||
frames: &[(usize, i64, bool, Vec<u8>)],
|
||||
) -> (Vec<u8>, u64) {
|
||||
let shared = Arc::new(Mutex::new(Cursor::new(Vec::new())));
|
||||
let writer = SharedWriter(shared.clone());
|
||||
let mut muxer = MkvMuxer::new(writer, tracks, None, 0.0, chapters).unwrap();
|
||||
for (t, pts, kf, data) in frames {
|
||||
muxer.write_frame(*t, *pts, *kf, data, None).unwrap();
|
||||
}
|
||||
let frame_count = muxer.frame_count;
|
||||
muxer.finish().unwrap();
|
||||
let data = shared.lock().unwrap().clone().into_inner();
|
||||
(data, frame_count)
|
||||
}
|
||||
|
||||
/// Find the Segment header in the buffer and return (segment_id_pos, segment_start_pos).
|
||||
/// segment_start = position immediately after Segment's id + size bytes.
|
||||
fn locate_segment(data: &[u8]) -> (usize, usize) {
|
||||
let segment_id_pos = find_id(data, ebml::SEGMENT).expect("segment id not found");
|
||||
// Segment is written via write_id + write_unknown_size: 4 byte id + 8 byte size
|
||||
(segment_id_pos, segment_id_pos + 4 + 8)
|
||||
}
|
||||
|
||||
/// Walk Segment's top-level children. Returns Vec<(id, data_start_offset, data_size)>
|
||||
/// where data_start_offset is absolute file offset and data_size is the element body size.
|
||||
fn segment_children(data: &[u8]) -> Vec<(u32, usize, u64)> {
|
||||
let (_, seg_start) = locate_segment(data);
|
||||
let mut out = Vec::new();
|
||||
let mut cursor = Cursor::new(&data[seg_start..]);
|
||||
while (cursor.position() as usize) < data.len() - seg_start {
|
||||
let pos_before = cursor.position();
|
||||
let (id, size, hdr_len) = match ebml::read_element_header(&mut cursor) {
|
||||
Ok(v) => v,
|
||||
Err(_) => break,
|
||||
};
|
||||
let data_abs = seg_start + pos_before as usize + hdr_len;
|
||||
out.push((id, data_abs, size));
|
||||
// Skip the body to advance to the next element.
|
||||
cursor
|
||||
.seek(io::SeekFrom::Current(size as i64))
|
||||
.expect("seek past element body");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Find every Cluster: returns Vec<(cluster_data_start_abs, cluster_data_size, cluster_timestamp_ms)>.
|
||||
fn find_clusters(data: &[u8]) -> Vec<(usize, u64, u64)> {
|
||||
let mut out = Vec::new();
|
||||
for (id, body_start, body_size) in segment_children(data) {
|
||||
if id == ebml::CLUSTER {
|
||||
let mut cursor = Cursor::new(&data[body_start..body_start + body_size as usize]);
|
||||
let (tid, tsize, _) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
assert_eq!(
|
||||
tid,
|
||||
ebml::CLUSTER_TIMESTAMP,
|
||||
"cluster must start with timestamp"
|
||||
);
|
||||
let ts = ebml::read_uint_val(&mut cursor, tsize as usize).unwrap();
|
||||
out.push((body_start, body_size, ts));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Parse the first SimpleBlock that appears in a cluster body slice.
|
||||
/// Returns (track_num, flags_byte). track_num decoded from VINT.
|
||||
fn first_simple_block(cluster_body: &[u8]) -> (u64, u8) {
|
||||
let mut cursor = Cursor::new(cluster_body);
|
||||
loop {
|
||||
let (id, size, _) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
if id == ebml::SIMPLE_BLOCK {
|
||||
let body_start = cursor.position() as usize;
|
||||
// Decode track VINT.
|
||||
let b0 = cluster_body[body_start];
|
||||
let (track_num, vint_len) = if b0 & 0x80 != 0 {
|
||||
((b0 & 0x7F) as u64, 1usize)
|
||||
} else if b0 & 0x40 != 0 {
|
||||
let b1 = cluster_body[body_start + 1];
|
||||
((((b0 & 0x3F) as u64) << 8) | b1 as u64, 2)
|
||||
} else {
|
||||
panic!("unsupported track vint width");
|
||||
};
|
||||
let flags = cluster_body[body_start + vint_len + 2];
|
||||
return (track_num, flags);
|
||||
}
|
||||
// Skip non-SimpleBlock child.
|
||||
cursor.seek(io::SeekFrom::Current(size as i64)).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse the Cues element body into Vec<(cue_time, cue_track, cue_cluster_position)>.
|
||||
fn parse_cues(data: &[u8]) -> Vec<(u64, u64, u64)> {
|
||||
let mut out = Vec::new();
|
||||
let (cues_id, cues_body_start, cues_body_size) = segment_children(data)
|
||||
.into_iter()
|
||||
.find(|(id, _, _)| *id == ebml::CUES)
|
||||
.expect("cues element not found");
|
||||
assert_eq!(cues_id, ebml::CUES);
|
||||
let cues_body = &data[cues_body_start..cues_body_start + cues_body_size as usize];
|
||||
let mut cursor = Cursor::new(cues_body);
|
||||
while (cursor.position() as usize) < cues_body.len() {
|
||||
let (id, size, _) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
assert_eq!(id, ebml::CUE_POINT);
|
||||
let cp_end = cursor.position() + size;
|
||||
let mut cue_time = 0u64;
|
||||
let mut cue_track = 0u64;
|
||||
let mut cue_pos = 0u64;
|
||||
while cursor.position() < cp_end {
|
||||
let (sid, ssize, _) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
match sid {
|
||||
ebml::CUE_TIME => {
|
||||
cue_time = ebml::read_uint_val(&mut cursor, ssize as usize).unwrap();
|
||||
}
|
||||
ebml::CUE_TRACK_POSITIONS => {
|
||||
let ctp_end = cursor.position() + ssize;
|
||||
while cursor.position() < ctp_end {
|
||||
let (iid, isize_, _) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
match iid {
|
||||
ebml::CUE_TRACK => {
|
||||
cue_track =
|
||||
ebml::read_uint_val(&mut cursor, isize_ as usize).unwrap();
|
||||
}
|
||||
ebml::CUE_CLUSTER_POSITION => {
|
||||
cue_pos =
|
||||
ebml::read_uint_val(&mut cursor, isize_ as usize).unwrap();
|
||||
}
|
||||
_ => {
|
||||
cursor.seek(io::SeekFrom::Current(isize_ as i64)).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
cursor.seek(io::SeekFrom::Current(ssize as i64)).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push((cue_time, cue_track, cue_pos));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Parse the SeekHead body into Vec<(seek_id, seek_position)>.
|
||||
fn parse_seekhead(data: &[u8]) -> Vec<(u32, u64)> {
|
||||
let mut out = Vec::new();
|
||||
let (sh_id, sh_body_start, sh_body_size) = segment_children(data)
|
||||
.into_iter()
|
||||
.find(|(id, _, _)| *id == ebml::SEEK_HEAD)
|
||||
.expect("seekhead not found");
|
||||
assert_eq!(sh_id, ebml::SEEK_HEAD);
|
||||
let sh_body = &data[sh_body_start..sh_body_start + sh_body_size as usize];
|
||||
let mut cursor = Cursor::new(sh_body);
|
||||
while (cursor.position() as usize) < sh_body.len() {
|
||||
let (id, size, _) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
assert_eq!(id, ebml::SEEK);
|
||||
let seek_end = cursor.position() + size;
|
||||
let mut seek_id_val: u32 = 0;
|
||||
let mut seek_pos_val: u64 = 0;
|
||||
while cursor.position() < seek_end {
|
||||
let (sid, ssize, _) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
match sid {
|
||||
ebml::SEEK_ID => {
|
||||
let raw = ebml::read_uint_val(&mut cursor, ssize as usize).unwrap();
|
||||
seek_id_val = raw as u32;
|
||||
}
|
||||
ebml::SEEK_POSITION => {
|
||||
seek_pos_val = ebml::read_uint_val(&mut cursor, ssize as usize).unwrap();
|
||||
}
|
||||
_ => {
|
||||
cursor.seek(io::SeekFrom::Current(ssize as i64)).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push((seek_id_val, seek_pos_val));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_starts_only_on_video_keyframe() {
|
||||
let tracks = [make_video_track(), make_audio_track()];
|
||||
let frames = frames_for(30.0, 1.0);
|
||||
let (data, _) = mux_to_bytes(&tracks, &[], &frames);
|
||||
let clusters = find_clusters(&data);
|
||||
assert!(!clusters.is_empty(), "expected at least one cluster");
|
||||
for (body_start, body_size, _ts) in clusters {
|
||||
let body = &data[body_start..body_start + body_size as usize];
|
||||
// Skip past the CLUSTER_TIMESTAMP element first.
|
||||
let mut cursor = Cursor::new(body);
|
||||
let (tid, tsize, _) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
assert_eq!(tid, ebml::CLUSTER_TIMESTAMP);
|
||||
cursor.seek(io::SeekFrom::Current(tsize as i64)).unwrap();
|
||||
let after_ts = cursor.position() as usize;
|
||||
let (track_num, flags) = first_simple_block(&body[after_ts..]);
|
||||
assert_eq!(
|
||||
track_num, 1,
|
||||
"first block in cluster must be track 1 (video)"
|
||||
);
|
||||
assert_eq!(
|
||||
flags & 0x80,
|
||||
0x80,
|
||||
"first block in cluster must have keyframe flag set, got 0x{:02X}",
|
||||
flags
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cue_count_equals_cluster_count() {
|
||||
let tracks = [make_video_track(), make_audio_track()];
|
||||
let frames = frames_for(30.0, 1.0);
|
||||
let (data, _) = mux_to_bytes(&tracks, &[], &frames);
|
||||
let clusters = find_clusters(&data);
|
||||
let cues = parse_cues(&data);
|
||||
assert_eq!(
|
||||
clusters.len(),
|
||||
cues.len(),
|
||||
"cluster count {} != cue count {}",
|
||||
clusters.len(),
|
||||
cues.len()
|
||||
);
|
||||
// For 30s @ 5s min cluster duration with 1s GOP, expect 6 clusters / 6 cues.
|
||||
assert_eq!(
|
||||
clusters.len(),
|
||||
6,
|
||||
"expected 6 clusters for 30s @ 5s cluster duration"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cue_positions_resolve_to_clusters() {
|
||||
let tracks = [make_video_track(), make_audio_track()];
|
||||
let frames = frames_for(30.0, 1.0);
|
||||
let (data, _) = mux_to_bytes(&tracks, &[], &frames);
|
||||
let (_, seg_start) = locate_segment(&data);
|
||||
let cues = parse_cues(&data);
|
||||
assert!(!cues.is_empty());
|
||||
for (_time, _track, pos) in cues {
|
||||
let abs = seg_start + pos as usize;
|
||||
let mut cursor = Cursor::new(&data[abs..]);
|
||||
let (id, _size, _hdr_len) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
assert_eq!(
|
||||
id,
|
||||
ebml::CLUSTER,
|
||||
"cue position 0x{:X} did not resolve to a cluster",
|
||||
pos
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cue_times_match_cluster_timestamps() {
|
||||
let tracks = [make_video_track(), make_audio_track()];
|
||||
let frames = frames_for(30.0, 1.0);
|
||||
let (data, _) = mux_to_bytes(&tracks, &[], &frames);
|
||||
let (_, seg_start) = locate_segment(&data);
|
||||
let cues = parse_cues(&data);
|
||||
for (time, _track, pos) in cues {
|
||||
let abs = seg_start + pos as usize;
|
||||
let mut cursor = Cursor::new(&data[abs..]);
|
||||
let (id, size, _hdr_len) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
assert_eq!(id, ebml::CLUSTER);
|
||||
let body_start = abs + (cursor.position() as usize);
|
||||
let body = &data[body_start..body_start + size as usize];
|
||||
let mut bc = Cursor::new(body);
|
||||
let (tid, tsize, _) = ebml::read_element_header(&mut bc).unwrap();
|
||||
assert_eq!(tid, ebml::CLUSTER_TIMESTAMP);
|
||||
let cluster_ts = ebml::read_uint_val(&mut bc, tsize as usize).unwrap();
|
||||
assert_eq!(
|
||||
cluster_ts, time,
|
||||
"cluster timestamp {} != cue time {}",
|
||||
cluster_ts, time
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seekhead_is_first_child_of_segment() {
|
||||
let tracks = [make_video_track(), make_audio_track()];
|
||||
let (data, _) = mux_to_bytes(&tracks, &[], &frames_for(10.0, 1.0));
|
||||
let children = segment_children(&data);
|
||||
assert!(!children.is_empty());
|
||||
assert_eq!(
|
||||
children[0].0,
|
||||
ebml::SEEK_HEAD,
|
||||
"first child of segment must be SeekHead, got id 0x{:X}",
|
||||
children[0].0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seekhead_points_to_real_elements() {
|
||||
let tracks = [make_video_track(), make_audio_track()];
|
||||
let (data, _) = mux_to_bytes(&tracks, &[], &frames_for(10.0, 1.0));
|
||||
let (_, seg_start) = locate_segment(&data);
|
||||
let entries = parse_seekhead(&data);
|
||||
let required = [ebml::INFO, ebml::TRACKS, ebml::CUES];
|
||||
for &want_id in &required {
|
||||
let entry = entries
|
||||
.iter()
|
||||
.find(|(id, _)| *id == want_id)
|
||||
.unwrap_or_else(|| panic!("seekhead missing entry for id 0x{:X}", want_id));
|
||||
let abs = seg_start + entry.1 as usize;
|
||||
let mut cursor = Cursor::new(&data[abs..]);
|
||||
let (got_id, _, _) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
assert_eq!(
|
||||
got_id, want_id,
|
||||
"seekhead entry for 0x{:X} resolves to wrong id 0x{:X}",
|
||||
want_id, got_id
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seekhead_omits_chapters_when_empty() {
|
||||
let tracks = [make_video_track()];
|
||||
let (data, _) = mux_to_bytes(&tracks, &[], &frames_for(5.0, 1.0));
|
||||
let entries = parse_seekhead(&data);
|
||||
assert_eq!(
|
||||
entries.len(),
|
||||
3,
|
||||
"expected 3 seek entries (Info, Tracks, Cues), got {}",
|
||||
entries.len()
|
||||
);
|
||||
assert!(
|
||||
entries.iter().all(|(id, _)| *id != ebml::CHAPTERS),
|
||||
"seekhead should not contain Chapters entry when chapters are empty"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pre_first_keyframe_frames_dropped() {
|
||||
let tracks = [make_video_track()];
|
||||
let frames = vec![
|
||||
(0usize, 0i64, false, vec![0x11; 16]),
|
||||
(0usize, 41_000_000i64, true, vec![0x22; 16]),
|
||||
];
|
||||
let (data, frame_count) = mux_to_bytes(&tracks, &[], &frames);
|
||||
assert_eq!(frame_count, 1, "muxer.frame_count must equal 1");
|
||||
let clusters = find_clusters(&data);
|
||||
assert_eq!(clusters.len(), 1, "expected exactly one cluster");
|
||||
let (body_start, body_size, _ts) = clusters[0];
|
||||
let body = &data[body_start..body_start + body_size as usize];
|
||||
let mut cursor = Cursor::new(body);
|
||||
// Skip CLUSTER_TIMESTAMP.
|
||||
let (tid, tsize, _) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
assert_eq!(tid, ebml::CLUSTER_TIMESTAMP);
|
||||
cursor.seek(io::SeekFrom::Current(tsize as i64)).unwrap();
|
||||
let mut sb_count = 0;
|
||||
while (cursor.position() as usize) < body.len() {
|
||||
let (id, sz, _) = ebml::read_element_header(&mut cursor).unwrap();
|
||||
if id == ebml::SIMPLE_BLOCK {
|
||||
sb_count += 1;
|
||||
}
|
||||
cursor.seek(io::SeekFrom::Current(sz as i64)).unwrap();
|
||||
}
|
||||
assert_eq!(sb_count, 1, "expected exactly one SimpleBlock in output");
|
||||
}
|
||||
}
|
||||
|
||||
+12
-7
@@ -18,7 +18,7 @@ use crate::disc::*;
|
||||
use std::io::{self, Read};
|
||||
|
||||
struct ReadState {
|
||||
reader: Box<dyn Read>,
|
||||
reader: Box<dyn Read + Send>,
|
||||
cluster_ts_ms: i64,
|
||||
/// Codec private data per track (track_number, hvcC/avcC bytes).
|
||||
codec_privates: Vec<(u16, Vec<u8>)>,
|
||||
@@ -26,7 +26,7 @@ struct ReadState {
|
||||
|
||||
enum Mode {
|
||||
Write {
|
||||
muxer: Option<MkvMuxer<Box<dyn WriteSeek>>>,
|
||||
muxer: Option<MkvMuxer<Box<dyn WriteSeek + Send>>>,
|
||||
},
|
||||
Read(ReadState),
|
||||
}
|
||||
@@ -40,7 +40,7 @@ pub struct MkvStream {
|
||||
impl MkvStream {
|
||||
/// Create for writing PES frames → MKV container.
|
||||
/// Codec privates come from title.codec_privates (populated by input stream).
|
||||
pub fn create(writer: Box<dyn WriteSeek>, title: &DiscTitle) -> io::Result<Self> {
|
||||
pub fn create(writer: Box<dyn WriteSeek + Send>, title: &DiscTitle) -> io::Result<Self> {
|
||||
let mut tracks = Vec::new();
|
||||
let mut has_default_video = false;
|
||||
let mut has_default_audio = false;
|
||||
@@ -79,7 +79,7 @@ impl MkvStream {
|
||||
}
|
||||
|
||||
/// Open an MKV file for reading → PES frames.
|
||||
pub fn open(mut reader: impl Read + 'static) -> io::Result<Self> {
|
||||
pub fn open(mut reader: impl Read + Send + 'static) -> io::Result<Self> {
|
||||
let (disc_title, codec_privates) = parse_mkv_header(&mut reader)?;
|
||||
Ok(Self {
|
||||
disc_title,
|
||||
@@ -138,6 +138,7 @@ impl crate::pes::Stream for MkvStream {
|
||||
pts: pts_ms * 1_000_000, // ms → ns
|
||||
keyframe,
|
||||
data,
|
||||
duration_ns: None,
|
||||
}));
|
||||
}
|
||||
_ => {
|
||||
@@ -150,9 +151,13 @@ impl crate::pes::Stream for MkvStream {
|
||||
|
||||
fn write(&mut self, frame: &crate::pes::PesFrame) -> io::Result<()> {
|
||||
match &mut self.mode {
|
||||
Mode::Write { muxer: Some(m) } => {
|
||||
m.write_frame(frame.track, frame.pts, frame.keyframe, &frame.data)
|
||||
}
|
||||
Mode::Write { muxer: Some(m) } => m.write_frame(
|
||||
frame.track,
|
||||
frame.pts,
|
||||
frame.keyframe,
|
||||
&frame.data,
|
||||
frame.duration_ns,
|
||||
),
|
||||
Mode::Write { muxer: None } => Ok(()),
|
||||
Mode::Read(_) => Err(crate::error::Error::StreamReadOnly.into()),
|
||||
}
|
||||
|
||||
+15
-2
@@ -17,8 +17,9 @@
|
||||
|
||||
// Public modules — types here are intentionally part of the consumable API.
|
||||
pub mod codec;
|
||||
pub mod demux_thread;
|
||||
pub mod disc;
|
||||
pub mod iso;
|
||||
pub mod pipelined_stream;
|
||||
pub mod resolve;
|
||||
|
||||
// Internal modules — implementation details. Their *types* are re-exported
|
||||
@@ -33,6 +34,17 @@ pub(crate) mod m2ts;
|
||||
/// to round-trip codec_privates that don't fit inside the underlying format).
|
||||
/// Exposed for integration tests that exercise the wire format directly.
|
||||
pub mod meta;
|
||||
|
||||
// ── Phase 3 sequential muxers ──────────────────────────────────────────────
|
||||
//
|
||||
// New container muxers that consume PES frames and write to a
|
||||
// `SequentialSink`. They are NOT refactors of the existing `MkvStream` /
|
||||
// `M2tsStream` (which round-trip via the legacy `Stream` trait + the
|
||||
// BD-TS framing); they're sequential-only and target the Phase 2 sink
|
||||
// split end-to-end.
|
||||
pub mod fmp4;
|
||||
pub mod hevc;
|
||||
pub mod m2ts_mux;
|
||||
pub(crate) mod mkv;
|
||||
pub(crate) mod mkvstream;
|
||||
pub(crate) mod network;
|
||||
@@ -43,11 +55,12 @@ pub(crate) mod ts;
|
||||
pub(crate) mod tsmux;
|
||||
|
||||
pub use disc::DiscStream;
|
||||
pub use iso::IsoSectorReader;
|
||||
pub use m2ts::M2tsStream;
|
||||
pub use mkvstream::MkvStream;
|
||||
pub use network::NetworkStream;
|
||||
pub use null::NullStream;
|
||||
pub use pipelined_stream::PipelinedPesStream;
|
||||
pub use resolve::build_iso_pipeline;
|
||||
pub use resolve::{InputOptions, StreamUrl, input, output, parse_url};
|
||||
pub use stdio::StdioStream;
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user