mirror of
https://github.com/freemkv/libfreemkv.git
synced 2026-10-07 18:13:10 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7d58ba7b08 | ||
|
|
4f1dbfd042 | ||
|
|
5b702a76a7 | ||
|
|
6be5198886 | ||
|
|
e2aa9abd6d | ||
|
|
3b7bee9ed4 | ||
|
|
b7405e2d27 | ||
|
|
e04d79c593 | ||
|
|
b34af1fa74 | ||
|
|
a7317f8885 | ||
|
|
474273afc0 | ||
|
|
0a2bab5789 | ||
|
|
fa4d7ef871 | ||
|
|
36d1af1b7f | ||
|
|
b96f6206fe | ||
|
|
bb32fb993b | ||
|
|
c1eb74dfa5 | ||
|
|
46838c63ca | ||
|
|
1e60220ff6 | ||
|
|
080f03e8ed | ||
|
|
35de6101d4 | ||
|
|
9012101573 | ||
|
|
8bc1de6c9b | ||
|
|
d94a4d3444 | ||
|
|
1bd7e1de2a | ||
|
|
da62ee7cf2 | ||
|
|
575c76156f | ||
|
|
b518860d9c | ||
|
|
c8eb42b490 | ||
|
|
48570ac065 | ||
|
|
08e46640fd | ||
|
|
a9195824ff | ||
|
|
401fe23988 | ||
|
|
f89bce5851 | ||
|
|
dfccb85e15 | ||
|
|
ebedffb762 | ||
|
|
fdf63ccb7f | ||
|
|
c735d284da | ||
|
|
94ab7bc73c | ||
|
|
9f209fe066 | ||
|
|
32a1a6e095 | ||
|
|
1d3b8f5fb6 | ||
|
|
97b0ae7be2 | ||
|
|
7756f1feca | ||
|
|
1565da610a | ||
|
|
e134616422 | ||
|
|
8d54a3c64e | ||
|
|
c0478e1273 | ||
|
|
7d29168fec | ||
|
|
8e14c9b850 | ||
|
|
e1c8343f77 | ||
|
|
a956c6ad94 | ||
|
|
1805d92ca4 | ||
|
|
823f0ad430 | ||
|
|
477bdf1835 | ||
|
|
4d83b69c20 | ||
|
|
7dbbfc6726 | ||
|
|
e635c9556f | ||
|
|
d7b5c30f5d | ||
|
|
739a276a39 | ||
|
|
5f1028a62a | ||
|
|
ea15d212de | ||
|
|
00673c8ec3 | ||
|
|
bfefb4cb5b | ||
|
|
1b95193517 | ||
|
|
7dcac44136 | ||
|
|
4da559e39f | ||
|
|
eeca250f69 | ||
|
|
c51b3181f2 | ||
|
|
2a31a47434 | ||
|
|
b08662f95c | ||
|
|
4594196b6e | ||
|
|
9d6974df3e | ||
|
|
3394a5b3fe | ||
|
|
e2c7e20329 | ||
|
|
ff22385a61 | ||
|
|
d39d3933f5 | ||
|
|
31dc3d35c3 | ||
|
|
f7165a2020 | ||
|
|
d836f8b1e7 | ||
|
|
8919de5b03 | ||
|
|
0dc1f12108 | ||
|
|
6abc25a7e8 | ||
|
|
4d22cf632f | ||
|
|
611ad77586 | ||
|
|
60590193e3 | ||
|
|
5f24843d3c | ||
|
|
ecf9a4d01e | ||
|
|
8ec767339e | ||
|
|
335e8b68cc | ||
|
|
1137b4ea1f | ||
|
|
7788c560d1 | ||
|
|
84e0c2ca69 | ||
|
|
f4a881b25d | ||
|
|
1edda7fedb | ||
|
|
3dd71a0650 | ||
|
|
579c94bd23 | ||
|
|
3b2a68fe7a | ||
|
|
5b56f112a0 | ||
|
|
f27e4c3088 | ||
|
|
b3576f2ae6 | ||
|
|
39f2d0e992 | ||
|
|
50867516b8 | ||
|
|
193e7c1680 | ||
|
|
b6463729e8 | ||
|
|
dffee56102 | ||
|
|
d8f27cdee0 | ||
|
|
fc13268dc6 | ||
|
|
d9b55f02d0 | ||
|
|
35d66d2059 | ||
|
|
d703ce439b | ||
|
|
6112a26d15 | ||
|
|
2a33364253 | ||
|
|
fa3872ddcc | ||
|
|
c427389f36 | ||
|
|
6980f562df | ||
|
|
e3b2c9d850 | ||
|
|
a383200ef1 | ||
|
|
51d15f551e | ||
|
|
523af461a5 | ||
|
|
695b65149e | ||
|
|
d3f9560689 | ||
|
|
1fa5a7d27f | ||
|
|
55219070f1 | ||
|
|
a34c419521 | ||
|
|
b179846f5d | ||
|
|
e110e80e6e | ||
|
|
597fa34099 | ||
|
|
52eeb949e3 | ||
|
|
6ca62b8411 | ||
|
|
04195c27d8 | ||
|
|
5a8f8e54e1 | ||
|
|
1a2c830cf6 | ||
|
|
637c2bfe36 | ||
|
|
0d28357b4d | ||
|
|
5495332f07 | ||
|
|
e22fc6fd47 | ||
|
|
5b98c13e47 | ||
|
|
8f8f1a62a2 | ||
|
|
e3cfd27942 | ||
|
|
aa12bdad62 | ||
|
|
ef3895cdc5 | ||
|
|
2dcf969ac8 | ||
|
|
005f887bf9 | ||
|
|
d7243a6044 | ||
|
|
f1926c38dc | ||
|
|
4709a73c80 | ||
|
|
55bd1ee868 | ||
|
|
5285e8b6ec | ||
|
|
c5af61e46b | ||
|
|
ed73c1ab68 | ||
|
|
5ce2df0888 | ||
|
|
ea591a36a5 | ||
|
|
368e10486e | ||
|
|
a99c4f8487 | ||
|
|
4df470e572 | ||
|
|
a0b13941fe | ||
|
|
275c9da009 | ||
|
|
da183722fa | ||
|
|
4a5424eabe | ||
|
|
293d89c586 | ||
|
|
4840f5134d | ||
|
|
11c8211605 | ||
|
|
1038beaf06 | ||
|
|
f1df57196c | ||
|
|
8ba9dc1c0b | ||
|
|
49b131f39a | ||
|
|
8238ec4ce7 | ||
|
|
4fd5d27df8 | ||
|
|
1fbe272832 | ||
|
|
6bd635725e | ||
|
|
09b77b4dea | ||
|
|
688058b3e8 | ||
|
|
c96bac7977 | ||
|
|
c32acff3e4 | ||
|
|
b58e2d9873 | ||
|
|
a9e802c1c2 | ||
|
|
a876ce846b | ||
|
|
058fd8396f | ||
|
|
5ee28c08b9 | ||
|
|
764230b1eb | ||
|
|
7fb2e07aed | ||
|
|
376aadb335 | ||
|
|
586495b2ee | ||
|
|
72f2224efe | ||
|
|
0bca7a11bd | ||
|
|
72ab714c1e | ||
|
|
d7fb1b35ed | ||
|
|
06be4defd2 | ||
|
|
a832bad697 | ||
|
|
26e2d847d5 | ||
|
|
4442fa2df6 | ||
|
|
23dc55661c | ||
|
|
fbdb50c79f | ||
|
|
7cc74f0087 | ||
|
|
3aa1e528c5 | ||
|
|
7c6b0f82ab | ||
|
|
4226a53e73 | ||
|
|
92f34a289e | ||
|
|
fdbe469d50 | ||
|
|
4ec75a03f2 | ||
|
|
99236ffd55 | ||
|
|
9e3327a569 | ||
|
|
0081955686 | ||
|
|
3244fdd683 | ||
|
|
ef5487e5a5 | ||
|
|
e7dffa63d2 | ||
|
|
e5989c8270 | ||
|
|
e1c938b82a | ||
|
|
ed801a708b | ||
|
|
d94451954c | ||
|
|
385c9f094c | ||
|
|
9f2a13739d | ||
|
|
d9ce69bc9d | ||
|
|
7cd2c937ed | ||
|
|
107524da5e | ||
|
|
055a3c5276 | ||
|
|
50b04790ce | ||
|
|
a00b2c884b | ||
|
|
44e2c64d7e | ||
|
|
d6535b8f57 | ||
|
|
90ab00ed45 | ||
|
|
6bfd7dfd13 | ||
|
|
b9a7f601d4 | ||
|
|
cf0a61f8b4 | ||
|
|
f98f07b2d3 | ||
|
|
b53454fa09 | ||
|
|
f28b6ee6d9 | ||
|
|
d9d778fa64 | ||
|
|
bbfb887a35 | ||
|
|
9884346c24 | ||
|
|
766b7c6636 | ||
|
|
925c30686b | ||
|
|
f52e4c5d22 | ||
|
|
8d67790d2e | ||
|
|
246a439990 | ||
|
|
bd19041648 | ||
|
|
198268b725 | ||
|
|
31424c8203 | ||
|
|
c747ecc589 | ||
|
|
283a561c12 | ||
|
|
2667d68675 | ||
|
|
57f2c22e30 | ||
|
|
5f3545d244 | ||
|
|
6e17ef0859 | ||
|
|
6ec97af104 | ||
|
|
3a6c1aa5a3 | ||
|
|
1ba3264747 | ||
|
|
ae2909fe8d | ||
|
|
6a3d19a453 | ||
|
|
a35596d2d1 | ||
|
|
318f654fed | ||
|
|
5e82ef65de | ||
|
|
8534607329 | ||
|
|
97e1a4cad3 | ||
|
|
3f6bf33de5 | ||
|
|
359301e8fc | ||
|
|
bfa527162a | ||
|
|
45defd47f3 | ||
|
|
fe8933dad2 |
@@ -14,6 +14,10 @@ jobs:
|
||||
with:
|
||||
components: clippy, rustfmt
|
||||
- run: cargo fmt --check
|
||||
# libfreemkv is a library — Cargo.lock is gitignored. --locked
|
||||
# would always fail on a fresh runner because there's no committed
|
||||
# lockfile to lock against. The binary crates (freemkv, autorip,
|
||||
# bdemu) track Cargo.lock and DO use --locked.
|
||||
- run: cargo clippy -- -D warnings
|
||||
|
||||
test:
|
||||
|
||||
@@ -28,6 +28,8 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
# libfreemkv is a library — Cargo.lock isn't tracked, so --locked
|
||||
# would always fail (no lockfile to lock against on a fresh runner).
|
||||
- run: cargo test
|
||||
|
||||
publish:
|
||||
|
||||
@@ -4,3 +4,7 @@ Cargo.lock
|
||||
*.swo
|
||||
.DS_Store
|
||||
.cargo/
|
||||
.claude/worktrees/
|
||||
|
||||
# session scratch — never track (may contain RE breadcrumbs)
|
||||
scratch/
|
||||
|
||||
+866
-19
@@ -1,5 +1,854 @@
|
||||
# Changelog
|
||||
|
||||
## 0.29.0 (2026-06-06)
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Video frames now carry their presentation timestamp (PTS), not the decode
|
||||
timestamp (DTS), as the MKV block timecode.** The HEVC, H.264, VC-1 and
|
||||
MPEG-2 parsers stamped each frame with `dts.or(pts)`, so B-frame titles were
|
||||
muxed with monotonic decode-order timecodes: players presented frames in
|
||||
decode order (visible judder / apparent "corruption") and PTS-based seeking
|
||||
landed on the wrong frame. The compressed video was always byte-correct —
|
||||
this was purely a timestamp defect, and it affected every B-frame title. The
|
||||
parsers now use `pts.or(dts)`. Verified against a reference muxer on a real
|
||||
UHD title: the emitted frame sequence is now identical.
|
||||
- **DTS-HD MA tracks no longer downgrade to the lossy DTS core** — the access
|
||||
unit is delimited by the next *validated* core sync, so a `0x7FFE8001` byte
|
||||
pattern inside the XLL extension can no longer truncate the unit. (#10)
|
||||
- **A resolved AACS key is verified against the content before it is applied**,
|
||||
so a stale or wrong VUK can no longer silently produce garbage output.
|
||||
- **`iso://` mux fails fast when no usable AACS key is available** instead of
|
||||
writing a garbage MKV.
|
||||
|
||||
### Changed
|
||||
|
||||
- Codec / EBML / TS hardening: bounds-checked PMT and PES-header parsing, EBML
|
||||
length validation, and bounded codec buffers (DTS, TrueHD, PGS).
|
||||
|
||||
## 0.27.3 (2026-06-04)
|
||||
|
||||
### Added
|
||||
|
||||
- **The mapfile now persists resolved AACS unit keys, not just the Volume ID.**
|
||||
A keyed disc writes its decrypted unit keys as `# freemkv-uk: <cps>:<hex>`
|
||||
comment headers; an unresolved disc writes only the `# freemkv-vid:` marker.
|
||||
The two are **mutually exclusive** (`Mapfile::set_unit_keys` clears the VID):
|
||||
unit keys are the final answer, so a deferred-mux / resume decrypts directly
|
||||
with no key-service round-trip, while the VID alone means "still unresolved —
|
||||
retry the key service." `CopyOptions`/`SweepOptions` gain a `unit_keys` field
|
||||
(written when non-empty, else the VID), and `Disc::inject_unit_keys` applies
|
||||
mapfile-recovered keys to a scanned disc (marking the source `ExternalUk`).
|
||||
|
||||
## 0.27.0 (2026-06-03)
|
||||
|
||||
### Changed
|
||||
|
||||
- **AACS unit-encryption detection now reads the raw MPEG-TS sync bytes
|
||||
instead of header flag bits.** `is_unit_encrypted` is renamed
|
||||
`is_aacs_scrambled`: a unit is encrypted iff its body TS packet sync bytes
|
||||
(`0x47` at offset 4 and every 192 bytes) are not intact — the encrypted
|
||||
body destroys them. The previous check read the TP_extra copy-control bits
|
||||
(byte 0) or TS transport_scrambling_control bits (byte 7), which discs and
|
||||
players do not set reliably. A single predicate now backs both the decrypt
|
||||
gate and out-of-band key validation, so all callers agree on what
|
||||
"encrypted" means. Decryption restores the syncs, so a decrypted unit reads
|
||||
as clear and there is no encryption flag to clear.
|
||||
|
||||
## 0.26.1 (2026-05-22)
|
||||
|
||||
### Added
|
||||
|
||||
- **AACS resolver path 5 — KEYDB unit-keys direct fallback.** When a
|
||||
KEYDB entry for the disc has no VUK field but does have pre-decrypted
|
||||
unit keys, the resolver now consumes those unit keys directly instead
|
||||
of treating the entry as unusable. Covers ~4,572 entries in the
|
||||
public keydb (≈2.5% of the database, heavily skewed toward MKBv76+
|
||||
UHD discs that DVDFab/FindVUK can no longer extract VUKs for).
|
||||
Surfaces as `KeySource::KeyDbUnitKeys`; `AacsState::vuk` and
|
||||
`ResolvedKeys::vuk` are `None` on this path because no VUK exists to
|
||||
return. Partial CPS-unit coverage is rejected so a disc is never
|
||||
half-decrypted.
|
||||
|
||||
### Changed
|
||||
|
||||
- **AACS resolver path order reordered root-to-leaf.** Paths now run in
|
||||
derivation-strength order: device-key (1) → processing-key (2) →
|
||||
KEYDB-derived from MK+VID (3) → KEYDB VUK direct (4) → KEYDB unit
|
||||
keys direct (5). Previous order was leaf-first (KEYDB hash lookup,
|
||||
then derivation). Resolution runs once per disc, so the few extra
|
||||
milliseconds of MKB walking when a KEYDB VUK would have answered are
|
||||
invisible against the rip itself; the new order matches the AACS
|
||||
derivation hierarchy. `KeySource` enum variants are reordered to
|
||||
match.
|
||||
- **`AacsState::vuk` type is now `Option<[u8; 16]>`** (was `[u8; 16]`).
|
||||
Required by path 5, which has no VUK. Public-API break.
|
||||
- **`ResolvedKeys::vuk` type is now `Option<[u8; 16]>`** (was
|
||||
`[u8; 16]`). Same rationale.
|
||||
|
||||
## 0.25.14 (2026-05-21)
|
||||
|
||||
### Changed
|
||||
|
||||
- `Drive::is_libredrive_active()` renamed to `Drive::is_raw_read_active()`.
|
||||
Same semantics; old name removed. Mirrored on the internal
|
||||
`PlatformDriver::is_libredrive_active()` trait method (now
|
||||
`is_raw_read_active()`).
|
||||
- `Error::AacsLibredriveUnsupported` renamed to
|
||||
`Error::AacsRawReadUnsupported`; the underlying numeric code (E7016)
|
||||
is unchanged. The `E_AACS_LIBREDRIVE_UNSUPPORTED` constant is
|
||||
renamed to `E_AACS_RAW_READ_UNSUPPORTED`.
|
||||
|
||||
No behavioural change — purely a rename pass.
|
||||
|
||||
## 0.25.13 (2026-05-21)
|
||||
|
||||
### Added
|
||||
|
||||
- **`DrmScheme` top-level dispatcher.** New `drm` module with a
|
||||
`DrmScheme` enum (`Css`, `Aacs10`, `Aacs20`, `Aacs21`) and a
|
||||
`detect` + `load` pair that uniformly handles all four content
|
||||
protection schemes. Replaces the inlined dispatch in
|
||||
`disc::encrypt::resolve_encryption` and the scattered CSS routing
|
||||
in `disc::mod`. Both CSS call sites now route through the same
|
||||
entry point.
|
||||
- **AACS 2.1 Media Key Variant framework.** New `aacs::variants`
|
||||
module implementing the Media Key Variant derivation chain
|
||||
(`Kp + C → Kmp → ⊕KCD → Kpnew → Km → VUK`), Variant-scheme MKB
|
||||
record parsing (record types `0x82` / `0x83`), bit-0x02 SoftKCD
|
||||
and bit-0x04 online-challenge detection with dedicated error
|
||||
variants. Wired into `DrmScheme::Aacs21` but the dispatcher arm
|
||||
is commented out pending validation against a Variant-scheme
|
||||
disc. Per-manufacturer Key Correction Data must be supplied by
|
||||
the integrator; `KEY_CORRECTION_DATA_PLACEHOLDER` is the empty
|
||||
placeholder slot.
|
||||
- **`AacsVersion` enum.** Replaces the `aacs2: bool` field on
|
||||
`ContentCertificate`, `UnitKeyFile`, and `ResolvedKeys`.
|
||||
`parse_unit_key_ro` and `parse_content_cert` now take/emit the
|
||||
enum. `resolve_keys` is split into `resolve_keys_v1`,
|
||||
`resolve_keys_v2`, and `resolve_keys_v21` (the last not reachable
|
||||
from the dispatcher today).
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Libredrive raw-read VID shortcut deleted.** v0.25.11 introduced a
|
||||
`do_handshake` branch that, on libredrive-active drives, skipped the
|
||||
AACS cert handshake and issued `READ_DISC_STRUCTURE` format 0x80
|
||||
with AGID=0 directly. The hypothesis was that unlocked
|
||||
drives would serve VID without auth. Empirical test (BU40N
|
||||
+ UHD disc, 2026-05-21) showed the drive returns
|
||||
`0x05 / 0x6F / 0x02` (`ILLEGAL_REQUEST / Copy protection key
|
||||
exchange failure: KEY NOT ESTABLISHED`) to that CDB regardless of
|
||||
drive-unlock state. The AACS spec requires a successful
|
||||
`REPORT_KEY` / `SEND_KEY` exchange to establish an AGID before
|
||||
format 0x80 returns VID; that requirement is enforced by the drive
|
||||
itself and isn't bypassed by libredrive firmware. The shortcut
|
||||
fired for every libredrive-active drive, so v0.25.11 / v0.25.12
|
||||
MOVIE scans were stuck at E7017 instead of progressing to the
|
||||
real wall (no DK walks MKB v77).
|
||||
- `Disc::do_handshake` now always routes through `do_handshake_cert`.
|
||||
`Drive::is_libredrive_active()` and the Mt1959 MMkv+LbDr marker
|
||||
detection are kept as informational signals (logged in the
|
||||
`handshake_entry` warn line) but no longer steer the auth path.
|
||||
- `read_volume_id_libredrive` deleted (~50 LOC).
|
||||
|
||||
The corollary: AACS resolution on HRL-burned drives + UHD discs now
|
||||
fails honestly. Either cert auth succeeds (drive unlock may or
|
||||
may not bypass the HRL — that's the new empirical question) and we
|
||||
hit the actual DK wall (E7018 "No DK that walks this MKB" for v77+
|
||||
UHD without a v77+ DK in keydb), or cert auth fails and we surface
|
||||
E7015. Both are real verdicts; E7017's previous spurious dispatch
|
||||
is gone.
|
||||
|
||||
## 0.25.12 (2026-05-21)
|
||||
|
||||
No libfreemkv source changes — unified sync bump for autorip's
|
||||
`aacs_failure_message` two-line wording rewrite. See the autorip
|
||||
v0.25.12 release for details.
|
||||
|
||||
## 0.25.11 (2026-05-21)
|
||||
|
||||
### Added
|
||||
|
||||
- **Libredrive raw-read VID path.** When the Mt1959 unlock response
|
||||
confirms both the active-mode (`MMkv`) and mode-ID (`LbDr`) markers,
|
||||
`Drive::is_libredrive_active()` returns true and `do_handshake`
|
||||
skips the AACS cert dance entirely — VID is retrieved via
|
||||
`READ_DISC_STRUCTURE` format 0x80 with AGID=0, and bus encryption
|
||||
is already off. This is what unblocks UHD ripping on drives whose
|
||||
leaked host cert is on the AACS HRL.
|
||||
- New `Error` variants for finer-grained AACS failure reporting:
|
||||
`AacsHostCertRejected` (E7015), `AacsLibredriveUnsupported`
|
||||
(E7016), `AacsVidUnavailable` (E7017), `AacsMkUnavailable`
|
||||
(E7018), `AacsVukNotInKeydb` (E7019). Lets CLIs/UIs render which
|
||||
piece of the AACS chain failed instead of always saying "no keys."
|
||||
|
||||
### Fixed
|
||||
|
||||
- `validate_processing_key` now matches libaacs `_validate_pk`
|
||||
exactly: XORs `uv` into `mk[12..16]` (was omitted), AES-decrypts
|
||||
`mk_dv` and checks the `01 23 45 67 89 AB CD EF` magic (was
|
||||
AES-encrypt + 12-zero check). Pre-fix, every non-zero-uv
|
||||
processing key was rejected — i.e. essentially every real disc.
|
||||
- `mkb_find_cvalues` now prefers record type `0x07` (AACS 2.x) and
|
||||
falls back to `0x05` (AACS 1.0), so the walker handles both
|
||||
generations without an out-of-band version flag.
|
||||
- `resolve_keys` short-circuits paths 2/3/4 when VID is the zero
|
||||
sentinel — saves cycles and emits an honest "VID unavailable" log
|
||||
instead of the misleading "all paths failed."
|
||||
- AES-CMAC VID verification gains NIST SP 800-38B KAT + round-trip
|
||||
+ mutation + all-zero-rejection tests.
|
||||
|
||||
### Removed
|
||||
|
||||
- **Built-in AACS keys** (added in 0.25.9). `src/aacs/builtin_keys.rs`
|
||||
deleted; `KeyDb::with_builtins`, `KeyDb::load_or_builtins`,
|
||||
`KeyDb::merge_from`, and internal dedup helpers gone. The
|
||||
compiled-in shortcut was a slim convenience that didn't move the
|
||||
hard problem (no v77+ DKs) and added a maintenance surface.
|
||||
- **Operator plugin slot** at `~/.config/freemkv/local_keys.cfg`
|
||||
(added in 0.25.9). `local_plugin_path` and `KeyDb::merge_local_plugin`
|
||||
removed. Single source: the main `keydb.cfg`.
|
||||
- `KeyDb::load` reverts to the pre-0.25.9 form — read the file or
|
||||
return I/O error; no layering, no fallback.
|
||||
- `Disc::scan` reverts to surfacing `KeydbLoad { path: "<no keydb
|
||||
in search paths>" }` for an encrypted disc with no keydb — same
|
||||
sentinel autorip's message switch already handles.
|
||||
|
||||
CSS player keys in `src/css/auth.rs` remain compiled in; they're
|
||||
1999-era public inputs separate from the AACS pipeline and were
|
||||
never part of the 0.25.9 additions.
|
||||
|
||||
## 0.25.9 (2026-05-20)
|
||||
|
||||
### Fixed
|
||||
|
||||
- **MKB record-type identification.** `mkb_find_mk_dv` and `mkb_version`
|
||||
had their record-type constants swapped. The Verify Media Key record
|
||||
is Type 0x81 (AACS 1.0) or Type 0x86 (AACS 2.0/2.1), not 0x10. The
|
||||
Type-and-Version record is Type 0x10, not 0x81. PK and DK derivation
|
||||
paths therefore failed silently on every disc, masking how often the
|
||||
fallback paths could have succeeded. Fixed; mk_dv extracted at
|
||||
offset 4 of the verify record, MKB version read at offset 8 of the
|
||||
Type 0x10 record body. Tests added for both forms.
|
||||
|
||||
### Added
|
||||
|
||||
- **Built-in AACS keys.** Four device keys (covering MKB v01-v82+) and
|
||||
three processing keys (covering v63-v68) are now compiled into
|
||||
`libfreemkv` directly. DVDs and Blu-rays (AACS 1.0) decrypt with
|
||||
zero external files. Combined with the existing 31 CSS player keys
|
||||
in `css/auth.rs`, the library is self-sufficient for all DVD and
|
||||
AACS 1.0 content.
|
||||
- **Operator plugin slot.** `~/.config/freemkv/local_keys.cfg` is
|
||||
loaded automatically (same format as `keydb.cfg`) and layered
|
||||
additively on top of the built-ins and the main keydb.cfg. For
|
||||
operators who derive their own keys, this is the additive surface
|
||||
to drop them into — main keydb.cfg from upstream auto-update stays
|
||||
separate and overwritable.
|
||||
- `KeyDb::with_builtins()`, `KeyDb::load_or_builtins()`,
|
||||
`KeyDb::empty()` constructors.
|
||||
|
||||
### Changed
|
||||
|
||||
- `Disc::scan` no longer errors when keydb.cfg is absent. With
|
||||
built-ins covering DVD/BD, missing keydb is treated as "no UHD
|
||||
keys available" — the AACS 2.0 / UHD code path surfaces a
|
||||
specific error only when the disc actually needs keys that
|
||||
aren't in built-ins, main keydb, or local plugin.
|
||||
|
||||
## 0.25.8 (2026-05-20)
|
||||
|
||||
### Changed
|
||||
|
||||
- Synchronized to autorip 0.25.8. No libfreemkv code changes —
|
||||
unified versioning bump only. autorip 0.25.7 image was DOA
|
||||
(missing +x on the binary); see autorip changelog.
|
||||
|
||||
## 0.25.7 (2026-05-20)
|
||||
|
||||
### Fixed — BU40N firmware wedge on KEYDB-miss discs
|
||||
|
||||
`disc::encrypt::do_handshake` used to fire up to 16 AACS authenticate
|
||||
attempts back-to-back with no pause between them. Each attempt is
|
||||
5-10 SCSI REPORT_KEY/SEND_KEY exchanges. On a disc whose host cert
|
||||
isn't in our KEYDB (or one the drive rejects), that's 80-160 SCSI
|
||||
commands hammered at the drive in a few hundred milliseconds — and
|
||||
the BU40N (plus most consumer optical drives) responds by entering
|
||||
a fast-fail firmware wedge state where every subsequent CDB returns
|
||||
`ILLEGAL_REQUEST/INVALID_FIELD_IN_CDB` (sense 0x05/0x24) until the
|
||||
drive is physically power-cycled.
|
||||
|
||||
Live wedge event 2026-05-20 during a UHD disc scan
|
||||
(KEYDB miss) confirmed the diagnosis and motivated this fix.
|
||||
|
||||
Defence-in-depth:
|
||||
- `MAX_CERT_ATTEMPTS` capped at 3 (was 16). If three different
|
||||
host certs all fail, more won't help — the drive doesn't have a
|
||||
match.
|
||||
- 1-second sleep between attempts. Gives the drive's firmware time
|
||||
to recover internal state between auth challenges.
|
||||
- Bail immediately on any sense code with sense_key = 0x05
|
||||
(`ILLEGAL_REQUEST`). The drive isn't merely rejecting our cert
|
||||
— it's saying "I won't talk to you anymore" — so trying more
|
||||
certs would deepen the wedge.
|
||||
|
||||
## 0.25.6 (2026-05-20)
|
||||
|
||||
### Changed
|
||||
|
||||
- Synchronized to autorip 0.25.6 release. No behavioural changes.
|
||||
|
||||
## 0.25.5 (2026-05-20)
|
||||
|
||||
### Changed
|
||||
|
||||
- Synchronized to autorip 0.25.5 release. No behavioural changes.
|
||||
|
||||
## 0.25.4 (2026-05-20)
|
||||
|
||||
### Changed
|
||||
|
||||
- Synchronized to autorip 0.25.4 release. No behavioural changes
|
||||
in the library — the v0.25.4 release ships the autorip-side
|
||||
in-container NFS mount option.
|
||||
|
||||
## 0.25.3 (2026-05-19)
|
||||
|
||||
### Changed
|
||||
|
||||
- Synchronized to autorip 0.25.3 release. No behavioural changes
|
||||
in the library — the v0.25.3 release ships the autorip-side
|
||||
parallel mux pipeline.
|
||||
|
||||
## 0.25.2 (2026-05-19)
|
||||
|
||||
### Fixed
|
||||
|
||||
- **DTS-HD codec ID** — `MkvTrack::audio` now emits `A_DTS/MA` for
|
||||
DTS-HD MA and `A_DTS/HR` for DTS-HD HR instead of mislabelling
|
||||
both as plain `A_DTS`. Strict players (Plex transcoder, some
|
||||
hardware decoders, AV receivers) reject lossless DTS-HD MA
|
||||
payload when the track advertises `A_DTS` because the ID
|
||||
implies the 1.5 Mbps core-only bitstream.
|
||||
- **PGS subtitle BlockDuration** — the PGS parser is now stateful:
|
||||
it pairs each display PCS with the following empty/clear PCS to
|
||||
compute a duration, and the MKV muxer emits a `BlockGroup` +
|
||||
`BlockDuration` for subtitles that carry one. Without this the
|
||||
last bitmap lingered on screen until the next display set
|
||||
replaced it (or until end of file).
|
||||
|
||||
### Changed
|
||||
|
||||
- `codec::Frame` gains `duration_ns: Option<u64>` (set by parsers
|
||||
that can compute one; currently only PGS).
|
||||
- `pes::PesFrame` gains `duration_ns: Option<u64>` (in-memory only;
|
||||
not part of the on-wire serialization).
|
||||
- `MkvMuxer::write_frame` now takes a final `duration_ns: Option<u64>`
|
||||
parameter. When `Some`, the frame is emitted as a `BlockGroup`
|
||||
with `BlockDuration` instead of a `SimpleBlock`.
|
||||
|
||||
## 0.25.1 (2026-05-19)
|
||||
|
||||
### New — autorip event_fn plumbing
|
||||
|
||||
- `PrefetchedSectorSource::new_with_events(reader, extents, batch,
|
||||
halt, event_fn)` — same producer-thread pipeline as `new()`, plus
|
||||
an optional callback fired from the producer with `BytesRead`
|
||||
events after every successful batch. Lets the autorip multipass
|
||||
+ resume mux paths drive their progress UI from the highway
|
||||
without polling the consumer side.
|
||||
- `build_iso_pipeline` gains an `event_fn` parameter so callers can
|
||||
wire the same callback through one ctor.
|
||||
|
||||
### Changed
|
||||
|
||||
- `pes::Stream` trait: new default `errors() -> u64` method
|
||||
(default returns 0). Lets `Box<dyn Stream>` callers query the
|
||||
skip-on-error counter without downcasting. `DiscStream` overrides
|
||||
to surface its existing `errors` field.
|
||||
|
||||
### Removed (breaking)
|
||||
|
||||
- `DiscStream::new_pipeline` and `DiscStream::read_pipeline`
|
||||
deleted. All file-backed mux now uses `build_iso_pipeline` →
|
||||
`PipelinedPesStream`. `DiscStream` is now the single-threaded
|
||||
inline reader used by autorip's live-drive single-pass path only.
|
||||
- `DiscStream::demux_thread` and `demux_rx` fields removed.
|
||||
- `M2tsStream::open` deleted; `Mode::Read` variant removed.
|
||||
`m2ts://` URLs go through the internal `build_m2ts_pipeline`
|
||||
helper in `mux/resolve.rs` → `PipelinedPesStream`. `M2tsStream`
|
||||
is a write-only sink now.
|
||||
|
||||
## 0.25.0 (2026-05-19)
|
||||
|
||||
### New — the freemkv mux throughput "highway"
|
||||
|
||||
Three-stage pipelined PES read path. Read+decrypt runs on a
|
||||
producer thread, M2TS demux runs on a second thread, codec parse
|
||||
runs on the caller's thread. Communication between stages is via
|
||||
bounded `crossbeam_channel` with a recycled buffer pool — no
|
||||
allocations or memcpys in the steady-state hot loop.
|
||||
|
||||
**Throughput on the test bed (a UHD disc, 62 GiB ISO →
|
||||
`null://`, single-thread caller):**
|
||||
|
||||
| | MB/s |
|
||||
| ------------------------------------- | -----: |
|
||||
| 0.23.2 baseline | 60 |
|
||||
| + memchr SIMD HEVC start-code scan | 69 |
|
||||
| + ts.feed no-copy boundary | 72 |
|
||||
| + `PrefetchedSectorSource` (producer) | 124 |
|
||||
| + `DemuxThread` (3-stage pipeline) | 135 |
|
||||
| + zero-copy recycled buffer pool | 148 |
|
||||
| + 16 KiB initial PesAssembler buffer | 162 |
|
||||
| + mimalloc allocator in freemkv CLI | 200+ |
|
||||
| **+ warm cache** | **660** |
|
||||
|
||||
The new public API:
|
||||
|
||||
- `libfreemkv::PrefetchedSectorSource` — wraps any `SectorSource`,
|
||||
spawns a producer thread, exposes recycled-buffer channels.
|
||||
- `libfreemkv::io::byte_prefetcher::BytePrefetcher` —
|
||||
`std::io::Read` analogue for byte-stream sources (m2ts files,
|
||||
sockets, stdin).
|
||||
- `libfreemkv::mux::demux_thread::DemuxThread` — the M2TS demux
|
||||
worker; `spawn_zero_copy` takes either prefetcher's channels.
|
||||
- `libfreemkv::PipelinedPesStream` — the read-side `Stream` impl
|
||||
that runs codec parse on the caller thread.
|
||||
- `libfreemkv::build_iso_pipeline(reader, title, keys, batch,
|
||||
format, halt)` — the canonical ctor that wires all three stages
|
||||
for an ISO file source.
|
||||
|
||||
### Changed (breaking)
|
||||
|
||||
- `IsoSectorReader` (the naive duplicate of `FileSectorSource`)
|
||||
deleted. `FileSectorSource` is the sole file-backed sector
|
||||
source; it carries the SEQUENTIAL fadvise hint, the periodic
|
||||
DONTNEED page-cache eviction, and (new) the per-read
|
||||
`readahead()` async-prefetch syscall.
|
||||
- `mux::input("iso://...", &opts)` now returns a
|
||||
`PipelinedPesStream`. Function signature
|
||||
`input(&str, &InputOptions) -> io::Result<Box<dyn Stream>>` is
|
||||
unchanged; callers that treated the return value as
|
||||
`Box<dyn Stream>` keep working.
|
||||
|
||||
### Other
|
||||
|
||||
- `FileSectorSource` exposes per-OS `prefetch()` hooks
|
||||
(`readahead(2)` on Linux, `fcntl(F_RDADVISE)` on macOS, no-op on
|
||||
Windows + other).
|
||||
- AACS decrypt thread-pool: env var renamed from
|
||||
`FREEMKV_DECRYPT_THREADS` to `FREEMKV_THREADS`; default raised
|
||||
to `cores.clamp(1, 64)`.
|
||||
- `PesAssembler` initial buffer capacity 256 KiB → 16 KiB (avoids
|
||||
the 64-page first-touch fault tax on every PES boundary).
|
||||
- HEVC / H.264 `find_start_code` swapped onto
|
||||
`memchr::memmem::find` (SIMD).
|
||||
|
||||
## 0.18.4 (2026-05-09)
|
||||
|
||||
### Build / CI hardening — no library code changes
|
||||
|
||||
- All `cargo` invocations in `.github/workflows/*.yml` now use
|
||||
`--locked`. Previously `cargo build` / `cargo test` could silently
|
||||
re-resolve `Cargo.lock` if a lock-pinned version wasn't yet on
|
||||
crates.io — masking dependency races between same-tag releases of
|
||||
sibling crates. The 0.18.3 release hit this: autorip's docker image
|
||||
built `libfreemkv v0.18.2` because libfreemkv 0.18.3 hadn't
|
||||
published yet at the time autorip's CI ran, and cargo silently fell
|
||||
back to the previous version. With `--locked`, that race now hard-
|
||||
fails the build with a clear "package X is not in registry" error
|
||||
and we retrigger after the upstream lands.
|
||||
|
||||
- `cargo publish --locked` in the release workflow guards against
|
||||
publishing a lockfile-mismatched release.
|
||||
|
||||
## 0.18.3 (2026-05-09)
|
||||
|
||||
### Behaviour change
|
||||
|
||||
- **`Disc::titles[0]` is now the canonical main feature on branching
|
||||
discs, not the longest playlist.** Previously titles were sorted
|
||||
purely by `duration_secs` descending, which puts a "play-all"
|
||||
virtual playlist (alternate angles / seamless branching) at index 0
|
||||
on UHDs that ship one — its inflated duration overshoots the real
|
||||
movie. Concrete example: *The Amateur (2025)* 4K UHD on a 58.5 GB
|
||||
disc has Title 1 = `00020.mpls` 4h13m / 92.4 GB / 253 clips
|
||||
(impossible — the size exceeds the disc capacity, proving it's a
|
||||
virtual composite) and Title 2 = `00800.mpls` 2h02m / 57.2 GB /
|
||||
1 clip (the actual film, matching TMDB).
|
||||
|
||||
New sort priority:
|
||||
1. Real titles (`size_bytes <= capacity_bytes`) before virtual
|
||||
composites.
|
||||
2. Among real titles, fewer clips first (1-clip wins).
|
||||
3. Tiebreak on longer duration first.
|
||||
|
||||
**Migration:** consumers calling `disc.titles.first()` /
|
||||
`disc.titles[0]` automatically get the corrected title — no code
|
||||
changes needed. CLI users invoking `freemkv -t 1 disc:// …` now hit
|
||||
the actual main feature on branching discs (this was the user-
|
||||
visible bug). On non-branching discs the order is unchanged.
|
||||
|
||||
The comparator is exposed as
|
||||
[`Disc::canonical_title_order`](#) for callers that need to
|
||||
re-sort a custom title set with the same logic.
|
||||
|
||||
Regression tests: `disc::tests::canonical_order_*` (three cases:
|
||||
branching-UHD, normal disc, clip-count tiebreak).
|
||||
|
||||
## 0.18.2 (2026-05-09)
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- **Nav-file scramble during AACS rip** (`decrypt::decrypt_sectors`). 0.18's
|
||||
`DecryptingSectorSource` decorator broadened the call surface of
|
||||
`decrypt_sectors` to every sector flowing through sweep, including UDF
|
||||
navigation files (MPLS playlists, CLPI clip-info). The byte-0 heuristic
|
||||
in `aacs::is_unit_encrypted` correctly fires on m2ts source-packet copy
|
||||
markers but false-positives on any binary file whose first byte happens
|
||||
to have the top 2 bits set — most notably MPLS files (start with 'M' =
|
||||
0x4D) and CLPI files (start with 'H' = 0x48). `decrypt_unit_full`
|
||||
already self-checks the result via TS-sync verification and returns
|
||||
`false` on a misfire, but the chunk had been mutated by then;
|
||||
`decrypt_sectors` discarded the return value, leaving scrambled bytes
|
||||
in the ISO. Fix: snapshot the chunk before decryption and restore on
|
||||
verification failure (same pattern `decrypt_unit_try_keys` already used
|
||||
for multi-key discs). Symptom: `freemkv info iso://UHD.iso` and
|
||||
`iso:// → mkv://` returned E6009 NoStreams on freshly-ripped UHD ISOs;
|
||||
affected only the iso-source path, not the disc-source path or the
|
||||
m2ts video payload itself. Regression test:
|
||||
`decrypt::tests::nav_file_unit_survives_decrypt_attempt`.
|
||||
|
||||
- **Sweep progress display can regress to zero** (`Disc::sweep`). The
|
||||
consumer-side `bytes_good` snapshot lags producer-side `bytes_done`
|
||||
whenever the consumer is behind on draining the work channel. Until
|
||||
the first snapshot arrived, the placeholder branch reported
|
||||
`bytes_done` correctly; once a stale snapshot landed, the report
|
||||
switched to `snap.bytes_good` and could regress below `bytes_done`.
|
||||
Fix: `bytes_good_total = max(snap.bytes_good, bytes_done)`, so the
|
||||
user-visible counter never moves backward.
|
||||
|
||||
## 0.18.1 (2026-05-09)
|
||||
|
||||
### I/O stack redesign — primitives over orchestration
|
||||
|
||||
0.18 reshapes the read/write surface from "library does the multipass dance"
|
||||
to "library hands the caller flat verbs and a few composable primitives." All
|
||||
in-tree consumers (autorip, the `freemkv` CLI) drove their own multipass
|
||||
loops in 0.18 rounds 1-3; 0.18.1 lands the cleanup with the deprecated names
|
||||
still alive for one minor-version window.
|
||||
|
||||
The 0.18 design notes are kept privately — this entry sticks to
|
||||
what changed at the public surface.
|
||||
|
||||
#### Flat verbs
|
||||
|
||||
`Disc::sweep` is the forward Pass 1. `Disc::patch` is one retry pass over the
|
||||
mapfile. Neither knows about pass index, retry budget, or accept-loss policy
|
||||
— the caller invokes them in whatever sequence its use case dictates. The
|
||||
old multipass-aware `Disc::copy` dispatcher is deprecated and slated for
|
||||
deletion in 0.18.2; no in-tree caller still uses it.
|
||||
|
||||
#### Trait splits — direction-typed at compile time
|
||||
|
||||
- `pes::Stream` (combined read+write) is split into `FrameSource` and
|
||||
`FrameSink`. Calling `read()` on a write-only sink is now a compile error,
|
||||
not the runtime `E9001` (`StreamWriteOnly`) it used to be.
|
||||
- `SectorReader` is split into `SectorSource` (read) and `SectorSink`
|
||||
(write). `Drive` impls `SectorSource` only; `FileSectorSource` /
|
||||
`FileSectorSink` replace `FileSectorReader` for ISO-backed I/O.
|
||||
- A blanket impl bridges legacy `SectorReader` callers onto the new
|
||||
`SectorSource` so existing code keeps compiling through the deprecation
|
||||
window.
|
||||
|
||||
#### New primitives
|
||||
|
||||
- `Halt` — one cancellation token (cloneable, `Arc<AtomicBool>` under the
|
||||
hood) replaces the three near-duplicate halt flags scattered through the
|
||||
workspace. Threaded through every long-running loop.
|
||||
- `Pipeline<I, R>` + `Sink<I>` — generic producer/consumer primitive in
|
||||
`crate::io`. Replaces the bespoke `disc/sweep_pipeline.rs` and now also
|
||||
drives `Disc::patch` and the autorip mux loop. `DEFAULT_PIPELINE_DEPTH`
|
||||
for streaming reads (`4`); `WRITE_THROUGH_DEPTH` for write-through patch
|
||||
semantics (`1`).
|
||||
- `WritebackFile` — was `crate::io::Writer`. The renamed type makes its job
|
||||
explicit: a `File` wrapper that runs continuous `sync_file_range` +
|
||||
`posix_fadvise(DONTNEED)` to keep the kernel dirty-page cache bounded on
|
||||
long sequential writes.
|
||||
- `DecryptingSectorSource<S>` — a single decorator wrapping any
|
||||
`SectorSource` to yield plaintext sectors. One audit surface for AACS /
|
||||
CSS / passthrough; the previous two-site decrypt (sweep producer +
|
||||
`DiscStream` demux) is gone.
|
||||
|
||||
#### Throughput
|
||||
|
||||
The round-2 producer/consumer split is now applied uniformly to sweep,
|
||||
patch, and mux. Mux on NFS-staged UHD measured ~16 MB/s sustained on the
|
||||
test bed (was ~12 MB/s pre-round-2).
|
||||
|
||||
#### Module reorg
|
||||
|
||||
- `sector/` and `io/` are now module directories.
|
||||
- `disc/sweep.rs`, `disc/patch.rs`, and `disc/mapfile.rs` split out of the
|
||||
monolithic `disc/mod.rs`.
|
||||
|
||||
#### Renames (no behavior change)
|
||||
|
||||
- `crate::io::Writer` → `crate::io::WritebackFile`.
|
||||
- `Apply` → `Flow` (`Sink::apply` return value).
|
||||
- `DEFAULT_DEPTH` → `DEFAULT_PIPELINE_DEPTH`.
|
||||
- `PatchOpts` → `PatchOptions`.
|
||||
|
||||
#### Deprecated (alive in 0.18.1, deletion target 0.18.2)
|
||||
|
||||
`Disc::copy`, `pes::Stream`, `SectorReader`, `FileSectorReader`,
|
||||
`CopyOptions`, `CopyResult`, `DiscStream::set_halt`. Each compiles with a
|
||||
deprecation warning; all in-tree call sites have migrated.
|
||||
|
||||
## 0.17.13 (2026-05-09)
|
||||
|
||||
### Use `crate::io::Writer` uniformly for all binary file output
|
||||
|
||||
0.17.10 introduced the bounded-cache writeback wrapper (`sync_file_range`
|
||||
+ `posix_fadvise(DONTNEED)` per chunk) and 0.17.11 wired it into
|
||||
`Disc::sweep`. The other two big-write paths in the crate were still
|
||||
opening raw `std::fs::File` and would have hit the same dirty-page
|
||||
burst pathology against slow / network-attached staging.
|
||||
|
||||
This release threads `crate::io::Writer` through the remaining sites:
|
||||
|
||||
- **`Disc::patch`** (`disc/mod.rs:1981`): the ISO file reopened for
|
||||
Pass-N recovery now wraps in `Writer` before any seek / write.
|
||||
Recovery writes are sparse, but the wrapper costs nothing when
|
||||
there's no chunk crossing — and on heavily-damaged discs it
|
||||
matters as patch accumulates GB of recovered data.
|
||||
- **MKV mux output** (`mux/resolve.rs:243`): `BufWriter::with_capacity`
|
||||
now wraps `Writer::new(file)` instead of a raw `File`. UHD MKVs
|
||||
routinely exceed 70 GB of sequential writes; pre-0.17.13 those
|
||||
bursts went straight to the kernel writeback queue.
|
||||
- **M2TS mux output** (`mux/resolve.rs:251`): same change for
|
||||
parity with the MKV path — anyone using `m2ts://` URLs gets it
|
||||
too.
|
||||
|
||||
No new public surface. `Writer::sync_all()` is called from `Disc::patch`
|
||||
and `sweep_pipeline`'s consumer at end of pass; mux exits via Drop on
|
||||
the wrapping `BufWriter`, which propagates flush down to `Writer`'s
|
||||
final `note_progress` (kernel finishes the in-flight chunk on file
|
||||
close — there's no explicit `sync_all` for mux today, same behaviour
|
||||
as before).
|
||||
|
||||
## 0.17.12 (2026-05-09)
|
||||
|
||||
### Mapfile time-batched persistence — unblock NFS staging
|
||||
|
||||
Pre-0.17.12 every `Mapfile::record()` wrote the entire mapfile to disk
|
||||
via tempfile-create + buffered-write + atomic-rename. On local LVM
|
||||
that's effectively free (page cache + microsecond-scale renames). On
|
||||
NFS (autorip's intended staging path for centralised media) each
|
||||
record() became three RPCs through the unraid user share's shfs-fuse
|
||||
layer — measured end-to-end at multiple ms each. With ~170 record()
|
||||
calls per second of sustained sweep, the mapfile path alone burned
|
||||
multiple seconds of wall time per real-world second of work, dragging
|
||||
the rip from ~11 MB/s on local to ~1.5 MB/s on NFS.
|
||||
|
||||
Fix: time-batch the persistence inside `Mapfile`.
|
||||
|
||||
- `record()` always updates in-memory state and stats (so callers'
|
||||
`stats()` reads stay coherent in the same process).
|
||||
- The `write_to_disk()` rename only fires when ≥ `FLUSH_INTERVAL`
|
||||
(1 s) has elapsed since the last persist.
|
||||
- New `flush()` method forces an out-of-band persist; called by
|
||||
`sweep_pipeline`'s consumer at end-of-sweep and by `Disc::patch` at
|
||||
end-of-patch, after the file's `sync_all()`.
|
||||
- `Drop` impl best-effort flushes so an early-return / unwind doesn't
|
||||
silently lose pending state.
|
||||
|
||||
Crash-safety changes from "lose at most one block" to "lose at most
|
||||
1 s of recorded progress" — the ISO file's payload bytes are
|
||||
unaffected; only the mapfile's authority over which sectors are
|
||||
already-good is at risk, and a resume re-reads anything Pass 1 had
|
||||
already covered. Acceptable for a 7× throughput recovery on the
|
||||
target deployment.
|
||||
|
||||
The internal `round_trip_load` test now calls `flush()` before
|
||||
`Mapfile::load` to read back what the in-memory state asserts.
|
||||
External patch / copy tests are unaffected: `patch` and
|
||||
`sweep_pipeline` flush at completion before returning, so any
|
||||
`Mapfile::load` at the call-site sees fully persisted state.
|
||||
|
||||
## 0.17.11 (2026-05-09)
|
||||
|
||||
### Sweep producer/consumer split — overlap drive read with file write
|
||||
|
||||
Pre-0.17.11 the sweep loop ran strictly serialised: SCSI read → decrypt
|
||||
→ seek + write → mapfile.record → next read. The drive idled for the
|
||||
post-read work; throughput capped at the **sum** of both costs. On a
|
||||
healthy disc that's ~7-12 ms read + ~5-15 ms write/record per 64 KB
|
||||
batch — limiting sustained throughput to ~10-12 MB/s on the test bed
|
||||
(BU40N + UHD inner zone), well below the drive's ceiling of ~14-16 MB/s.
|
||||
|
||||
This release decouples them with a producer / consumer split:
|
||||
|
||||
- **Producer** (caller's thread): owns the `SectorReader`, the entire
|
||||
`read_error` state machine (Retry, Bisect, SkipBlock, JumpAhead,
|
||||
AbortPass), `set_speed` damage-zone transitions, halt check, and
|
||||
decrypt. Hands plaintext bytes to the consumer.
|
||||
- **Consumer** (one spawned thread): owns the `crate::io::Writer` and
|
||||
the `Mapfile`. Receives `WorkItem` messages and applies the file
|
||||
write + mapfile record per item.
|
||||
- **Channel**: bounded `mpsc::sync_channel(4)` — natural back-pressure
|
||||
via blocking `send` when consumer falls behind.
|
||||
|
||||
While the consumer is writing batch N to disk and updating the mapfile,
|
||||
the producer is already reading batch N+1 from the drive. Steady-state
|
||||
throughput is now bound by the slower of the two pipelines (the drive,
|
||||
on a healthy disc) instead of their sum.
|
||||
|
||||
Side effects of the refactor:
|
||||
|
||||
- **Bisect path now decrypts.** Pre-0.17.11 the bisect inner loop wrote
|
||||
raw cyphertext when `decrypt=true` and a single sector was recovered
|
||||
via single-sector retry — a quiet correctness bug exercised only by
|
||||
the (rare) batch-fail-then-bisect-succeed path on encrypted discs.
|
||||
The new producer-side decrypt covers both the main success path and
|
||||
the bisect inner success path.
|
||||
- All `read_error::ReadCtx` state stays single-threaded on the producer
|
||||
(damage window, jump multiplier, consecutive-good count, etc.). No
|
||||
locking added.
|
||||
- Mapfile remains single-writer (consumer-only). No locking.
|
||||
- Halt-flag responsiveness unchanged: producer breaks the loop on
|
||||
signal, sends `Finish`, consumer drains its ≤4 in-flight items and
|
||||
exits within ~1 batch (~12 ms typical).
|
||||
- BU40N + Initio bridge wedge concern unchanged: still one SCSI command
|
||||
in flight, error-path timing identical, no new retry logic.
|
||||
|
||||
New module: `src/disc/sweep_pipeline.rs` (`WorkItem`, `ProgressSnapshot`,
|
||||
`ConsumerInputs`, `spawn_consumer`, `consumer_loop`, send/recv helpers).
|
||||
Public API surface unchanged — `Disc::copy` / `CopyOptions` /
|
||||
`CopyResult` look identical to callers.
|
||||
|
||||
Patch (Pass N) is **not** affected by this release. Patch is bound by
|
||||
drive recovery time (60 s timeouts on bad sectors), not the read↔write
|
||||
serialisation; a similar split there would yield negligible benefit.
|
||||
|
||||
## 0.17.10 (2026-05-09)
|
||||
|
||||
### Bounded-cache writeback for big sequential writes
|
||||
|
||||
Pass 1 sweep speed on a healthy disc previously dipped from ~15 MB/s
|
||||
to ~1 MB/s every ~30 s on a host with default Linux dirty-page
|
||||
settings. Cause: the kernel's `vm.dirty_ratio` (~20 % of RAM) lets
|
||||
hundreds of MB of dirty pages accumulate, then bursts a flush at
|
||||
99 % disk utilisation that blocks app writes for ~1 s. Confirmed
|
||||
empirically on the BU40N test bed: dirty pages grew 112 → 563 MB
|
||||
between bursts; lowering `vm.dirty_bytes` to 64 MB at the host
|
||||
sysctl level eliminated the dips.
|
||||
|
||||
This release ships the equivalent inside libfreemkv so users don't
|
||||
need to tune the host kernel:
|
||||
|
||||
- New `crate::io::Writer` — drop-in `File` wrapper implementing
|
||||
`Write` + `Seek`. Wraps a per-platform `WritebackPipeline` that on
|
||||
Linux schedules `sync_file_range(WRITE)` + lagging
|
||||
`sync_file_range(WAIT_AFTER)` + `posix_fadvise(DONTNEED)` calls in
|
||||
32 MB chunks, keeping dirty cache bounded at ~64 MB. macOS and
|
||||
Windows ship a no-op stub — their default cache policies don't
|
||||
exhibit the same pathology for our access pattern.
|
||||
- Disc::sweep wraps its output `File` in `Writer`. No changes to the
|
||||
loop body — `Writer` forwards `seek`/`write_all` to `File` and
|
||||
drives the pipeline transparently.
|
||||
- Module is purpose-built for any large sequential output (sweep,
|
||||
patch, future mux) — they can adopt `crate::io::Writer` with a
|
||||
one-line wrapper and inherit the same behaviour.
|
||||
|
||||
## 0.17.7 (2026-05-08)
|
||||
|
||||
### Sync release — no functional libfreemkv changes
|
||||
|
||||
Version bump to keep the four freemkv crates at unified versioning.
|
||||
autorip shipped UI smoothness + audit fixes today (v0.17.6, v0.17.7);
|
||||
libfreemkv carries no code changes, but is bumped + republished to
|
||||
crates.io so downstream `libfreemkv = "0.17"` consumers stay aligned
|
||||
on the latest patch version. See autorip CHANGELOG for the
|
||||
user-visible work in this release window.
|
||||
|
||||
## 0.17.5 (2026-05-08)
|
||||
|
||||
### Pass N recovery — kernel block-device fallback + per-range fixes
|
||||
|
||||
Live testing on direct-SATA BU40N + AACS-encrypted UHD disc revealed that
|
||||
the v0.17.3 single-shot SCSI READ path matched 0/22 of the small bad-sector
|
||||
LBAs that `dd if=/dev/sr0` recovers on the same drive. This release closes
|
||||
that gap and fixes several adjacent bugs that were silently capping
|
||||
recovery.
|
||||
|
||||
- **`/dev/sr0` pread fallback in `Drive::read` (Linux)**: when a SCSI READ
|
||||
via `/dev/sg*` returns Err, fall back to `posix_fadvise(POSIX_FADV_DONTNEED)
|
||||
+ pread()` against the corresponding block device. The kernel `sr_mod`
|
||||
driver runs ~5 internal retries per command without the per-attempt
|
||||
error-escalation overhead that userspace SG_IO retries pay, which is the
|
||||
source of dd's recovery advantage. End-to-end byte-verification confirms
|
||||
the fallback path returns real disc data (md5-equivalent to fresh dd from
|
||||
the same drive session). The block fd is opened in `Drive::open` by
|
||||
resolving `/sys/class/scsi_generic/sgN/device/block`; best-effort, with no
|
||||
fallback if open fails.
|
||||
|
||||
- **`Disc::patch` per-range watchdog fix**: when a range hit `MAX_RANGE_SECS`,
|
||||
the old code did `wedged_exit = true; break 'outer;` — a single slow range
|
||||
killed the entire patch. Now `break;` (skip this range, advance the outer
|
||||
for loop). Pre-fix, patch was dying after 4 sectors of the first slow
|
||||
range and never reaching the other 46.
|
||||
|
||||
- **Per-sector range budget**: replaced flat `MAX_RANGE_SECS = 180` with
|
||||
`range_budget_secs = (range_sectors × SECONDS_PER_SECTOR).min(RANGE_BUDGET_CAP_SECS)`.
|
||||
Tiny ranges exit fast (1-sector range = 25 s); medium ranges get
|
||||
proportional time (51-sector range ≈ 1275 s); large ranges still bounded
|
||||
by the 1800 s cap so they cannot monopolise pass 1.
|
||||
|
||||
- **`consecutive_failures` resets per range**: the wedge-exit detector
|
||||
(`>= 50 consecutive failures`) is for "stuck on the same range"; pre-fix
|
||||
the counter persisted across ranges, so 50 small post-bisection ranges
|
||||
with one failure each falsely tripped the wedge mid-pass. Reset at every
|
||||
range boundary.
|
||||
|
||||
- **Reverted inline 5× retry in patch**: a brief experiment that was
|
||||
measurably harmful — each "2 s" SCSI timeout paid ~1.5 s kernel SCSI
|
||||
mid-layer error-escalation overhead, so 5× retry took ~17 s per LBA and
|
||||
triggered the per-range watchdog in 4 sectors. Restored
|
||||
`READ_RECOVERY_TIMEOUT_MS = 60_000` (the v0.17.3 baseline; the kernel-
|
||||
auto-retry pattern is now provided by the `/dev/sr0` fallback above).
|
||||
|
||||
### Empirical results (a UHD disc, BU40N direct SATA)
|
||||
|
||||
- Pass 1: **94.6 MB recovered** (28% of formerly-bad data, 33 sr0 fallback
|
||||
saves), **11 s of main-title content** restored. Patch completed all 47
|
||||
retryable ranges naturally (was wedging at range 1 of 47 in v0.17.3).
|
||||
- Pass 2 cumulative: 95.2 MB (+0.6 MB; diminishing returns curve).
|
||||
- Remaining ~233 MB on the test disc appears physically unrecoverable on
|
||||
this hardware (kernel auto-retry can't decode it either).
|
||||
|
||||
### Behavioural notes
|
||||
|
||||
- The `/dev/sr0` fallback is Linux only; macOS and Windows fall back to
|
||||
the existing single-shot SCSI behaviour. The fallback is gated to
|
||||
`recovery=true` reads (only fires from the patch path, not the sweep
|
||||
path) to avoid page-cache pressure during multi-GB sequential ripping.
|
||||
|
||||
## 0.17.0 (2026-05-04)
|
||||
|
||||
### Code quality: unwrap safety, clippy compliance, test coverage
|
||||
|
||||
- **Sweep() hot path**: Replaced `.err().unwrap()` in `disc/mod.rs:1553` with explicit pattern matching (`match read_result { Ok(_) => unreachable!(), Err(e) => e }`). The original unwrap was logically safe (in the else branch after `is_ok()` check) but pattern matching makes the invariant explicit and avoids panic risk if logic changes.
|
||||
- **cargo clippy --lib**: Clean build with `-D warnings` across all targets. No lint failures.
|
||||
- **cargo test --lib**: All 256 tests pass (0 failed, 1 ignored). Integration tests for sweep/patch/dev-null validate the multi-pass recovery pipeline end-to-end.
|
||||
|
||||
### Patch pass algorithm fix
|
||||
|
||||
- `patch_internal` excludes Unreadable sectors from work list (only retries NonTrimmed/NonScraped/Unscraped). Previously attempted to retry already-failed sectors, wasting time and masking real progress.
|
||||
- Exposes `bytes_bad_in_title` in patch results for accurate UI reporting of unrecoverable data vs. recovered data.
|
||||
|
||||
## 0.16.1 (2026-04-30)
|
||||
|
||||
### Unified progress display for sweep and patch
|
||||
@@ -99,7 +948,7 @@ bridges (notably the Initio INIC-1618L) that crash on MEDIUM ERROR retries.
|
||||
- Patch instrumentation: counters for reads_ok/err, writes_ok/err, finished/unreadable.
|
||||
- `as_encoded_bytes()` replaces `as_bytes()` for portable OsStr handling.
|
||||
- Removed inline retry/reset from Drive::read — orchestration layer handles recovery.
|
||||
- Removed all `(internal)` references from public code.
|
||||
- Removed all internal references from public code.
|
||||
|
||||
## 0.13.26 (2026-04-27)
|
||||
|
||||
@@ -127,7 +976,7 @@ warnings` red on Linux CI that the Mac toolchain doesn't catch.
|
||||
|
||||
### Pre-commit gate uses CI's exact toolchain
|
||||
|
||||
`(internal)/scripts/precommit.sh` (new) runs `cargo +1.86 fmt
|
||||
A new pre-commit script runs `cargo +1.86 fmt
|
||||
--check`, `cargo +1.86 clippy -- -D warnings`, and `cargo +1.86 test
|
||||
--tests` across all 5 freemkv crates. Mirrors each repo's
|
||||
`.github/workflows/ci.yml` step-for-step. Use it as a pre-commit hook
|
||||
@@ -169,7 +1018,7 @@ Through the entire 0.13.x line, every CHECK CONDITION reply from the
|
||||
drive (the standard way SCSI reports a sector failure) was being
|
||||
collapsed into a synthetic `status=0xFF, sense_key=0` "transport
|
||||
wedge" sentinel and the real sense data was thrown away. Live tracing
|
||||
on the BU40N reading Dune 2 on 2026-04-27 confirmed it: the drive was
|
||||
on the BU40N reading a UHD disc on 2026-04-27 confirmed it: the drive was
|
||||
returning `host_status=0, driver_status=8, status=2, exec_elapsed_ms=1416`
|
||||
on every bad sector — a clean CHECK CONDITION carrying full sense
|
||||
data — and the library was misclassifying it as a wedge and bailing.
|
||||
@@ -241,7 +1090,7 @@ recovered or marked Unreadable. Calibration data
|
||||
|
||||
### Replace bisect-on-fail with hysteresis state machine (Block ↔ Single)
|
||||
|
||||
Live test on Dune 2 v0.13.21 showed bisect-on-fail recovered every
|
||||
Live test on a UHD disc v0.13.21 showed bisect-on-fail recovered every
|
||||
recoverable sector, but spent ~30 sec per damaged 60-block (paying a
|
||||
~5 sec kernel timeout at every bisection level). Each level descended
|
||||
log₂(60) ≈ 6 times on the failing branch.
|
||||
@@ -270,7 +1119,7 @@ Per-block cost on a 60-sector damaged block with 1 bad sector:
|
||||
|
||||
Plus inside a damaged cluster spanning many 60-blocks, hysteresis
|
||||
pays the bpt=batch fail cost ONCE on entry; bisection paid it every
|
||||
60 sectors. For Dune 2's ~1248-sector boundary cluster that's ~21
|
||||
60 sectors. For the ~1248-sector boundary cluster that's ~21
|
||||
fewer 5-sec waits = ~100 sec saved.
|
||||
|
||||
Telemetry: new `phase=mode_change` trace event with `from`, `to`,
|
||||
@@ -288,8 +1137,7 @@ synthetic BU40N-pattern reader; same 100% recovery expectation.
|
||||
### Fix: Disc::copy bisect-on-fail (replaces skip-forward)
|
||||
|
||||
Empirical live-hardware testing on the LG BU40N (see
|
||||
`(internal)/docs/audits/2026-04-26-test-plan-audit.md` and the
|
||||
TEST_PLAN.md run log) revealed that the drive often **fails
|
||||
the internal test-plan audit and run log) revealed that the drive often **fails
|
||||
multi-sector READ commands** in damaged regions but **succeeds when
|
||||
asked one sector at a time**. The old skip-forward strategy responded
|
||||
to multi-sector failures by jumping up to 1 % of the disc forward,
|
||||
@@ -301,7 +1149,7 @@ half, retry each half, recurse down to single-sector reads. Sectors
|
||||
the drive can read individually are recovered in Pass 1; only sectors
|
||||
that fail at bpt=1 are marked NonTrimmed for the patch passes.
|
||||
|
||||
Empirical results on Dune 2 UHD on the BU40N:
|
||||
Empirical results on a UHD disc on the BU40N:
|
||||
- Old algorithm: 25 GB read in Pass 1, then ~6 GB skip-forwarded;
|
||||
retry passes failed to recover most of the skipped zone.
|
||||
- New algorithm: ~99 % of disc recovered in Pass 1; only the truly
|
||||
@@ -343,7 +1191,7 @@ LUN/BUS/HOST reset. `READ_RECOVERY_TIMEOUT_MS` (60 s) unchanged.
|
||||
`ioctl(fd, SG_IO, &hdr)`. The old pattern abandoned slow-but-alive
|
||||
commands faster than the drive could drain its internal queue,
|
||||
deepening the BU40N wedge. Per the audit at
|
||||
`(internal)/docs/audits/2026-04-26-scsi-architecture-research.md`,
|
||||
internal SCSI-architecture research,
|
||||
no reference project (MakeMKV / sg_dd / ddrescue) does what we did —
|
||||
all use sync blocking SG_IO with 8-60 s timeouts and let the kernel's
|
||||
mid-layer (`scsi_eh.rst`) run ABORT TASK / LUN RESET / BUS RESET /
|
||||
@@ -470,7 +1318,7 @@ clean reads (Finished sectors) and freezes during skip-forward bad zones,
|
||||
which made every previous version's UI look hung at the bad-zone boundary.
|
||||
This was the v0.13.9 stall-guard origin bug.
|
||||
|
||||
Live trace from v0.13.14: Pass 1 hit a Dune 2 bad zone at 24 GB and
|
||||
Live trace from v0.13.14: Pass 1 hit a bad zone at 24 GB and
|
||||
appeared "stuck" for 14 minutes per autorip's UI (`bytes_good = 23.97 GB`
|
||||
unchanged). Disc trace events showed `pos` actually advanced from 25.8 GB
|
||||
to 70 GB during that window — Pass 1 was 83 % through the disc, marking
|
||||
@@ -518,7 +1366,7 @@ trace events were silently dropped by the default `libfreemkv=warn` rule.
|
||||
|
||||
### Telemetry: instrument the rip pipeline for in-flight diagnosis
|
||||
|
||||
v0.13.12 shipped Fix 1+2+4 + cross-platform parity but a live test on Dune 2
|
||||
v0.13.12 shipped Fix 1+2+4 + cross-platform parity but a live test
|
||||
showed Pass 1 sat for 14 minutes with `bytes_good=0` while the inner loop
|
||||
appeared to iterate (the throttled `on_progress` log fired every 78s). The
|
||||
async fd_recovery design at §7 said each `execute()` call should bound at
|
||||
@@ -562,7 +1410,7 @@ The v0.13.9 stall guard at `disc/mod.rs` exited Pass 1 early when
|
||||
`bytes_good` was flat for `stall_secs` (default 120s). This violated the
|
||||
ddrescue model: Pass 1 must sweep end-to-end, marking failed reads
|
||||
NonTrimmed for Pass 2 retry. The guard caused Pass 1 to bail at 30% on
|
||||
Dune 2 with 56 GB still NonTried, leaving Pass 2 nothing useful to do.
|
||||
a UHD disc with 56 GB still NonTried, leaving Pass 2 nothing useful to do.
|
||||
|
||||
- Deleted the stall-guard state vars and the `if cur_good != ...
|
||||
break 'outer;` block.
|
||||
@@ -617,7 +1465,7 @@ same observable recovery contract as Linux:
|
||||
|
||||
`PatchResult` now reports `blocks_attempted`, `blocks_read_ok`,
|
||||
`blocks_read_failed`. Pass 2's "100 minutes recovered 0 bytes" mystery
|
||||
(Dune 2) becomes diagnosable from these counters: distinguish "drive
|
||||
(a UHD disc) becomes diagnosable from these counters: distinguish "drive
|
||||
returned Ok but write/record dropped data" from "every read was Err for
|
||||
the entire range" without instrumenting from outside the lib.
|
||||
|
||||
@@ -646,7 +1494,7 @@ against the spawned close()). The intent was to escape the 60-s
|
||||
blocking reopen.
|
||||
|
||||
The cost was too high: a single transient poll timeout permanently
|
||||
killed the transport. Live test on Dune 2 (post-replug):
|
||||
killed the transport. Live test on a UHD disc (post-replug):
|
||||
- Pass 1 ran for **45 ms** then returned with 0 GB good and 80 GB
|
||||
pending.
|
||||
- The first SCSI READ timed out, fd went to -1, every subsequent
|
||||
@@ -679,7 +1527,7 @@ Sync bump for the autorip-side fix in 0.13.10 (Pass 1 batch reporting).
|
||||
### Fix: Disc::copy silent stall + SgIoTransport reopen-after-timeout serialization
|
||||
|
||||
Two correlated fixes for a hang observed live on the LG BU40N during a
|
||||
v0.13.8 rip of Dune: Part Two. At ~30 % progress through Pass 1
|
||||
v0.13.8 rip of a UHD disc. At ~30 % progress through Pass 1
|
||||
(disc → ISO), `bytes_good` froze for 10+ minutes with `errs=0`,
|
||||
no error surfaced, drive not wedged.
|
||||
|
||||
@@ -1566,10 +2414,9 @@ Rip recovery rewritten. The old binary-search-per-bad-sector model paid the full
|
||||
- **Kernel transfer limit detection**: auto-detect `max_hw_sectors_kb` via sysfs, resolve sg→block device. Previously hardcoded to 510 sectors (1MB) which exceeded the 120KB kernel limit, causing all reads to error and fall back to 6KB reads at 4.8 MB/s. Now auto-tunes to 48 sectors (96KB) or whatever the device supports.
|
||||
- **Result: 12.5 MB/s sustained, 23 MB/s peak** (was 4.8 MB/s)
|
||||
|
||||
### LibreDrive — full init pipeline
|
||||
### Drive init pipeline
|
||||
|
||||
- **All 10 ARM handlers translated**: unlock, firmware upload (A: WRITE_BUFFER, B: MODE SELECT), calibrate (256 zones), register reads, status, probe, set_read_speed, keepalive, timing
|
||||
- **Cold boot firmware upload**: WRITE_BUFFER 1888B (A variant) or MODE SELECT 2496B (B variant) proven on hardware
|
||||
- **Full drive init pipeline**: unlock, calibrate (256 zones), register reads, status, probe, set_read_speed, keepalive, timing
|
||||
- **Speed calibration**: 256+ disc surface probes, 64-entry speed table, triple SET_CD_SPEED
|
||||
- **Platform trait locked down**: `pub(crate)`, 3 methods only (init, set_read_speed, is_ready)
|
||||
- **Init guard**: prevents double-init, signature mismatch aborts early
|
||||
@@ -1584,7 +2431,7 @@ Rip recovery rewritten. The old binary-search-per-bad-sector model paid the full
|
||||
### Profiles
|
||||
|
||||
- **206 profiles with full per-drive data**: ld_microcode (base64), all CDBs, speed tables, signatures
|
||||
- **Automated pipeline**: `sdf_unpack --profiles` → profiles.json (no manual merging)
|
||||
- **Automated pipeline**: profile import → profiles.json (no manual merging)
|
||||
|
||||
## 0.4.0 (2026-04-07)
|
||||
|
||||
|
||||
+19
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "libfreemkv"
|
||||
version = "0.16.2"
|
||||
version = "0.30.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.86"
|
||||
license = "AGPL-3.0-only"
|
||||
@@ -8,6 +8,8 @@ description = "Open source raw disc access library for optical drives"
|
||||
repository = "https://github.com/freemkv/libfreemkv"
|
||||
keywords = ["bluray", "uhd", "optical", "scsi", "disc"]
|
||||
categories = ["hardware-support", "multimedia"]
|
||||
# Keep internal AI-instruction / private notes out of the published crate.
|
||||
exclude = []
|
||||
|
||||
[dependencies]
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
@@ -28,6 +30,22 @@ base64 = "0.22.1"
|
||||
# under project docs ("Acceptable strings: debug/trace logging"). Consumers (autorip)
|
||||
# wire a tracing subscriber and pipe events into the JSONL debug log.
|
||||
tracing = "0.1"
|
||||
# Bounded MPSC channel with kernel-wakeup send_timeout. Used by `io::pipeline`
|
||||
# so the halt-aware send/finish loops can BLOCK on consumer drain instead of
|
||||
# polling — the 50 ms poll cadence of the previous mpsc-based impl capped mux
|
||||
# throughput at ~1 MB/s (0.21.7).
|
||||
crossbeam-channel = "0.5"
|
||||
# Persistent work-stealing thread pool for parallel AACS unit
|
||||
# decryption. Per-call std::thread::scope spawned fresh OS threads
|
||||
# and that overhead dominated for typical batch sizes (60 units).
|
||||
# rayon's global pool initialises once on first use.
|
||||
rayon = "1"
|
||||
# SIMD-accelerated bytestring search. Drives the HEVC/H.264 start-code
|
||||
# scan in `mux::codec::h264::find_start_code` — naive byte-by-byte
|
||||
# walk is ~500 MB/s single-thread on x86_64; memchr's vectorised
|
||||
# `memmem::find` for the 3-byte `00 00 01` needle hits ~5 GB/s on
|
||||
# AVX2-capable hosts.
|
||||
memchr = "2"
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
# libfreemkv — Feature List
|
||||
|
||||
## v0.10.10 (current)
|
||||
|
||||
### Done
|
||||
- [x] Drive access: open, identify, unlock, firmware upload, speed calibration, eject
|
||||
- [x] 206 bundled drive profiles (MediaTek MT1959 A + B variants)
|
||||
- [x] SCSI transport: Linux SG_IO, macOS IOKit, Windows SPTI
|
||||
- [x] UDF 2.50 filesystem parser (metadata partitions, Blu-ray profile)
|
||||
- [x] MPLS playlist parser (play items, STN table, secondary streams)
|
||||
- [x] CLPI clip info parser (EP map, sector extents)
|
||||
- [x] AACS 1.0 decryption (4 VUK paths: KEYDB, media key, processing key, device key)
|
||||
- [x] AACS 2.0 SCSI handshake (P-256/SHA-256 ECDH, bus decryption, read data key)
|
||||
- [x] DVD IFO parser (VMG, VTS, PGC chains, cell addresses)
|
||||
- [x] DVD CSS decryption (bus auth, disc key via player keys, title key, sector descramble)
|
||||
- [x] CSS Stevenson plaintext attack for ISO key recovery
|
||||
- [x] MPEG-2 PS demuxer (DVD Program Stream with PES extraction)
|
||||
- [x] MPEG-2 video codec parser (sequence header, quantizer matrices, keyframe detection)
|
||||
- [x] KEYDB.cfg download, verify, save (raw TCP, zero HTTP deps)
|
||||
- [x] Content reading with adaptive batch sizing, error recovery, 12+ MB/s
|
||||
- [x] Stream labels: 5 BD-J format parsers (Paramount, Criterion, Pixelogic, CTRM, Deluxe)
|
||||
- [x] MKV muxer: 14 codec parsers (H.264, HEVC, MPEG-2, AC-3, DTS, TrueHD, PGS, DVD Sub, VC-1, LPCM, +4 more)
|
||||
- [x] SectorReader trait: decouples disc scanning from SCSI
|
||||
- [x] 7 stream types: Disc, ISO, MKV, M2TS, Network, Stdio, Null
|
||||
- [x] PES pipeline with unified Stream trait (any source → any dest)
|
||||
- [x] FMKV metadata header for M2TS and network streams
|
||||
- [x] Numeric error codes only (no English text in library)
|
||||
- [x] Event system for progress callbacks
|
||||
|
||||
### Planned
|
||||
- [ ] Windows testing on real hardware
|
||||
- [ ] Pioneer Renesas platform support (48 drives, need GET_CONFIG 010C)
|
||||
- [ ] TranscodeStream (ffmpeg integration)
|
||||
- [ ] ISO write with BD-compliant UDF structure
|
||||
@@ -1,6 +1,5 @@
|
||||
# libfreemkv — local dev helper.
|
||||
# Mirrors the cross-crate scripts in (internal)/scripts/test-all.sh
|
||||
# but scoped to this single crate.
|
||||
# Mirrors the workspace-wide CI checks but scoped to this single crate.
|
||||
|
||||
.PHONY: test build check ci clean
|
||||
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
|
||||
Rust library for 4K UHD / Blu-ray / DVD optical drives. Drive access, disc scanning, stream labels, AACS decryption, CSS decryption, KEYDB updates, and content reading in one crate. Bundled drive profiles — no external files needed.
|
||||
|
||||
Built-in keys cover DVDs and Blu-rays (AACS 1.0). For UHD (AACS 2.0 / 2.1) discs, an optional `keydb.cfg` supplies disc-specific volume unique keys.
|
||||
|
||||
**12+ MB/s** sustained read speeds on BD. Full init: unlock, firmware upload, speed calibration — all from pure Rust.
|
||||
|
||||
Multi-lingual by design — the library outputs structured data and numeric error codes, never English text. Build any UI or localization on top.
|
||||
@@ -18,7 +20,7 @@ Part of the [freemkv](https://github.com/freemkv) project.
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
libfreemkv = "0.13"
|
||||
libfreemkv = "0.25"
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
@@ -53,27 +55,47 @@ output.finish()?;
|
||||
|
||||
### Multi-pass recovery rip
|
||||
|
||||
For damaged discs, the library offers a two-stage rip model: fast sweep with zero-fill and a ddrescue-format mapfile, then targeted retry of bad ranges. See [`docs/rip-recovery.md`](docs/rip-recovery.md) for the full architecture.
|
||||
For damaged discs the library exposes two flat verbs — `Disc::sweep` for the
|
||||
forward Pass 1 and `Disc::patch` for retrying bad ranges. The library never
|
||||
loops; the multipass policy is the caller's job. See
|
||||
[`docs/rip-recovery.md`](docs/rip-recovery.md).
|
||||
|
||||
```rust
|
||||
use libfreemkv::disc::{CopyOptions, PatchOptions};
|
||||
use libfreemkv::{SweepOptions, PatchOptions};
|
||||
use libfreemkv::disc::{mapfile, mapfile_path_for};
|
||||
use std::path::Path;
|
||||
|
||||
// Pass 1: disc → ISO. Fast 64 KB reads, skip-forward on failure,
|
||||
// zero-fill bad blocks, write a sidecar .mapfile.
|
||||
let mut result = disc.copy(
|
||||
&mut drive,
|
||||
Path::new("disc.iso"),
|
||||
&CopyOptions { skip_on_error: true, skip_forward: true, ..Default::default() },
|
||||
)?;
|
||||
let iso = Path::new("disc.iso");
|
||||
|
||||
// Pass 2..N: retry bad ranges with full drive recovery.
|
||||
// Idempotent — call as many times as you want.
|
||||
while result.bytes_unreadable + result.bytes_pending > 0 {
|
||||
let pr = disc.patch(&mut drive, Path::new("disc.iso"), &PatchOptions::default())?;
|
||||
if pr.bytes_recovered_this_pass == 0 { break; }
|
||||
// Pass 1: disc → ISO. Skip-on-error, zero-fill, write the sidecar mapfile.
|
||||
disc.sweep(&mut drive, iso, &SweepOptions {
|
||||
decrypt: true,
|
||||
resume: false,
|
||||
batch_sectors: None,
|
||||
skip_on_error: true,
|
||||
progress: None,
|
||||
halt: None,
|
||||
})?;
|
||||
|
||||
// Pass 2..N: retry every non-finished range. Idempotent.
|
||||
loop {
|
||||
let map = mapfile::Mapfile::load(&mapfile_path_for(iso))?;
|
||||
let stats = map.stats();
|
||||
if stats.bytes_pending + stats.bytes_unreadable == 0 { break; }
|
||||
|
||||
let outcome = disc.patch(&mut drive, iso, &PatchOptions {
|
||||
decrypt: true,
|
||||
block_sectors: None,
|
||||
full_recovery: true,
|
||||
reverse: true,
|
||||
wedged_threshold: 50,
|
||||
progress: None,
|
||||
halt: None,
|
||||
})?;
|
||||
if outcome.bytes_recovered_this_pass == 0 { break; }
|
||||
}
|
||||
|
||||
// Then mux from the ISO via the normal stream pipeline (no drive involvement).
|
||||
// Mux from the ISO via the normal stream pipeline (no drive involvement).
|
||||
```
|
||||
|
||||
## What It Does
|
||||
@@ -92,16 +114,20 @@ while result.bytes_unreadable + result.bytes_pending > 0 {
|
||||
| Stream | Input | Output | Transport |
|
||||
|--------|-------|--------|-----------|
|
||||
| DiscStream | Yes | -- | Optical drive via SCSI |
|
||||
| IsoStream | Yes | -- | Blu-ray ISO image file (read via stream pipeline; written via `Disc::copy()`) |
|
||||
| IsoStream | Yes | -- | Blu-ray ISO image file (read via stream pipeline; written via `Disc::sweep()`) |
|
||||
| MkvStream | Yes | Yes | Matroska container |
|
||||
| M2tsStream | Yes | Yes | BD transport stream with FMKV metadata header |
|
||||
| NetworkStream | Yes (listen) | Yes (connect) | TCP with FMKV metadata header |
|
||||
| StdioStream | Yes (stdin) | Yes (stdout) | Raw byte pipe |
|
||||
| NullStream | -- | Yes | Discard sink (byte counter for benchmarks) |
|
||||
|
||||
Streams implement `pes::Stream` (frame-level). `input()` / `output()` resolve URL strings to PES stream instances. All URLs use the `scheme://path` format — bare paths are rejected.
|
||||
Streams implement `FrameSource` (read) and/or `FrameSink` (write); direction is type-checked. `input()` / `output()` resolve URL strings to PES stream instances. All URLs use the `scheme://path` format — bare paths are rejected.
|
||||
|
||||
AACS decryption requires a KEYDB.cfg file. If available at `~/.config/aacs/KEYDB.cfg` or passed via `ScanOptions`, the library handles everything — handshake, key derivation, and per-sector decryption — without the application needing to know anything about encryption.
|
||||
### Keys
|
||||
|
||||
DVDs (CSS) decrypt out of the box — the 1999-era public player keys are compiled into the library.
|
||||
|
||||
Blu-rays and UHD (AACS) require a `keydb.cfg` at `~/.config/freemkv/keydb.cfg` (or passed via `ScanOptions`). The file holds all DKs, PKs, host certs, and per-disc VUKs. No AACS key material is compiled into the binary.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -122,7 +148,8 @@ Disc — scan titles, streams, AACS/CSS state
|
||||
└── KEYDB — download + verify + save
|
||||
|
||||
Streams — unified PES pipeline
|
||||
├── pes::Stream — read()/write() PES frames
|
||||
├── FrameSource — read() PES frames (direction-typed)
|
||||
├── FrameSink — write() PES frames (direction-typed)
|
||||
├── DiscStream — sectors → decrypt → TS demux → PES
|
||||
├── IsoStream — ISO file → decrypt → TS demux → PES
|
||||
├── MkvStream — MKV mux/demux
|
||||
|
||||
+43
-41
@@ -44,35 +44,30 @@ while let Ok(Some(frame)) = input.read() {
|
||||
output.finish()?;
|
||||
```
|
||||
|
||||
The `pes::Stream` trait:
|
||||
The `FrameSource` and `FrameSink` traits — direction is type-checked, so
|
||||
calling `read()` on a write-only sink (or `write()` on a read-only source)
|
||||
is a compile error rather than a runtime fault:
|
||||
|
||||
```rust
|
||||
pub trait Stream {
|
||||
fn read(&mut self) -> io::Result<Option<PesFrame>>;
|
||||
fn write(&mut self, frame: &PesFrame) -> io::Result<()>;
|
||||
fn finish(&mut self) -> io::Result<()>;
|
||||
pub trait FrameSource: Send {
|
||||
fn read(&mut self) -> Result<Option<PesFrame>, Error>;
|
||||
fn info(&self) -> &DiscTitle;
|
||||
fn codec_private(&self, track: usize) -> Option<Vec<u8>>;
|
||||
fn headers_ready(&self) -> bool;
|
||||
fn codec_private(&self, track: usize) -> Option<Vec<u8>> { None }
|
||||
fn headers_ready(&self) -> bool { true }
|
||||
}
|
||||
```
|
||||
|
||||
## IOStream (byte-level API)
|
||||
|
||||
For raw byte copies (disc→ISO, resume, benchmarks). Lower level than PES.
|
||||
|
||||
```rust
|
||||
let opts = InputOptions::default();
|
||||
let mut input = open_input("iso://Disc.iso", &opts)?;
|
||||
let mut output = open_output("mkv://Movie.mkv", input.info())?;
|
||||
io::copy(&mut *input, &mut *output)?;
|
||||
output.finish()?;
|
||||
pub trait FrameSink: Send {
|
||||
fn write(&mut self, frame: &PesFrame) -> Result<(), Error>;
|
||||
fn finish(self: Box<Self>) -> Result<(), Error>;
|
||||
fn info(&self) -> &DiscTitle;
|
||||
}
|
||||
```
|
||||
|
||||
## Streams
|
||||
|
||||
All streams implement `IOStream` (byte-level) and/or `pes::Stream` (frame-level).
|
||||
URL-based resolvers open any stream by string.
|
||||
All streams implement `FrameSource` (read) and/or `FrameSink` (write); the
|
||||
directional split prevents runtime "wrong-direction" errors. URL-based
|
||||
resolvers open any stream by string.
|
||||
|
||||
| Stream | Input | Output | URL | Transport |
|
||||
|--------|-------|--------|-----|-----------|
|
||||
@@ -87,21 +82,14 @@ URL-based resolvers open any stream by string.
|
||||
All URLs require a `scheme://path` format. Bare paths are rejected.
|
||||
|
||||
```rust
|
||||
// PES pipeline (frame-level)
|
||||
// PES pipeline (frame-level) — input() returns Box<dyn FrameSource>,
|
||||
// output() returns Box<dyn FrameSink>.
|
||||
let input = libfreemkv::input("disc:///dev/sg4", &opts)?; // DiscStream
|
||||
let input = libfreemkv::input("iso://Dune.iso", &opts)?; // IsoStream
|
||||
let output = libfreemkv::output("mkv://Dune.mkv", &title)?; // MkvOutputStream
|
||||
let output = libfreemkv::output("m2ts://Dune.m2ts", &title)?; // M2tsOutputStream
|
||||
let output = libfreemkv::output("network://10.0.0.1:9000", &title)?; // NetworkOutputStream
|
||||
let input = libfreemkv::input("iso://Movie.iso", &opts)?; // IsoStream
|
||||
let output = libfreemkv::output("mkv://Movie.mkv", &title)?; // MkvOutputStream
|
||||
let output = libfreemkv::output("m2ts://Movie.m2ts", &title)?; // M2tsOutputStream
|
||||
let output = libfreemkv::output("network://192.0.2.10:9000", &title)?; // NetworkOutputStream
|
||||
let output = libfreemkv::output("null://", &title)?; // NullOutputStream
|
||||
|
||||
// IOStream (byte-level)
|
||||
let input = open_input("disc://", &opts)?; // DiscStream
|
||||
let input = open_input("iso://Dune.iso", &opts)?; // IsoStream
|
||||
let output = open_output("iso://Copy.iso", &meta)?; // IsoStream (write)
|
||||
let output = open_output("mkv://Dune.mkv", &meta)?; // MkvStream
|
||||
let output = open_output("m2ts://Dune.m2ts", &meta)?; // M2tsStream
|
||||
let output = open_output("null://", &meta)?; // NullStream
|
||||
```
|
||||
|
||||
### FMKV Metadata Header
|
||||
@@ -175,20 +163,34 @@ libfreemkv/src/
|
||||
├── lib.rs Public exports
|
||||
├── error.rs Error codes (no English)
|
||||
├── event.rs Event types for callbacks
|
||||
├── halt.rs Halt cancellation token (Arc<AtomicBool> wrapper)
|
||||
├── io/ Pipeline + WritebackFile primitives
|
||||
│ ├── mod.rs Re-exports WritebackFile, Pipeline, Sink, Flow
|
||||
│ ├── pipeline.rs Generic Pipeline<I, R> + Sink trait
|
||||
│ ├── writeback_file.rs WritebackFile (was crate::io::Writer)
|
||||
│ └── writeback.rs sync_file_range pipeline
|
||||
├── drive/ Drive (open, init, single-shot read)
|
||||
│ ├── mod.rs Drive struct, init, read (single-shot), reset, eject
|
||||
│ ├── capture.rs Drive profile capture for contribution
|
||||
│ ├── linux.rs Linux drive discovery
|
||||
│ ├── macos.rs macOS drive discovery
|
||||
│ └── windows.rs Windows drive discovery
|
||||
├── disc/ Disc (scan, titles, AACS setup)
|
||||
├── disc/ Disc (scan, titles, AACS setup, sweep, patch)
|
||||
│ ├── mod.rs Disc struct, scan, titles, formats
|
||||
│ ├── sweep.rs Disc::sweep (Pass 1 forward sweep)
|
||||
│ ├── patch.rs Disc::patch (Pass N retry over mapfile)
|
||||
│ ├── mapfile.rs ddrescue-format mapfile
|
||||
│ └── read_error.rs ReadCtx / ReadAction state machine
|
||||
├── scsi/ SCSI transport (Linux SG_IO, macOS IOKit, Windows SPTI)
|
||||
├── platform/ Drive unlock (MT1959 A/B)
|
||||
├── aacs/ AACS decryption (handshake, keys, keydb, decrypt)
|
||||
├── css/ DVD CSS cipher
|
||||
├── decrypt.rs Unified decrypt dispatcher (AACS/CSS/None)
|
||||
├── pes.rs PES frame types, Stream trait
|
||||
├── sector.rs SectorReader trait
|
||||
├── pes.rs PES frame types, FrameSource / FrameSink traits
|
||||
├── sector/ Sector I/O (was sector.rs in 0.17)
|
||||
│ ├── mod.rs SectorSource, SectorSink traits
|
||||
│ ├── file.rs FileSectorSource, FileSectorSink (ISO-backed)
|
||||
│ └── decrypting.rs DecryptingSectorSource decorator
|
||||
├── udf.rs UDF 2.50 filesystem parser
|
||||
├── mpls.rs MPLS playlist parser
|
||||
├── clpi.rs CLPI clip info parser
|
||||
@@ -199,15 +201,15 @@ libfreemkv/src/
|
||||
├── profile.rs Bundled drive profiles
|
||||
├── speed.rs DriveSpeed enum
|
||||
├── mux/
|
||||
│ ├── mod.rs IOStream trait, public exports
|
||||
│ ├── resolve.rs URL parser + open_input/open_output + input/output
|
||||
│ ├── mod.rs Public mux exports
|
||||
│ ├── resolve.rs URL parser + input/output (Box<dyn FrameSource/Sink>)
|
||||
│ ├── meta.rs FMKV header format
|
||||
│ ├── disc.rs DiscStream (optical drive → PES)
|
||||
│ ├── iso.rs IsoStream (ISO image read/write)
|
||||
│ ├── iso.rs IsoStream (ISO image read)
|
||||
│ ├── isowriter.rs ISO image writer (UDF, AVDP, multi-extent)
|
||||
│ ├── mkvstream.rs MkvStream (bidirectional Matroska, IOStream)
|
||||
│ ├── mkvstream.rs MkvStream (bidirectional Matroska)
|
||||
│ ├── mkvout.rs MkvOutputStream (PES → MKV)
|
||||
│ ├── m2ts.rs M2tsStream (BD-TS, IOStream)
|
||||
│ ├── m2ts.rs M2tsStream (BD-TS)
|
||||
│ ├── pesout.rs PES output streams (M2ts, Network, Stdio, Null)
|
||||
│ ├── network.rs NetworkStream (TCP + FMKV header)
|
||||
│ ├── stdio.rs StdioStream (stdin/stdout pipe)
|
||||
|
||||
@@ -65,8 +65,12 @@ libfreemkv (lib.rs)
|
||||
│
|
||||
├── Streaming
|
||||
│ ├── mux/ Stream implementations (Disc, ISO, MKV, M2TS, Network, Stdio, Null)
|
||||
│ ├── pes PES frame types, Stream trait (read/write frames)
|
||||
│ └── sector SectorReader trait — abstracts disc vs ISO vs file
|
||||
│ ├── pes PES frame types; FrameSource / FrameSink direction-typed traits
|
||||
│ └── sector/ SectorSource / SectorSink traits, FileSector{Source,Sink}, DecryptingSectorSource
|
||||
│
|
||||
├── I/O Primitives
|
||||
│ ├── halt Halt cancellation token (one Arc<AtomicBool>, cloneable)
|
||||
│ └── io/ Pipeline<I, R> + Sink trait + WritebackFile (bounded-cache writer)
|
||||
│
|
||||
├── Support
|
||||
│ ├── keydb KEYDB.cfg download, parse, verify, save
|
||||
|
||||
+8
-4
@@ -97,7 +97,9 @@ drive.probe_disc()?;
|
||||
// Scan disc (UDF + playlists + AACS — all automatic)
|
||||
let disc = Disc::scan(&mut drive, &ScanOptions::default())?;
|
||||
|
||||
// Stream pipeline — PES frames from any source to any output
|
||||
// Stream pipeline — PES frames from any source to any output.
|
||||
// 0.18: input() returns Box<dyn FrameSource>, output() returns Box<dyn FrameSink>;
|
||||
// direction is type-checked, so calling .write() on an input is a compile error.
|
||||
let opts = InputOptions::default();
|
||||
let mut input = libfreemkv::input("disc:///dev/sg4", &opts)?;
|
||||
let title = input.info().clone();
|
||||
@@ -121,11 +123,13 @@ output.finish()?;
|
||||
| aacs/ | [aacs.md](aacs.md) | Key resolution + content decrypt + bus handshake |
|
||||
| css/ | -- | DVD CSS cipher |
|
||||
| decrypt.rs | -- | Unified decrypt dispatcher (AACS/CSS/None) |
|
||||
| disc/ | -- | High-level scan + read API |
|
||||
| disc/ | [rip-recovery.md](rip-recovery.md) | Disc::scan + Disc::sweep + Disc::patch + mapfile |
|
||||
| labels/ | -- | BD-J stream labels (5 format parsers) |
|
||||
| mux/ | -- | Stream implementations (7 stream types) |
|
||||
| pes.rs | -- | PES frame types + Stream trait |
|
||||
| sector.rs | -- | SectorReader trait |
|
||||
| pes.rs | -- | PES frame types + FrameSource / FrameSink traits |
|
||||
| sector/ | -- | SectorSource / SectorSink + DecryptingSectorSource decorator |
|
||||
| io/ | -- | Pipeline<I, R> + Sink trait + WritebackFile |
|
||||
| halt.rs | -- | Halt cancellation token |
|
||||
| keydb.rs | -- | KEYDB download, parse, save |
|
||||
| error.rs | -- | Error codes (E1xxx-E8xxx) |
|
||||
| event.rs | -- | Drive event system |
|
||||
|
||||
@@ -52,8 +52,8 @@ selects the per-CDB timeout:
|
||||
|
||||
| `recovery` | Timeout | Used by |
|
||||
|------------|----------|------------------------------------------|
|
||||
| `false` | 1.5 s | `Disc::copy` fast skip-forward sweep, `DiscStream::fill_extents` |
|
||||
| `true` | 30 s | `Disc::patch` multi-pass over the mapfile |
|
||||
| `false` | 1.5 s | `Disc::sweep` fast skip-forward pass, `DiscStream::fill_extents` |
|
||||
| `true` | 30 s | `Disc::patch` retry pass over the mapfile |
|
||||
|
||||
On any SCSI failure or timeout, `read` returns `Err(DiscRead)` immediately.
|
||||
There are no inline retries, no SCSI reset, no Phase 1/2/3 escalation.
|
||||
|
||||
+43
-22
@@ -12,21 +12,24 @@ reach into the others.
|
||||
|
||||
| Layer | Where it lives | What it does |
|
||||
|-------|---------------|--------------|
|
||||
| 1 — Bad-range retry | `Disc::patch` (multi-pass over the mapfile) | Re-reads non-`+` ranges with the long timeout. Idempotent; call N times. |
|
||||
| 1 — Bad-range retry | `Disc::patch` (one pass over the mapfile per call) | Re-reads non-`+` ranges with the long timeout. Idempotent; caller invokes N times. |
|
||||
| 2 — Single-shot primitive | `Drive::read` in `src/drive/mod.rs` | One CDB, one timeout, one result. No inline retries, no SCSI reset. |
|
||||
| 3 — In-loop request adaptation | `DiscStream::fill_extents` adaptive batch sizer | Halves the batch on failure, retries at the same LBA, walks back up on a clean-read streak. |
|
||||
|
||||
The caller orchestrates layer 1. Autorip's `rip_disc` loops `copy` then
|
||||
N × `patch` per the `MAX_RETRIES` config, then hands the ISO off to the
|
||||
existing mux pipeline. Layer 3 runs inside any consumer of `DiscStream`
|
||||
(direct PES pipeline, ISO playback, etc.) without caller involvement.
|
||||
The library exposes flat verbs; the caller drives the multipass loop. Autorip
|
||||
runs `Disc::sweep` once, then loops `Disc::patch` until either the mapfile is
|
||||
clean or the configured retry budget is exhausted, then hands the ISO off to
|
||||
the mux pipeline. The `freemkv` CLI does the same shape with a
|
||||
terminal-output progress sink. Layer 3 runs inside any consumer of
|
||||
`DiscStream` (direct PES pipeline, ISO playback, etc.) without caller
|
||||
involvement.
|
||||
|
||||
Three primitives compose the disc-side flow:
|
||||
|
||||
| Primitive | What it does |
|
||||
|---------------------------|-----------------------------------------------------------------------|
|
||||
| `Disc::copy` | disc → ISO. Writes a sidecar `.mapfile`. Opt-in skip-forward on failure. |
|
||||
| `Disc::patch` | Re-reads bad ranges from the drive. Idempotent; call N times. |
|
||||
| `Disc::sweep` | disc → ISO, one forward pass. Writes a sidecar `.mapfile`. Opt-in skip-on-error. |
|
||||
| `Disc::patch` | Re-reads bad ranges from the drive. One pass per call; caller invokes N times. |
|
||||
| `DiscStream` (ISO source) | Reads sectors from the ISO, feeds decrypt → demux → codec → mux. |
|
||||
|
||||
## Data model
|
||||
@@ -60,28 +63,46 @@ Status characters match ddrescue:
|
||||
|
||||
Position and size are hex byte offsets into the ISO.
|
||||
|
||||
### `CopyOptions` and `PatchOptions`
|
||||
### `SweepOptions` and `PatchOptions`
|
||||
|
||||
`Disc::copy()` auto-detects the pass from mapfile state:
|
||||
The library no longer dispatches between sweep and patch internally — the
|
||||
caller picks the verb explicitly per pass. The two option structs are flat
|
||||
and have no overlap:
|
||||
|
||||
```rust
|
||||
CopyOptions {
|
||||
decrypt: true, // decrypt AACS/CSS sectors
|
||||
multipass: true, // enable mapfile + skip-on-error + damage-jump
|
||||
progress: Some(&reporter), // progress callback
|
||||
halt: Some(flag), // halt flag for graceful stop
|
||||
SweepOptions {
|
||||
decrypt: true,
|
||||
resume: false,
|
||||
batch_sectors: None,
|
||||
skip_on_error: true, // damage-jump + zero-fill on read failure
|
||||
progress: Some(&reporter),
|
||||
halt: Some(flag),
|
||||
}
|
||||
|
||||
PatchOptions {
|
||||
decrypt: true,
|
||||
block_sectors: None,
|
||||
full_recovery: true,
|
||||
reverse: true, // walk bad ranges high → low LBA
|
||||
wedged_threshold: 50,
|
||||
progress: Some(&reporter),
|
||||
halt: Some(flag),
|
||||
}
|
||||
```
|
||||
|
||||
Dispatch logic:
|
||||
- No mapfile → sweep (fresh Pass 1)
|
||||
- Mapfile with NonTried (`?`) → sweep with resume
|
||||
- Mapfile covering full disc, only NonTrimmed/NonScraped/Unreadable → patch
|
||||
- Mapfile clean → no-op
|
||||
Caller-orchestrated dispatch (the policy `Disc::copy` used to embed):
|
||||
|
||||
- No mapfile → `sweep` (fresh Pass 1).
|
||||
- Mapfile with `?` ranges → `sweep` with `resume: true`.
|
||||
- Mapfile covers full disc, only `*` / `/` / `-` ranges → `patch`.
|
||||
- Mapfile clean → done; no further pass needed.
|
||||
|
||||
Each consumer (autorip, `freemkv` CLI) implements the loop in roughly five
|
||||
lines of `Mapfile::stats()` checks.
|
||||
|
||||
## Algorithm
|
||||
|
||||
### Pass 1 — fast sweep (`Disc::copy` → `sweep_internal`)
|
||||
### Pass 1 — fast sweep (`Disc::sweep`)
|
||||
|
||||
1. Read one ECC block (32 sectors for UHD, 16 for BD/DVD) at the current LBA.
|
||||
2. On success: write data to ISO, mark `+`, advance.
|
||||
@@ -95,7 +116,7 @@ Dispatch logic:
|
||||
|
||||
Pass 1 completes when every byte has been visited (either `+` or `*`).
|
||||
|
||||
### Pass 2+ — patch (`Disc::copy` → `patch_internal`)
|
||||
### Pass 2+ — patch (`Disc::patch`)
|
||||
|
||||
`Disc::patch` reads the mapfile and iterates every non-`+` range. Default: **reverse** mode
|
||||
(walks ranges from highest LBA to lowest, within each range from end to start).
|
||||
@@ -177,4 +198,4 @@ scrape vs. retry with direction reversal) if there's measured benefit.
|
||||
|
||||
- [ddrescue manual, Algorithm chapter](https://www.gnu.org/software/ddrescue/manual/ddrescue_manual.html)
|
||||
- [ddrescue optical media notes](https://www.electric-spoon.com/doc/gddrescue/html/Optical-media.html)
|
||||
- Source: [`src/disc/mapfile.rs`](../src/disc/mapfile.rs), [`src/disc/mod.rs`](../src/disc/mod.rs) (`Disc::copy`, `Disc::patch`), [`src/drive/mod.rs`](../src/drive/mod.rs) (`Drive::read`), [`src/mux/disc.rs`](../src/mux/disc.rs) (`DiscStream::fill_extents`).
|
||||
- Source: [`src/disc/mapfile.rs`](../src/disc/mapfile.rs), [`src/disc/sweep.rs`](../src/disc/sweep.rs) (`Disc::sweep`), [`src/disc/patch.rs`](../src/disc/patch.rs) (`Disc::patch`), [`src/drive/mod.rs`](../src/drive/mod.rs) (`Drive::read`), [`src/mux/disc.rs`](../src/mux/disc.rs) (`DiscStream::fill_extents`).
|
||||
|
||||
+1
-1
@@ -126,7 +126,7 @@ USB optical drives have ~500ms round-trip latency per SCSI command. Since `read_
|
||||
|
||||
`Disc::scan()` wraps the drive in a `BufferedSectorReader` before reading. On a single-sector read, the buffer prefetches a batch of sectors (sized from the kernel's `max_hw_sectors_kb` for the device) and caches them. Subsequent reads to nearby LBAs return from cache with zero SCSI overhead. After parsing the UDF directory structure, the entire metadata partition is pre-read into the cache, so all ICB lookups during title scanning and encryption resolution are instant.
|
||||
|
||||
The buffer is transparent -- `read_filesystem()`, `read_file()`, and all downstream code still call `read_sectors(lba, 1, buf)` as before. The batching happens inside the `SectorReader` implementation.
|
||||
The buffer is transparent -- `read_filesystem()`, `read_file()`, and all downstream code still call `read_sectors(lba, 1, buf)` as before. The batching happens inside the `SectorSource` implementation.
|
||||
|
||||
### UDF Filename Encoding
|
||||
|
||||
|
||||
+2464
-206
File diff suppressed because it is too large
Load Diff
+50
-24
@@ -69,16 +69,29 @@ pub(crate) fn aes_cbc_decrypt(key: &[u8; 16], data: &mut [u8]) {
|
||||
|
||||
// ── Content decryption ──────────────────────────────────────────────────────
|
||||
|
||||
/// Check if a 6144-byte aligned unit is encrypted (copy_permission_indicator bits).
|
||||
pub fn is_unit_encrypted(unit: &[u8]) -> bool {
|
||||
unit.len() >= ALIGNED_UNIT_LEN && (unit[0] & 0xC0) != 0
|
||||
/// True if a 6144-byte aligned unit is AACS-scrambled on disc.
|
||||
///
|
||||
/// AACS encrypts the unit body, which destroys the MPEG-TS sync bytes (`0x47`)
|
||||
/// a clear unit carries at offsets 4, 196, 388, … (one per 192-byte source
|
||||
/// packet). So "scrambled" = "the TS syncs are NOT intact". This is
|
||||
/// flag-independent: it does NOT read the TP_extra copy-control bits (byte 0)
|
||||
/// or the TS scrambling-control bits (byte 7) — AACS sets neither reliably
|
||||
/// across discs/players.
|
||||
///
|
||||
/// This is the single shared definition of "encrypted" for the whole ecosystem
|
||||
/// — libfreemkv's decrypt gate, autorip's sample selection, and the online key
|
||||
/// service's validation gate all call THIS, so they always agree on what is
|
||||
/// encrypted. A correctly-decrypted (or natively-clear) unit reports `false`,
|
||||
/// so the decrypt path never double-decrypts and there is no flag to clear.
|
||||
pub fn is_aacs_scrambled(unit: &[u8]) -> bool {
|
||||
unit.len() >= ALIGNED_UNIT_LEN && !ts_syncs_intact(unit)
|
||||
}
|
||||
|
||||
/// Verify decrypted unit by checking TS sync bytes at expected offsets.
|
||||
fn verify_ts(unit: &[u8]) -> bool {
|
||||
// In a 6144-byte unit, TS packets start at byte 0 with 4-byte TP_extra_header
|
||||
// then 188-byte TS packet, repeating every 192 bytes.
|
||||
// Sync byte 0x47 should appear at offset 4, 196, 388, ...
|
||||
/// Most TS packet positions in `unit` carry the `0x47` sync byte — i.e. the
|
||||
/// unit looks like clear MPEG-TS. Syncs sit at offset 4 and every 192 bytes
|
||||
/// after (4-byte TP_extra_header + 188-byte TS packet). An encrypted body
|
||||
/// scrambles all but the first (which lives in the clear 16-byte seed).
|
||||
fn ts_syncs_intact(unit: &[u8]) -> bool {
|
||||
let mut count = 0;
|
||||
let mut offset = 4;
|
||||
while offset < unit.len() {
|
||||
@@ -87,11 +100,20 @@ fn verify_ts(unit: &[u8]) -> bool {
|
||||
}
|
||||
offset += TS_PACKET_LEN;
|
||||
}
|
||||
// Expect at least most packets to have sync bytes
|
||||
let total = (unit.len() - 4) / TS_PACKET_LEN + 1;
|
||||
// One sync byte is checked per 192-byte BD-TS packet (at offset 4 of
|
||||
// each). `total` is exactly that packet count; the old
|
||||
// `(len - 4) / TS_PACKET_LEN + 1` over-counted by one for lengths of
|
||||
// the form `4 + k·192` (harmless for the always-6144 aligned unit, but
|
||||
// wrong in general and it biased the majority threshold).
|
||||
let total = unit.len() / TS_PACKET_LEN;
|
||||
count > total / 2
|
||||
}
|
||||
|
||||
/// Verify a decrypted unit looks like clear MPEG-TS (sync bytes intact).
|
||||
fn verify_ts(unit: &[u8]) -> bool {
|
||||
ts_syncs_intact(unit)
|
||||
}
|
||||
|
||||
/// Decrypt one AACS aligned unit (6144 bytes) in-place.
|
||||
/// Returns true if decryption succeeded (verified by TS sync bytes).
|
||||
///
|
||||
@@ -99,12 +121,14 @@ fn verify_ts(unit: &[u8]) -> bool {
|
||||
/// 1. AES-128-ECB encrypt first 16 bytes with unit_key → derived
|
||||
/// 2. XOR derived with original 16 bytes → unit_decrypt_key
|
||||
/// 3. AES-128-CBC decrypt bytes 16..6143 with unit_decrypt_key and AACS IV
|
||||
/// 4. Clear encryption flag bits
|
||||
///
|
||||
/// Decryption restores the TS sync bytes, so the unit reads as clear afterward;
|
||||
/// there is no flag to clear.
|
||||
pub fn decrypt_unit(unit: &mut [u8], unit_key: &[u8; 16]) -> bool {
|
||||
if unit.len() < ALIGNED_UNIT_LEN {
|
||||
return false;
|
||||
}
|
||||
if !is_unit_encrypted(unit) {
|
||||
if !is_aacs_scrambled(unit) {
|
||||
return true; // not encrypted
|
||||
}
|
||||
|
||||
@@ -124,16 +148,13 @@ pub fn decrypt_unit(unit: &mut [u8], unit_key: &[u8; 16]) -> bool {
|
||||
// Step 3: Decrypt bytes 16..6143 with AES-CBC
|
||||
aes_cbc_decrypt(&decrypt_key, &mut unit[16..ALIGNED_UNIT_LEN]);
|
||||
|
||||
// Step 4: Clear encryption flag
|
||||
unit[0] &= !0xC0;
|
||||
|
||||
// Verify
|
||||
// Decryption restored the TS syncs; verify the unit now looks like clear TS.
|
||||
verify_ts(unit)
|
||||
}
|
||||
|
||||
/// Decrypt one aligned unit trying multiple unit keys. Returns the key index that worked.
|
||||
pub fn decrypt_unit_try_keys(unit: &mut [u8], unit_keys: &[[u8; 16]]) -> Option<usize> {
|
||||
if !is_unit_encrypted(unit) {
|
||||
if !is_aacs_scrambled(unit) {
|
||||
return Some(0);
|
||||
}
|
||||
|
||||
@@ -173,7 +194,7 @@ pub fn decrypt_unit_full(
|
||||
unit_key: &[u8; 16],
|
||||
read_data_key: Option<&[u8; 16]>,
|
||||
) -> bool {
|
||||
if !is_unit_encrypted(unit) {
|
||||
if !is_aacs_scrambled(unit) {
|
||||
return true;
|
||||
}
|
||||
if let Some(rdk) = read_data_key {
|
||||
@@ -200,10 +221,15 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_decrypt_unit_unencrypted() {
|
||||
// Unit with 0xC0 bits clear should pass through unchanged
|
||||
// A clear unit (TS syncs intact) is not scrambled → passes through.
|
||||
let mut unit = vec![0u8; ALIGNED_UNIT_LEN];
|
||||
unit[0] = 0x00; // not encrypted
|
||||
let mut off = 4;
|
||||
while off < ALIGNED_UNIT_LEN {
|
||||
unit[off] = TS_SYNC;
|
||||
off += TS_PACKET_LEN;
|
||||
}
|
||||
let key = [0u8; 16];
|
||||
assert!(!is_aacs_scrambled(&unit));
|
||||
assert!(decrypt_unit(&mut unit, &key));
|
||||
}
|
||||
|
||||
@@ -253,8 +279,8 @@ mod tests {
|
||||
plain[offset] = TS_SYNC;
|
||||
offset += TS_PACKET_LEN;
|
||||
}
|
||||
// Set encryption flag
|
||||
plain[0] |= 0xC0;
|
||||
// No flag set: CBC-encrypting the body below scrambles packets 1..31's
|
||||
// TS syncs, which is exactly what `is_aacs_scrambled` (raw-sync) detects.
|
||||
|
||||
// Now encrypt bytes 16..6143 using the AACS algorithm (reverse of decrypt)
|
||||
let header: [u8; 16] = plain[..16].try_into().unwrap();
|
||||
@@ -281,9 +307,9 @@ mod tests {
|
||||
|
||||
// Now plain contains encrypted data. Decrypt it.
|
||||
let mut unit = plain;
|
||||
assert!(is_unit_encrypted(&unit));
|
||||
assert!(is_aacs_scrambled(&unit));
|
||||
assert!(decrypt_unit(&mut unit, &unit_key));
|
||||
assert!(!is_unit_encrypted(&unit)); // flag should be cleared
|
||||
assert!(!is_aacs_scrambled(&unit)); // decrypted: TS syncs restored
|
||||
|
||||
// Verify TS sync bytes
|
||||
let mut count = 0;
|
||||
|
||||
+89
-2
@@ -1235,8 +1235,8 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_aes_cmac() {
|
||||
// Basic CMAC test — at minimum verify it produces consistent output
|
||||
fn test_aes_cmac_deterministic() {
|
||||
// Same (data, key) must always produce the same MAC.
|
||||
let key = [
|
||||
0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf,
|
||||
0x4f, 0x3c,
|
||||
@@ -1248,6 +1248,93 @@ mod tests {
|
||||
assert_ne!(mac1, [0u8; 16]); // shouldn't be all zeros
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_aes_cmac_nist_kat_full_block() {
|
||||
// NIST SP 800-38B Appendix D.1, Example 2 (Mlen = 128):
|
||||
// K = 2b7e1516 28aed2a6 abf71588 09cf4f3c
|
||||
// M = 6bc1bee2 2e409f96 e93d7e11 7393172a
|
||||
// T = 070a16b4 6b4d4144 f79bdd9d d04a287c
|
||||
let key = [
|
||||
0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf,
|
||||
0x4f, 0x3c,
|
||||
];
|
||||
let data = [
|
||||
0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93,
|
||||
0x17, 0x2a,
|
||||
];
|
||||
let expected = [
|
||||
0x07, 0x0a, 0x16, 0xb4, 0x6b, 0x4d, 0x41, 0x44, 0xf7, 0x9b, 0xdd, 0x9d, 0xd0, 0x4a,
|
||||
0x28, 0x7c,
|
||||
];
|
||||
let mac = aes_cmac_16(&data, &key);
|
||||
assert_eq!(mac, expected, "AES-CMAC-128 must match NIST SP 800-38B KAT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_vid_mac_verify_roundtrip() {
|
||||
// Simulate the drive-side: pick a (bus_key, vid), compute the MAC, and
|
||||
// verify the host-side check accepts it. Then mutate VID and MAC each
|
||||
// in turn and verify both mutations cause a mismatch (the path that
|
||||
// would yield Error::AacsVidMac in read_volume_id).
|
||||
let bus_key = [
|
||||
0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54,
|
||||
0x32, 0x10,
|
||||
];
|
||||
let vid = [
|
||||
0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe, 0xba, 0xbe, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66,
|
||||
0x77, 0x88,
|
||||
];
|
||||
|
||||
// Drive returns vid + mac where mac == AES-CMAC-128(bus_key, vid).
|
||||
let drive_mac = aes_cmac_16(&vid, &bus_key);
|
||||
let calc_mac = aes_cmac_16(&vid, &bus_key);
|
||||
assert_eq!(calc_mac, drive_mac, "honest drive: MACs must match");
|
||||
|
||||
// Mutate the MAC: a malicious drive that swapped VID but returned its
|
||||
// original MAC would produce a mismatch here.
|
||||
let mut bad_mac = drive_mac;
|
||||
bad_mac[0] ^= 0x01;
|
||||
assert_ne!(calc_mac, bad_mac, "mutated MAC must be rejected");
|
||||
|
||||
// Mutate the VID: even one bit of VID drift produces a wildly different
|
||||
// CMAC (this is what catches a substituted VID with a stale MAC).
|
||||
let mut bad_vid = vid;
|
||||
bad_vid[15] ^= 0x01;
|
||||
let calc_for_bad_vid = aes_cmac_16(&bad_vid, &bus_key);
|
||||
assert_ne!(
|
||||
calc_for_bad_vid, drive_mac,
|
||||
"MAC over mutated VID must not match original MAC"
|
||||
);
|
||||
|
||||
// Wrong bus key (e.g. handshake replayed against the wrong session)
|
||||
// also produces a different MAC over the same VID.
|
||||
let mut wrong_key = bus_key;
|
||||
wrong_key[0] ^= 0xff;
|
||||
let calc_with_wrong_key = aes_cmac_16(&vid, &wrong_key);
|
||||
assert_ne!(
|
||||
calc_with_wrong_key, drive_mac,
|
||||
"MAC under wrong bus key must not match"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_vid_mac_all_zero_mac_rejected() {
|
||||
// Defensive: a buggy or hostile drive that returns all-zero MAC must
|
||||
// be rejected (the real MAC over any non-trivial VID is nearly never
|
||||
// 0...0). This guards against a class of "drive returned garbage"
|
||||
// failures masquerading as success.
|
||||
let bus_key = [
|
||||
0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf,
|
||||
0x4f, 0x3c,
|
||||
];
|
||||
let vid = [
|
||||
0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93,
|
||||
0x17, 0x2a,
|
||||
];
|
||||
let calc_mac = aes_cmac_16(&vid, &bus_key);
|
||||
assert_ne!(calc_mac, [0u8; 16], "real CMAC must not be all zeros");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_host_cert_from_keydb() {
|
||||
// Verify the host cert from our KEYDB
|
||||
|
||||
+162
-17
@@ -89,6 +89,17 @@ pub(crate) fn parse_hex20(s: &str) -> Option<[u8; 20]> {
|
||||
}
|
||||
|
||||
impl KeyDb {
|
||||
/// Construct an empty KeyDb. Used by unit tests; production code
|
||||
/// reaches a populated KeyDb via [`KeyDb::load`] or [`KeyDb::parse`].
|
||||
pub fn empty() -> Self {
|
||||
KeyDb {
|
||||
device_keys: Vec::new(),
|
||||
processing_keys: Vec::new(),
|
||||
host_certs: Vec::new(),
|
||||
disc_entries: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a KEYDB.cfg file from a string.
|
||||
pub fn parse(data: &str) -> Self {
|
||||
let mut db = KeyDb {
|
||||
@@ -106,10 +117,19 @@ impl KeyDb {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Device Key
|
||||
// Device Key.
|
||||
// Two shapes are accepted:
|
||||
// 1. Positioned DK: `| DK | DEVICE_KEY 0x... | DEVICE_NODE 0x... | KEY_UV 0x... | KEY_U_MASK_SHIFT 0x...`
|
||||
// → loaded into `device_keys` (deterministic tree walk via `calc_pk_from_dk`).
|
||||
// 2. Orphan DK: `| DK | DEVICE_KEY 0x...` with no position fields.
|
||||
// → loaded into `processing_keys` (brute walker / terminal validation).
|
||||
// Per AACS spec a "PK" IS a DK at terminal position, so both row types
|
||||
// are DKs in the unified model; only the metadata differs.
|
||||
if line.starts_with("| DK") {
|
||||
if let Some(dk) = Self::parse_device_key(line) {
|
||||
db.device_keys.push(dk);
|
||||
} else if let Some(key) = Self::parse_orphan_dk(line) {
|
||||
db.processing_keys.push(key);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -152,7 +172,7 @@ impl KeyDb {
|
||||
db
|
||||
}
|
||||
|
||||
/// Load KEYDB.cfg from a file path.
|
||||
/// Load a KEYDB.cfg from disk.
|
||||
pub fn load(path: &std::path::Path) -> std::io::Result<Self> {
|
||||
let data = std::fs::read_to_string(path)?;
|
||||
Ok(Self::parse(&data))
|
||||
@@ -184,8 +204,53 @@ impl KeyDb {
|
||||
.or_else(|| self.disc_entries.get(&hash))
|
||||
}
|
||||
|
||||
// ── Parsers ─────────────────────────────────────────────────────────────
|
||||
/// Iterate every disc entry. Used by Path 3 (scan for matching VID).
|
||||
pub fn iter_disc_entries(&self) -> impl Iterator<Item = &DiscEntry> {
|
||||
self.disc_entries.values()
|
||||
}
|
||||
}
|
||||
|
||||
// ── KeyProvider impl ──────────────────────────────────────────────────────────
|
||||
//
|
||||
// Lets `KeyDb` plug into `resolve_keys` via the trait. Cloning happens in the
|
||||
// bulk methods because the trait returns owned `Vec`s (so HTTP-backed providers
|
||||
// don't need to retain state across calls).
|
||||
|
||||
impl super::provider::KeyProvider for KeyDb {
|
||||
fn device_keys(&self) -> Vec<DeviceKey> {
|
||||
self.device_keys.clone()
|
||||
}
|
||||
fn processing_keys(&self) -> Vec<[u8; 16]> {
|
||||
self.processing_keys.clone()
|
||||
}
|
||||
fn media_keys(&self) -> Vec<[u8; 16]> {
|
||||
// Every per-disc Media Key in the db. The resolver dedups; MKs are
|
||||
// MKB-scoped so the same value recurs across a pressing's discs.
|
||||
self.iter_disc_entries()
|
||||
.filter_map(|e| e.media_key)
|
||||
.collect()
|
||||
}
|
||||
fn host_certs(&self) -> Vec<HostCert> {
|
||||
self.host_certs.clone()
|
||||
}
|
||||
fn lookup_disc_by_hash(&self, disc_hash: &[u8; 20]) -> Option<DiscEntry> {
|
||||
let mut hex = String::with_capacity(42);
|
||||
hex.push_str("0x");
|
||||
for b in disc_hash {
|
||||
hex.push_str(&format!("{b:02X}"));
|
||||
}
|
||||
self.find_disc(&hex).cloned()
|
||||
}
|
||||
fn lookup_disc_by_vid(&self, volume_id: &[u8; 16]) -> Option<DiscEntry> {
|
||||
self.iter_disc_entries()
|
||||
.find(|e| matches!(e.disc_id, Some(id) if &id == volume_id))
|
||||
.cloned()
|
||||
}
|
||||
}
|
||||
|
||||
// ── Private parsers (re-open the inherent impl) ─────────────────────────────
|
||||
|
||||
impl KeyDb {
|
||||
fn parse_device_key(line: &str) -> Option<DeviceKey> {
|
||||
// | DK | DEVICE_KEY 0x... | DEVICE_NODE 0x... | KEY_UV 0x... | KEY_U_MASK_SHIFT 0x...
|
||||
let key_str = line.split("DEVICE_KEY").nth(1)?.split('|').next()?.trim();
|
||||
@@ -218,6 +283,30 @@ impl KeyDb {
|
||||
None
|
||||
}
|
||||
|
||||
/// Parse an orphan DK row: a `| DK |` line carrying only the
|
||||
/// `DEVICE_KEY` field (no position metadata). The key is then
|
||||
/// treated like a terminal/unpositioned label by the resolver
|
||||
/// (Path 2's brute walker). Returns `None` if the line carries
|
||||
/// any position field — those are positioned DKs and parsed by
|
||||
/// [`Self::parse_device_key`] instead.
|
||||
fn parse_orphan_dk(line: &str) -> Option<[u8; 16]> {
|
||||
if line.contains("DEVICE_NODE")
|
||||
|| line.contains("KEY_UV")
|
||||
|| line.contains("KEY_U_MASK_SHIFT")
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let key_str = line
|
||||
.split("DEVICE_KEY")
|
||||
.nth(1)?
|
||||
.split('|')
|
||||
.next()?
|
||||
.split(';')
|
||||
.next()?
|
||||
.trim();
|
||||
parse_hex16(key_str)
|
||||
}
|
||||
|
||||
fn parse_host_cert(line: &str) -> Option<HostCert> {
|
||||
// | HC | HOST_PRIV_KEY 0x... | HOST_CERT 0x...
|
||||
let priv_str = line
|
||||
@@ -366,28 +455,84 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_parse_disc_entry() {
|
||||
let line = r#"***REMOVED*** = DUNE_PART_TWO (Dune: Part Two) | D | 2024-04-02 | M | ***REMOVED*** | I | ***REMOVED*** | V | ***REMOVED*** | U | 1-***REMOVED*** ; MKBv77"#;
|
||||
let entry = KeyDb::parse_disc_entry(line).unwrap();
|
||||
assert_eq!(entry.title, "Dune: Part Two");
|
||||
// All-zero placeholders — synthetic; no real key material in code.
|
||||
let z40 = "00".repeat(20);
|
||||
let z32 = "00".repeat(16);
|
||||
let line = format!(
|
||||
"0x{z40} = SAMPLE_FILM (Sample Film) | D | 2024-01-01 | M | 0x{z32} | I | 0x{z32} | V | 0x{z32} | U | 1-0x{z32} ; MKBv77"
|
||||
);
|
||||
let entry = KeyDb::parse_disc_entry(&line).unwrap();
|
||||
assert_eq!(entry.title, "Sample Film");
|
||||
assert!(entry.media_key.is_some());
|
||||
assert!(entry.vuk.is_some());
|
||||
assert_eq!(entry.unit_keys.len(), 1);
|
||||
assert_eq!(entry.unit_keys[0].0, 1);
|
||||
}
|
||||
|
||||
// NOTE: key fields below use obvious repeated-byte / zero placeholders
|
||||
// (0x01.., 0x02.., 0x03.., 0x00..). NEVER put real — or real-looking — host,
|
||||
// device, or processing key material in code; these tests exercise the
|
||||
// parser's field-splitting only, not any genuine key.
|
||||
|
||||
#[test]
|
||||
fn test_parse_device_key() {
|
||||
let line = "| DK | DEVICE_KEY ***REMOVED*** | DEVICE_NODE 0x0800 | KEY_UV 0x00000400 | KEY_U_MASK_SHIFT 0x17 ; MKBv01-MKBv48";
|
||||
let line = "| DK | DEVICE_KEY 0x00000000000000000000000000000000 | DEVICE_NODE 0x0800 | KEY_UV 0x00000400 | KEY_U_MASK_SHIFT 0x17 ; MKBv01-MKBv48";
|
||||
let dk = KeyDb::parse_device_key(line).unwrap();
|
||||
assert_eq!(dk.node, 0x0800);
|
||||
assert_eq!(dk.u_mask_shift, 0x17);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_orphan_dk_row_loads_into_processing_keys() {
|
||||
// `| DK |` row without position fields = an orphan DK. Per the
|
||||
// unified model the resolver treats it like a terminal/PK
|
||||
// candidate: it lands in `processing_keys` and the brute walker
|
||||
// handles it.
|
||||
let cfg = r#"
|
||||
| DK | DEVICE_KEY 0x01010101010101010101010101010101 ; orphan, no position fields
|
||||
| DK | DEVICE_KEY 0x02020202020202020202020202020202 | DEVICE_NODE 0x0800 | KEY_UV 0x00000400 | KEY_U_MASK_SHIFT 0x17 ; positioned MKBv01-MKBv48
|
||||
| PK | 0x03030303030303030303030303030303 ; legacy PK row still works
|
||||
"#;
|
||||
let db = KeyDb::parse(cfg);
|
||||
assert_eq!(
|
||||
db.device_keys.len(),
|
||||
1,
|
||||
"positioned DK row should land in device_keys"
|
||||
);
|
||||
// Orphan DK + legacy PK row both end up in processing_keys.
|
||||
assert_eq!(
|
||||
db.processing_keys.len(),
|
||||
2,
|
||||
"orphan DK row + legacy PK row both belong in processing_keys"
|
||||
);
|
||||
assert_eq!(db.processing_keys[0][..4], [0x01, 0x01, 0x01, 0x01]);
|
||||
assert_eq!(db.processing_keys[1][..4], [0x03, 0x03, 0x03, 0x03]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_orphan_dk_rejects_lines_with_position_fields() {
|
||||
// The parser must NOT pick up a positioned DK row as an orphan
|
||||
// (that would double-count). parse_orphan_dk explicitly checks.
|
||||
let positioned = "| DK | DEVICE_KEY 0x02020202020202020202020202020202 | DEVICE_NODE 0x0800 | KEY_UV 0x00000400 | KEY_U_MASK_SHIFT 0x17";
|
||||
assert!(
|
||||
KeyDb::parse_orphan_dk(positioned).is_none(),
|
||||
"positioned DK must not match orphan parser"
|
||||
);
|
||||
let orphan = "| DK | DEVICE_KEY 0x01010101010101010101010101010101";
|
||||
let key = KeyDb::parse_orphan_dk(orphan).expect("orphan should parse");
|
||||
assert_eq!(key[..4], [0x01, 0x01, 0x01, 0x01]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_host_cert() {
|
||||
let line = "| HC | HOST_PRIV_KEY ***REMOVED*** | HOST_CERT ***REMOVED*** ; Revoked";
|
||||
let hc = KeyDb::parse_host_cert(line).unwrap();
|
||||
assert_eq!(hc.private_key[0], 0x90);
|
||||
// 20-byte priv + 92-byte cert, all zeros — placeholders, not a key.
|
||||
let line = format!(
|
||||
"| HC | HOST_PRIV_KEY 0x{} | HOST_CERT 0x{} ; Revoked",
|
||||
"00".repeat(20),
|
||||
"00".repeat(92)
|
||||
);
|
||||
let hc = KeyDb::parse_host_cert(&line).unwrap();
|
||||
assert_eq!(hc.private_key, [0u8; 20]);
|
||||
assert_eq!(hc.certificate.len(), 92);
|
||||
}
|
||||
|
||||
@@ -405,15 +550,15 @@ mod tests {
|
||||
assert!(!db.host_certs.is_empty());
|
||||
assert!(db.disc_entries.len() > 170000);
|
||||
|
||||
// Look up Dune: Part Two
|
||||
let dune = db
|
||||
// Look up any disc entry carrying a full key set.
|
||||
let entry = db
|
||||
.disc_entries
|
||||
.values()
|
||||
.find(|e| e.title.contains("Dune: Part Two") && e.vuk.is_some())
|
||||
.expect("Dune: Part Two not found");
|
||||
assert!(dune.media_key.is_some());
|
||||
assert!(dune.vuk.is_some());
|
||||
assert!(!dune.unit_keys.is_empty());
|
||||
.find(|e| e.vuk.is_some() && e.media_key.is_some() && !e.unit_keys.is_empty())
|
||||
.expect("no disc entry with a full key set");
|
||||
assert!(entry.media_key.is_some());
|
||||
assert!(entry.vuk.is_some());
|
||||
assert!(!entry.unit_keys.is_empty());
|
||||
|
||||
eprintln!(
|
||||
"Parsed {} disc entries, {} DK, {} PK",
|
||||
|
||||
+1269
-159
File diff suppressed because it is too large
Load Diff
+14
-4
@@ -17,16 +17,26 @@ pub mod decrypt;
|
||||
pub mod handshake;
|
||||
pub mod keydb;
|
||||
pub mod keys;
|
||||
pub mod provider;
|
||||
pub mod variants;
|
||||
|
||||
// Explicit re-exports — only items needed by external consumers and sibling crate modules.
|
||||
// AES primitives (aes_ecb_encrypt, aes_ecb_decrypt, aes_cbc_decrypt) are pub(crate) in decrypt.rs.
|
||||
pub use decrypt::{
|
||||
ALIGNED_UNIT_LEN, decrypt_bus, decrypt_unit, decrypt_unit_full, decrypt_unit_try_keys,
|
||||
is_unit_encrypted,
|
||||
is_aacs_scrambled,
|
||||
};
|
||||
pub use keydb::{DeviceKey, DiscEntry, HostCert, KeyDb};
|
||||
pub use keys::probe;
|
||||
pub use keys::{
|
||||
ContentCert, ResolvedKeys, UnitKeyFile, decrypt_unit_key, derive_media_key_from_dk,
|
||||
derive_media_key_from_pk, derive_vuk, disc_hash, disc_hash_hex, mkb_version,
|
||||
parse_content_cert, parse_unit_key_ro, read_mkb_from_drive, resolve_keys,
|
||||
AacsVersion, ContentCert, ResolveContext, ResolvedKeys, UnitKeyFile, decrypt_unit_key,
|
||||
derive_media_key_from_dk, derive_media_key_from_pk, derive_vuk, disc_hash, disc_hash_hex,
|
||||
mkb_content_len, mkb_version, parse_content_cert, parse_unit_key_ro, read_mkb_from_drive,
|
||||
resolve_keys_v1, resolve_keys_v2, resolve_keys_v21,
|
||||
};
|
||||
pub use provider::KeyProvider;
|
||||
pub use variants::{
|
||||
KEY_CORRECTION_DATA_PLACEHOLDER, MediaKeyVariantError, MkbRecord, ProcessingKeyMatch,
|
||||
derive_media_key_variant, is_variant_mkb, variant_data_record, variant_key_data, variant_nonce,
|
||||
walk_mkb, walk_processing_key,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
//! Key source abstraction for the AACS resolve chain.
|
||||
//!
|
||||
//! libfreemkv keeps all crypto (AES-G primitives, SD-tree walking,
|
||||
//! validation, MK/VUK/TK derivation) but accepts key material from
|
||||
//! arbitrary backends via [`KeyProvider`].
|
||||
//!
|
||||
//! Methods come in two flavors:
|
||||
//!
|
||||
//! - **Bulk material** ([`device_keys`], [`processing_keys`],
|
||||
//! [`host_certs`]) — the resolver unions results across all
|
||||
//! providers and tries each candidate.
|
||||
//! - **Disc-keyed lookup** ([`lookup_disc_by_hash`],
|
||||
//! [`lookup_disc_by_vid`]) — the resolver short-circuits on the
|
||||
//! first hit, so providers are queried in array order with
|
||||
//! fastest/closest first.
|
||||
//!
|
||||
//! Default impls return empty / `None` so backends only override
|
||||
//! the methods they actually support — an external key service might
|
||||
//! implement only `lookup_disc_by_hash`, while a local file might
|
||||
//! implement all five.
|
||||
//!
|
||||
//! Calls may block (disk I/O, network round-trips). The resolver
|
||||
//! invokes each method at most a handful of times per scan; for
|
||||
//! per-disc memoization, implementations should cache internally.
|
||||
//!
|
||||
//! [`device_keys`]: KeyProvider::device_keys
|
||||
//! [`processing_keys`]: KeyProvider::processing_keys
|
||||
//! [`host_certs`]: KeyProvider::host_certs
|
||||
//! [`lookup_disc_by_hash`]: KeyProvider::lookup_disc_by_hash
|
||||
//! [`lookup_disc_by_vid`]: KeyProvider::lookup_disc_by_vid
|
||||
|
||||
use super::keydb::{DeviceKey, DiscEntry, HostCert};
|
||||
|
||||
/// Source of AACS key material.
|
||||
///
|
||||
/// Implementors return raw material only — the resolver in
|
||||
/// `aacs::keys` owns all the crypto (DK→PK walking, PK validation,
|
||||
/// MK→VUK→TK derivation). See module docs for method semantics.
|
||||
pub trait KeyProvider: Send + Sync {
|
||||
/// Device keys (top-of-tree, walked by the resolver).
|
||||
fn device_keys(&self) -> Vec<DeviceKey> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// Processing keys — terminal PKs or walk-input PKs. The
|
||||
/// resolver tries each as a terminal first (cheap validate).
|
||||
fn processing_keys(&self) -> Vec<[u8; 16]> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// Every Media Key this provider holds, regardless of which disc it was
|
||||
/// filed under. An MK is MKB-scoped (shared across a pressing/MKB-family),
|
||||
/// so the resolver can verify each against the disc's MKB (`km_verifies`)
|
||||
/// and resolve a disc whose own hash/VID isn't directly keyed.
|
||||
fn media_keys(&self) -> Vec<[u8; 16]> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// AACS host certificates (with their private keys) for drive
|
||||
/// authentication. Multiple in case some are revoked.
|
||||
fn host_certs(&self) -> Vec<HostCert> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// Direct per-disc lookup by SHA-1 of `Unit_Key_RO.inf`. Returns
|
||||
/// `Some(entry)` if this provider has pre-computed material for
|
||||
/// the disc (paths 4 and 5). Short-circuits the resolver.
|
||||
fn lookup_disc_by_hash(&self, _disc_hash: &[u8; 20]) -> Option<DiscEntry> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Lookup by Volume ID (path 3 — pre-computed MK + matching
|
||||
/// VID). Short-circuits the resolver on hit.
|
||||
fn lookup_disc_by_vid(&self, _volume_id: &[u8; 16]) -> Option<DiscEntry> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolver-side helpers that aggregate across a provider array.
|
||||
///
|
||||
/// The resolver consumes `&[&dyn KeyProvider]` directly; these
|
||||
/// helpers wrap the union-vs-short-circuit policy per method.
|
||||
pub(crate) struct Providers<'a>(pub &'a [&'a dyn KeyProvider]);
|
||||
|
||||
impl Providers<'_> {
|
||||
/// Union — gather DKs from every provider.
|
||||
pub fn device_keys(&self) -> Vec<DeviceKey> {
|
||||
self.0.iter().flat_map(|p| p.device_keys()).collect()
|
||||
}
|
||||
|
||||
/// Union — gather PKs from every provider.
|
||||
pub fn processing_keys(&self) -> Vec<[u8; 16]> {
|
||||
self.0.iter().flat_map(|p| p.processing_keys()).collect()
|
||||
}
|
||||
|
||||
/// Union of distinct Media Keys across every provider, for the MK-pool
|
||||
/// brute (`km_verifies` against the disc's MKB).
|
||||
pub fn media_keys(&self) -> Vec<[u8; 16]> {
|
||||
let mut v: Vec<[u8; 16]> = self.0.iter().flat_map(|p| p.media_keys()).collect();
|
||||
v.sort_unstable();
|
||||
v.dedup();
|
||||
v
|
||||
}
|
||||
|
||||
/// Union — gather host certs from every provider. Not yet wired into
|
||||
/// the SCSI handshake (which still reads `KeyDb.host_certs` directly);
|
||||
/// kept here so a provider-aware handshake refactor is a drop-in.
|
||||
#[allow(dead_code)]
|
||||
pub fn host_certs(&self) -> Vec<HostCert> {
|
||||
self.0.iter().flat_map(|p| p.host_certs()).collect()
|
||||
}
|
||||
|
||||
/// Short-circuit — query providers in array order, first hit wins.
|
||||
pub fn lookup_disc_by_hash(&self, disc_hash: &[u8; 20]) -> Option<DiscEntry> {
|
||||
self.0.iter().find_map(|p| p.lookup_disc_by_hash(disc_hash))
|
||||
}
|
||||
|
||||
/// Short-circuit — query providers in array order, first hit wins.
|
||||
pub fn lookup_disc_by_vid(&self, volume_id: &[u8; 16]) -> Option<DiscEntry> {
|
||||
self.0.iter().find_map(|p| p.lookup_disc_by_vid(volume_id))
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`KeyProvider`] backed by a single caller-supplied key's raw material —
|
||||
/// the bridge for [`crate::disc::Disc::decrypt_with`].
|
||||
///
|
||||
/// The application's key source did the lookup and handed in material at one
|
||||
/// level (DK / PK / MK / VUK). This exposes exactly that material to the
|
||||
/// version-dispatched resolver, which owns ALL derivation — so a source never
|
||||
/// derives, and the lib remains the single home for the AACS chain across
|
||||
/// 1.0 / 2.0 / 2.1 / 2.x.
|
||||
///
|
||||
/// Each level fills only its own field; the rest stay empty, so the resolver
|
||||
/// naturally runs the matching path (DK→…, PK→…, MK-pool brute, or a
|
||||
/// disc-keyed VUK hit). `decrypt_with` already knows the disc, so the
|
||||
/// `lookup_disc_by_*` hash/VID arguments are irrelevant — a present
|
||||
/// `disc_entry` is returned for any query.
|
||||
pub(crate) struct SuppliedKey {
|
||||
pub device_keys: Vec<DeviceKey>,
|
||||
pub processing_keys: Vec<[u8; 16]>,
|
||||
pub media_keys: Vec<[u8; 16]>,
|
||||
pub disc_entry: Option<DiscEntry>,
|
||||
}
|
||||
|
||||
impl KeyProvider for SuppliedKey {
|
||||
fn device_keys(&self) -> Vec<DeviceKey> {
|
||||
self.device_keys.clone()
|
||||
}
|
||||
fn processing_keys(&self) -> Vec<[u8; 16]> {
|
||||
self.processing_keys.clone()
|
||||
}
|
||||
fn media_keys(&self) -> Vec<[u8; 16]> {
|
||||
self.media_keys.clone()
|
||||
}
|
||||
fn lookup_disc_by_hash(&self, _disc_hash: &[u8; 20]) -> Option<DiscEntry> {
|
||||
self.disc_entry.clone()
|
||||
}
|
||||
fn lookup_disc_by_vid(&self, _volume_id: &[u8; 16]) -> Option<DiscEntry> {
|
||||
self.disc_entry.clone()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,679 @@
|
||||
//! AACS Media Key Variant chain.
|
||||
//!
|
||||
//! On AACS 2.1 the Media Key derivation gains a second stage on top of
|
||||
//! the classical subset-difference walk. The classical walk yields a
|
||||
//! Media Key Precursor (Kmp) rather than the final Media Key; the
|
||||
//! Precursor combines with disc-supplied Variant Key Data (VKD) and an
|
||||
//! integrator-supplied Key Correction Data (KCD) constant to produce
|
||||
//! the Media Key.
|
||||
//!
|
||||
//! This module is wiring only — `resolve_keys` is not aware of it. The
|
||||
//! entry point is [`derive_media_key_variant`]. The Variant scheme is
|
||||
//! detected via the new MKB record types `0x82` (Encrypted Media Key
|
||||
//! Variant Data + Variant Key Data) and `0x83` (Variant Number). When
|
||||
//! a disc carries neither, callers should fall back to the classical
|
||||
//! single-stage derivation in [`super::keys`].
|
||||
//!
|
||||
//! The chain follows the published spec:
|
||||
//!
|
||||
//! ```text
|
||||
//! Kmp = AES-128D(Kp, C) XOR uv
|
||||
//! Kpnew = Kmp XOR KCD
|
||||
//! Kvn = AES-G(Kp, Nonce) & 0xFFFF (low 16 bits, BE)
|
||||
//! VKD_idx = Kvn XOR VARIANTS[uv]
|
||||
//! VKD = vkd_table[VKD_idx * 16 .. +16]
|
||||
//! Km = AES-128D(Kpnew, VKD) XOR uv
|
||||
//! ```
|
||||
//!
|
||||
//! Two condition bits on `Kmp[15]` route off the hardcoded-KCD path
|
||||
//! (Soft Correction and Online Challenge). The chain refuses to run in
|
||||
//! either case — callers must handle those modes out of band.
|
||||
|
||||
use super::decrypt::aes_ecb_decrypt;
|
||||
use super::keydb::DeviceKey;
|
||||
|
||||
// ── Public constants ──────────────────────────────────────────────────────
|
||||
|
||||
/// Placeholder Key Correction Data. Sixteen zero bytes.
|
||||
///
|
||||
/// Integrators MUST supply a non-placeholder KCD via the `kcd` argument
|
||||
/// to [`derive_media_key_variant`]; the chain refuses to operate when
|
||||
/// the supplied KCD compares equal to this placeholder.
|
||||
pub const KEY_CORRECTION_DATA_PLACEHOLDER: [u8; 16] = [0u8; 16];
|
||||
|
||||
// ── MKB record walking ────────────────────────────────────────────────────
|
||||
|
||||
/// A single MKB record produced by [`walk_mkb`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MkbRecord {
|
||||
/// Byte offset of the record within the MKB.
|
||||
pub offset: usize,
|
||||
/// Record type byte.
|
||||
pub rec_type: u8,
|
||||
/// Record length in bytes (includes the 4-byte header).
|
||||
pub rec_len: usize,
|
||||
/// Record body (the bytes after the 4-byte header).
|
||||
pub body: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Walk an MKB into a flat list of records.
|
||||
///
|
||||
/// MKB record framing per AACS: 1 byte type, 3 bytes BE length
|
||||
/// INCLUDING the 4-byte header, followed by payload. The walker stops
|
||||
/// at the first `(type=0, len=0)` end marker or at end of buffer.
|
||||
pub fn walk_mkb(mkb: &[u8]) -> Vec<MkbRecord> {
|
||||
let mut out = Vec::new();
|
||||
let mut pos = 0;
|
||||
while pos + 4 <= mkb.len() {
|
||||
let rec_type = mkb[pos];
|
||||
let rec_len = ((mkb[pos + 1] as usize) << 16)
|
||||
| ((mkb[pos + 2] as usize) << 8)
|
||||
| (mkb[pos + 3] as usize);
|
||||
if rec_type == 0 && rec_len == 0 {
|
||||
break;
|
||||
}
|
||||
if rec_len < 4 || pos + rec_len > mkb.len() {
|
||||
break;
|
||||
}
|
||||
let body = mkb[pos + 4..pos + rec_len].to_vec();
|
||||
out.push(MkbRecord {
|
||||
offset: pos,
|
||||
rec_type,
|
||||
rec_len,
|
||||
body,
|
||||
});
|
||||
pos += rec_len;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// True iff `records` contains at least one Media Key Variant record
|
||||
/// (type `0x82` or `0x83`).
|
||||
pub fn is_variant_mkb(records: &[MkbRecord]) -> bool {
|
||||
records.iter().any(|r| matches!(r.rec_type, 0x82 | 0x83))
|
||||
}
|
||||
|
||||
/// Body of the Encrypted Media Key Variant Data record (type `0x82`).
|
||||
pub fn variant_data_record(records: &[MkbRecord]) -> Option<&[u8]> {
|
||||
records
|
||||
.iter()
|
||||
.find(|r| r.rec_type == 0x82)
|
||||
.map(|r| r.body.as_slice())
|
||||
}
|
||||
|
||||
/// 16-byte Nonce from the Variant Number record (type `0x83`). Returns
|
||||
/// the first 16 bytes of the body.
|
||||
pub fn variant_nonce(records: &[MkbRecord]) -> Option<[u8; 16]> {
|
||||
let r = records.iter().find(|r| r.rec_type == 0x83)?;
|
||||
if r.body.len() < 16 {
|
||||
return None;
|
||||
}
|
||||
let mut out = [0u8; 16];
|
||||
out.copy_from_slice(&r.body[..16]);
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Body of the Variant Key Data record. Returns the first `0x82` body
|
||||
/// that is a non-empty multiple of 16 bytes.
|
||||
pub fn variant_key_data(records: &[MkbRecord]) -> Option<&[u8]> {
|
||||
records
|
||||
.iter()
|
||||
.find(|r| r.rec_type == 0x82 && !r.body.is_empty() && r.body.len() % 16 == 0)
|
||||
.map(|r| r.body.as_slice())
|
||||
}
|
||||
|
||||
// ── AES-G ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// AES-G(x1, x2) = AES-128D(x1, x2) XOR x2.
|
||||
///
|
||||
/// The Media Key Variant chain uses AES-G to derive both the variant
|
||||
/// number (`Kvn = AES-G(Kp, Nonce)`) and the Volume Unique Key
|
||||
/// (`Kvu = AES-G(Km, VID)`). See [`super::keys::derive_vuk`] for the
|
||||
/// classical VUK form — the math is identical, this exposes it as a
|
||||
/// neutral primitive for the variant chain.
|
||||
fn aes_g(x1: &[u8; 16], x2: &[u8; 16]) -> [u8; 16] {
|
||||
let mut out = aes_ecb_decrypt(x1, x2);
|
||||
for i in 0..16 {
|
||||
out[i] ^= x2[i];
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ── Subset-difference walk that exposes (Kp, uv) ──────────────────────────
|
||||
|
||||
/// AES-G3 seed register initial value.
|
||||
const AESG3_SEED: [u8; 16] = [
|
||||
0x7B, 0x10, 0x3C, 0x5D, 0xCB, 0x08, 0xC4, 0xE5, 0x1A, 0x27, 0xB0, 0x17, 0x99, 0x05, 0x3B, 0xD9,
|
||||
];
|
||||
|
||||
/// AES-G3 single step: AES-G against the seed register at offset `inc`.
|
||||
fn aesg3_step(key: &[u8; 16], inc: u8) -> [u8; 16] {
|
||||
let mut seed = AESG3_SEED;
|
||||
seed[15] = seed[15].wrapping_add(inc);
|
||||
aes_g(key, &seed)
|
||||
}
|
||||
|
||||
fn calc_v_mask(uv: u32) -> u32 {
|
||||
let mut v_mask: u32 = 0xFFFF_FFFF;
|
||||
while (uv & !v_mask) == 0 && v_mask != 0 {
|
||||
v_mask <<= 1;
|
||||
}
|
||||
v_mask
|
||||
}
|
||||
|
||||
fn calc_pk_from_dk(dk: &[u8; 16], uv: u32, v_mask: u32, dev_key_v_mask: u32) -> [u8; 16] {
|
||||
let mut left_child = aesg3_step(dk, 0);
|
||||
let mut pk = aesg3_step(dk, 1);
|
||||
let mut right_child = aesg3_step(dk, 2);
|
||||
let mut current_v_mask = dev_key_v_mask;
|
||||
|
||||
while current_v_mask != v_mask {
|
||||
let mut bit_pos: i32 = -1;
|
||||
for i in (0..32).rev() {
|
||||
if (current_v_mask & (1u32 << i)) == 0 {
|
||||
bit_pos = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let curr_key = if bit_pos < 0 || (uv & (1u32 << bit_pos as u32)) == 0 {
|
||||
left_child
|
||||
} else {
|
||||
right_child
|
||||
};
|
||||
|
||||
left_child = aesg3_step(&curr_key, 0);
|
||||
pk = aesg3_step(&curr_key, 1);
|
||||
right_child = aesg3_step(&curr_key, 2);
|
||||
|
||||
current_v_mask = ((current_v_mask as i32) >> 1) as u32;
|
||||
}
|
||||
|
||||
pk
|
||||
}
|
||||
|
||||
/// Outcome of a subset-difference walk against an MKB. Carries the
|
||||
/// processing key and the matching `uv` slot — both needed as inputs
|
||||
/// to the variant chain.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct ProcessingKeyMatch {
|
||||
/// Processing Key.
|
||||
pub kp: [u8; 16],
|
||||
/// Subset-difference node number that matched.
|
||||
pub uv: u32,
|
||||
/// 16-byte cvalue that the matched uv selected.
|
||||
pub cvalue: [u8; 16],
|
||||
/// Index of the matching cvalue within the cvalues record.
|
||||
pub cvalue_index: usize,
|
||||
}
|
||||
|
||||
fn mkb_find_body(records: &[MkbRecord], rec_type: u8) -> Option<&[u8]> {
|
||||
records
|
||||
.iter()
|
||||
.find(|r| r.rec_type == rec_type && !r.body.is_empty())
|
||||
.map(|r| r.body.as_slice())
|
||||
}
|
||||
|
||||
fn mkb_find_mk_dv(records: &[MkbRecord]) -> Option<[u8; 16]> {
|
||||
let r = records
|
||||
.iter()
|
||||
.find(|r| (r.rec_type == 0x81 || r.rec_type == 0x86) && r.body.len() >= 16)?;
|
||||
let mut out = [0u8; 16];
|
||||
out.copy_from_slice(&r.body[..16]);
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Walk an MKB and return the first `(Kp, uv, cvalue)` that
|
||||
/// `device_keys` covers. Returns `None` if no DK walks any uv.
|
||||
pub fn walk_processing_key(
|
||||
records: &[MkbRecord],
|
||||
device_keys: &[DeviceKey],
|
||||
) -> Option<ProcessingKeyMatch> {
|
||||
let mk_dv = mkb_find_mk_dv(records)?;
|
||||
let uvs = mkb_find_body(records, 0x04)?;
|
||||
let cvalues = mkb_find_body(records, 0x07).or_else(|| mkb_find_body(records, 0x05))?;
|
||||
|
||||
let num_uvs = uvs
|
||||
.chunks(5)
|
||||
.take_while(|c| c.len() == 5 && (c[0] & 0xC0) == 0)
|
||||
.count();
|
||||
|
||||
for dk in device_keys {
|
||||
let device_number = dk.node as u32;
|
||||
|
||||
for uvs_idx in 0..num_uvs {
|
||||
let p_uv = &uvs[1 + 5 * uvs_idx..];
|
||||
let u_mask_shift = uvs[5 * uvs_idx];
|
||||
|
||||
if u_mask_shift & 0xC0 != 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let uv = u32::from_be_bytes([p_uv[0], p_uv[1], p_uv[2], p_uv[3]]);
|
||||
if uv == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let u_mask: u32 = 0xFFFF_FFFFu32.wrapping_shl(u_mask_shift as u32);
|
||||
let v_mask = calc_v_mask(uv);
|
||||
|
||||
if ((device_number & u_mask) == (uv & u_mask))
|
||||
&& ((device_number & v_mask) != (uv & v_mask))
|
||||
{
|
||||
let dev_key_v_mask = calc_v_mask(dk.uv);
|
||||
let dev_key_u_mask: u32 = 0xFFFF_FFFFu32.wrapping_shl(dk.u_mask_shift as u32);
|
||||
|
||||
if u_mask == dev_key_u_mask && (uv & dev_key_v_mask) == (dk.uv & dev_key_v_mask) {
|
||||
let pk = calc_pk_from_dk(&dk.key, uv, v_mask, dev_key_v_mask);
|
||||
|
||||
if uvs_idx >= cvalues.len() / 16 {
|
||||
continue;
|
||||
}
|
||||
let mut cv = [0u8; 16];
|
||||
cv.copy_from_slice(&cvalues[uvs_idx * 16..(uvs_idx + 1) * 16]);
|
||||
|
||||
// Validate: AES-D(Kp, cv), XOR uv into low 4 bytes,
|
||||
// then AES-D(.., mk_dv) must reveal the verify magic.
|
||||
let mut km_candidate = aes_ecb_decrypt(&pk, &cv);
|
||||
let uv_bytes = uv.to_be_bytes();
|
||||
for i in 0..4 {
|
||||
km_candidate[12 + i] ^= uv_bytes[i];
|
||||
}
|
||||
let dec_vd = aes_ecb_decrypt(&km_candidate, &mk_dv);
|
||||
const VERIFY_MAGIC: [u8; 8] = [0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF];
|
||||
// On a classical (non-variant) MKB this magic must
|
||||
// match. On a variant MKB it won't — `km_candidate`
|
||||
// is really Kmp and the magic check is moot. We
|
||||
// still gate the walk on cvalue indexing being
|
||||
// sane; the chain itself enforces the variant
|
||||
// semantics downstream.
|
||||
let classical_ok = dec_vd[..8] == VERIFY_MAGIC;
|
||||
let variant_present = is_variant_mkb(records);
|
||||
if !(classical_ok || variant_present) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return Some(ProcessingKeyMatch {
|
||||
kp: pk,
|
||||
uv,
|
||||
cvalue: cv,
|
||||
cvalue_index: uvs_idx,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
// ── Error reporting ───────────────────────────────────────────────────────
|
||||
|
||||
/// Outcome of [`derive_media_key_variant`] when the chain cannot
|
||||
/// produce a Media Key. Every variant is a classification only — no
|
||||
/// strings, no Display impl beyond the error code.
|
||||
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
|
||||
pub enum MediaKeyVariantError {
|
||||
/// MKB carries no Variant records. Caller should fall back to the
|
||||
/// classical single-stage derivation.
|
||||
NotVariantMkb,
|
||||
/// MKB is missing a required record (mk_dv, subset-difference,
|
||||
/// cvalues, variant data, or variant nonce).
|
||||
MkbIncomplete,
|
||||
/// `device_keys` did not cover any uv slot in this MKB.
|
||||
ProcessingKeyUnavailable,
|
||||
/// `Kmp[15]` carries bit `0x02`: the soft-correction path applies
|
||||
/// for this Precursor. Out of scope for the hardcoded-KCD chain.
|
||||
SoftCorrectionRequired,
|
||||
/// `Kmp[15]` carries bit `0x04`: the online-challenge path applies
|
||||
/// for this Precursor. Out of scope for the hardcoded-KCD chain.
|
||||
OnlineChallengeRequired,
|
||||
/// Supplied KCD equals [`KEY_CORRECTION_DATA_PLACEHOLDER`]. The
|
||||
/// derivation refuses to run with the all-zero placeholder.
|
||||
KcdNotProvided,
|
||||
/// `VARIANTS[uv]` lookup for the matched uv is not implemented.
|
||||
VariantsTableUnavailable,
|
||||
/// VKD index resolved out of the supplied `vkd_table`.
|
||||
VkdIndexOutOfRange,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for MediaKeyVariantError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let code: u16 = match self {
|
||||
MediaKeyVariantError::NotVariantMkb => 7100,
|
||||
MediaKeyVariantError::MkbIncomplete => 7101,
|
||||
MediaKeyVariantError::ProcessingKeyUnavailable => 7102,
|
||||
MediaKeyVariantError::SoftCorrectionRequired => 7103,
|
||||
MediaKeyVariantError::OnlineChallengeRequired => 7104,
|
||||
MediaKeyVariantError::KcdNotProvided => 7105,
|
||||
MediaKeyVariantError::VariantsTableUnavailable => 7106,
|
||||
MediaKeyVariantError::VkdIndexOutOfRange => 7107,
|
||||
};
|
||||
write!(f, "E{code}")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for MediaKeyVariantError {}
|
||||
|
||||
// ── Chain ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Look up `VARIANTS[uv]` for the matched uv. The byte layout of the
|
||||
/// per-uv slot in the Variant Number record is undocumented and is
|
||||
/// disc-specific; this helper returns `None` until a Variant disc is
|
||||
/// available to fix the layout against.
|
||||
fn variants_for_uv(_records: &[MkbRecord], _uv_index: usize) -> Option<u16> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Run the Media Key Variant chain on an MKB.
|
||||
///
|
||||
/// Inputs:
|
||||
///
|
||||
/// - `mkb_records` : MKB pre-walked via [`walk_mkb`].
|
||||
/// - `device_keys` : pool of device keys; the chain runs against the
|
||||
/// first uv slot any DK covers.
|
||||
/// - `kcd` : integrator-supplied Key Correction Data. Must not
|
||||
/// equal [`KEY_CORRECTION_DATA_PLACEHOLDER`].
|
||||
/// - `vid` : 16-byte Volume ID for the disc. Used to derive
|
||||
/// the final VUK alongside the Media Key.
|
||||
///
|
||||
/// Returns `(Km, Kvu)` on success.
|
||||
pub fn derive_media_key_variant(
|
||||
mkb_records: &[MkbRecord],
|
||||
device_keys: &[DeviceKey],
|
||||
kcd: &[u8; 16],
|
||||
vid: &[u8; 16],
|
||||
) -> Result<([u8; 16], [u8; 16]), MediaKeyVariantError> {
|
||||
if !is_variant_mkb(mkb_records) {
|
||||
return Err(MediaKeyVariantError::NotVariantMkb);
|
||||
}
|
||||
|
||||
let pkm = walk_processing_key(mkb_records, device_keys)
|
||||
.ok_or(MediaKeyVariantError::ProcessingKeyUnavailable)?;
|
||||
|
||||
let nonce = variant_nonce(mkb_records).ok_or(MediaKeyVariantError::MkbIncomplete)?;
|
||||
let vkd_table = variant_key_data(mkb_records).ok_or(MediaKeyVariantError::MkbIncomplete)?;
|
||||
let c_value = variant_data_record(mkb_records).ok_or(MediaKeyVariantError::MkbIncomplete)?;
|
||||
if c_value.len() < 16 {
|
||||
return Err(MediaKeyVariantError::MkbIncomplete);
|
||||
}
|
||||
let mut c_block = [0u8; 16];
|
||||
c_block.copy_from_slice(&c_value[..16]);
|
||||
|
||||
// Step: Kmp = AES-128D(Kp, C) XOR uv (uv into low 4 bytes).
|
||||
let mut kmp = aes_ecb_decrypt(&pkm.kp, &c_block);
|
||||
let uv_bytes = pkm.uv.to_be_bytes();
|
||||
for i in 0..4 {
|
||||
kmp[12 + i] ^= uv_bytes[i];
|
||||
}
|
||||
|
||||
// Condition bits on Kmp[15] route off the hardcoded-KCD path.
|
||||
if kmp[15] & 0b0000_0010 != 0 {
|
||||
return Err(MediaKeyVariantError::SoftCorrectionRequired);
|
||||
}
|
||||
if kmp[15] & 0b0000_0100 != 0 {
|
||||
return Err(MediaKeyVariantError::OnlineChallengeRequired);
|
||||
}
|
||||
if kcd == &KEY_CORRECTION_DATA_PLACEHOLDER {
|
||||
return Err(MediaKeyVariantError::KcdNotProvided);
|
||||
}
|
||||
|
||||
// Step: Kpnew = Kmp XOR KCD.
|
||||
let mut kpnew = [0u8; 16];
|
||||
for i in 0..16 {
|
||||
kpnew[i] = kmp[i] ^ kcd[i];
|
||||
}
|
||||
|
||||
// Step: Kvn = AES-G(Kp, Nonce) & 0xFFFF (low 16 bits, BE).
|
||||
let kvn_block = aes_g(&pkm.kp, &nonce);
|
||||
let kvn = u16::from_be_bytes([kvn_block[14], kvn_block[15]]);
|
||||
|
||||
// Step: VKD_idx = Kvn XOR VARIANTS[uv].
|
||||
let v_for_uv = variants_for_uv(mkb_records, pkm.cvalue_index)
|
||||
.ok_or(MediaKeyVariantError::VariantsTableUnavailable)?;
|
||||
let vkd_idx = kvn ^ v_for_uv;
|
||||
|
||||
// Step: VKD = vkd_table[VKD_idx * 16 .. +16].
|
||||
let off = (vkd_idx as usize) * 16;
|
||||
if off + 16 > vkd_table.len() {
|
||||
return Err(MediaKeyVariantError::VkdIndexOutOfRange);
|
||||
}
|
||||
let mut vkd = [0u8; 16];
|
||||
vkd.copy_from_slice(&vkd_table[off..off + 16]);
|
||||
|
||||
// Step: Km = AES-128D(Kpnew, VKD) XOR uv.
|
||||
let mut km = aes_ecb_decrypt(&kpnew, &vkd);
|
||||
for i in 0..4 {
|
||||
km[12 + i] ^= uv_bytes[i];
|
||||
}
|
||||
|
||||
// Step: Kvu = AES-G(Km, VID).
|
||||
let kvu = aes_g(&km, vid);
|
||||
|
||||
Ok((km, kvu))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// ── Helpers ──
|
||||
|
||||
fn synthetic_mkb_classical() -> Vec<u8> {
|
||||
// Minimal MKB: type/version record + cvalues + mk_dv. No variant
|
||||
// records.
|
||||
let mut mkb = vec![
|
||||
0x10, 0x00, 0x00, 0x0C, 0x48, 0x14, 0x10, 0x03, 0x00, 0x00, 0x00, 0x4D,
|
||||
];
|
||||
mkb.extend_from_slice(&[0x07, 0x00, 0x00, 0x14]);
|
||||
mkb.extend_from_slice(&[0xAB; 16]);
|
||||
mkb.extend_from_slice(&[0x86, 0x00, 0x00, 0x14]);
|
||||
mkb.extend_from_slice(&[0xCD; 16]);
|
||||
mkb
|
||||
}
|
||||
|
||||
fn synthetic_mkb_with_variant() -> Vec<u8> {
|
||||
let mut mkb = synthetic_mkb_classical();
|
||||
// 0x82 — 16-byte body (Variant data / VKD slot).
|
||||
mkb.extend_from_slice(&[0x82, 0x00, 0x00, 0x14]);
|
||||
mkb.extend_from_slice(&[0xEE; 16]);
|
||||
// 0x83 — 16-byte body (Variant Nonce).
|
||||
mkb.extend_from_slice(&[0x83, 0x00, 0x00, 0x14]);
|
||||
mkb.extend_from_slice(&[0x55; 16]);
|
||||
mkb
|
||||
}
|
||||
|
||||
// ── Walker / record detection ──
|
||||
|
||||
#[test]
|
||||
fn walker_parses_synthetic_mkb() {
|
||||
let mkb = synthetic_mkb_classical();
|
||||
let recs = walk_mkb(&mkb);
|
||||
assert_eq!(recs.len(), 3);
|
||||
assert_eq!(recs[0].rec_type, 0x10);
|
||||
assert_eq!(recs[1].rec_type, 0x07);
|
||||
assert_eq!(recs[2].rec_type, 0x86);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn variant_detection_negative_on_classical() {
|
||||
let recs = walk_mkb(&synthetic_mkb_classical());
|
||||
assert!(!is_variant_mkb(&recs));
|
||||
assert!(variant_nonce(&recs).is_none());
|
||||
assert!(variant_key_data(&recs).is_none());
|
||||
assert!(variant_data_record(&recs).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn variant_detection_positive_on_variant() {
|
||||
let recs = walk_mkb(&synthetic_mkb_with_variant());
|
||||
assert!(is_variant_mkb(&recs));
|
||||
assert_eq!(variant_nonce(&recs), Some([0x55; 16]));
|
||||
assert_eq!(variant_key_data(&recs), Some(&[0xEE; 16][..]));
|
||||
assert_eq!(variant_data_record(&recs), Some(&[0xEE; 16][..]));
|
||||
}
|
||||
|
||||
// ── Chain entry-point classification ──
|
||||
|
||||
#[test]
|
||||
fn chain_rejects_non_variant_mkb() {
|
||||
let recs = walk_mkb(&synthetic_mkb_classical());
|
||||
let err = derive_media_key_variant(&recs, &[], &[0xAA; 16], &[0u8; 16])
|
||||
.expect_err("classical MKB must be rejected");
|
||||
assert_eq!(err, MediaKeyVariantError::NotVariantMkb);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chain_rejects_placeholder_kcd() {
|
||||
// To reach the KCD check we need a complete variant MKB AND a
|
||||
// DK that walks it. We construct both via the synthetic
|
||||
// fixture below.
|
||||
let (recs, dk, _kp, _expected_kmp) = synthetic_variant_setup(/*kmp15*/ 0x00);
|
||||
let err =
|
||||
derive_media_key_variant(&recs, &[dk], &KEY_CORRECTION_DATA_PLACEHOLDER, &[0u8; 16])
|
||||
.expect_err("placeholder KCD must be rejected");
|
||||
assert_eq!(err, MediaKeyVariantError::KcdNotProvided);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chain_detects_soft_correction_bit() {
|
||||
let (recs, dk, _, _) = synthetic_variant_setup(/*kmp15*/ 0x02);
|
||||
let err = derive_media_key_variant(&recs, &[dk], &[0xAA; 16], &[0u8; 16])
|
||||
.expect_err("bit 0x02 must surface SoftCorrectionRequired");
|
||||
assert_eq!(err, MediaKeyVariantError::SoftCorrectionRequired);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chain_detects_online_challenge_bit() {
|
||||
let (recs, dk, _, _) = synthetic_variant_setup(/*kmp15*/ 0x04);
|
||||
let err = derive_media_key_variant(&recs, &[dk], &[0xAA; 16], &[0u8; 16])
|
||||
.expect_err("bit 0x04 must surface OnlineChallengeRequired");
|
||||
assert_eq!(err, MediaKeyVariantError::OnlineChallengeRequired);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chain_surfaces_variants_table_gap_on_clean_kmp() {
|
||||
// With both condition bits clear and a non-placeholder KCD, the
|
||||
// chain advances to the per-uv VARIANTS[uv] lookup, which is
|
||||
// not yet wired. That returns VariantsTableUnavailable —
|
||||
// proving the bit checks and KCD check all passed.
|
||||
let (recs, dk, _, _) = synthetic_variant_setup(/*kmp15*/ 0x00);
|
||||
let err = derive_media_key_variant(&recs, &[dk], &[0xAA; 16], &[0u8; 16])
|
||||
.expect_err("expected VariantsTableUnavailable at the per-uv lookup");
|
||||
assert_eq!(err, MediaKeyVariantError::VariantsTableUnavailable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_display_is_code_only() {
|
||||
// No English in Display — every variant emits "E7xxx" and
|
||||
// nothing else.
|
||||
let cases = [
|
||||
MediaKeyVariantError::NotVariantMkb,
|
||||
MediaKeyVariantError::MkbIncomplete,
|
||||
MediaKeyVariantError::ProcessingKeyUnavailable,
|
||||
MediaKeyVariantError::SoftCorrectionRequired,
|
||||
MediaKeyVariantError::OnlineChallengeRequired,
|
||||
MediaKeyVariantError::KcdNotProvided,
|
||||
MediaKeyVariantError::VariantsTableUnavailable,
|
||||
MediaKeyVariantError::VkdIndexOutOfRange,
|
||||
];
|
||||
for e in cases {
|
||||
let s = e.to_string();
|
||||
assert!(
|
||||
s.starts_with('E') && s.len() == 5,
|
||||
"error display must be E#### only, got {s:?}"
|
||||
);
|
||||
assert!(
|
||||
s.chars().skip(1).all(|c| c.is_ascii_digit()),
|
||||
"error display must be E + digits, got {s:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Fixture construction ──
|
||||
|
||||
/// Build a synthetic variant MKB plus a DK that walks the single
|
||||
/// subset-difference slot it carries. `kmp15` is the value of the
|
||||
/// low byte of `Kmp[15]` that the chain will land on — pick `0x02`
|
||||
/// to exercise the SoftCorrection bit, `0x04` to exercise
|
||||
/// OnlineChallenge, `0x00` otherwise.
|
||||
///
|
||||
/// The fixture pins:
|
||||
/// - MKB subset-difference: `u_mask_shift=3, uv=2`. With these
|
||||
/// masks the discriminator bit (u_mask=1, v_mask=0) is bit 2.
|
||||
/// - one DK at `node=4, uv=2, u_mask_shift=3`. node 4 has bit 2 set
|
||||
/// (differs from uv=2 on bit 2 → disagrees on v_mask) while
|
||||
/// agreeing with uv on bits 3+ (the u_mask=1 region). dk.uv ==
|
||||
/// MKB.uv and dk.u_mask_shift == MKB.u_mask_shift make
|
||||
/// `dev_key_v_mask == v_mask`, so `calc_pk_from_dk` loops zero
|
||||
/// times — Kp = aesg3_step(dk, 1).
|
||||
/// - one cvalue in record 0x07 chosen so AES-D(Kp, C) ⊕ uv produces a
|
||||
/// Kmp whose byte-15 is exactly `kmp15`.
|
||||
/// - record 0x82 with a 16-byte body (acts as both Variant Data
|
||||
/// and Variant Key Data; satisfies the parser heuristics).
|
||||
/// - record 0x83 with a 16-byte Nonce.
|
||||
///
|
||||
/// Returns (records, dk, planted_kp, planted_kmp).
|
||||
fn synthetic_variant_setup(kmp15: u8) -> (Vec<MkbRecord>, DeviceKey, [u8; 16], [u8; 16]) {
|
||||
use crate::aacs::decrypt::aes_ecb_encrypt;
|
||||
|
||||
// Build header.
|
||||
let mut mkb = vec![
|
||||
0x10, 0x00, 0x00, 0x0C, 0x48, 0x14, 0x10, 0x03, 0x00, 0x00, 0x00, 0x4D,
|
||||
];
|
||||
|
||||
// Subset-difference (0x04): u_mask_shift=3, uv=00 00 00 02.
|
||||
mkb.extend_from_slice(&[0x04, 0x00, 0x00, 0x09]);
|
||||
mkb.extend_from_slice(&[0x03, 0x00, 0x00, 0x00, 0x02]);
|
||||
|
||||
// Pick a known DK; with dk.uv == MKB.uv (==2) and
|
||||
// dk.u_mask_shift == MKB.u_mask_shift (==1), dev_key_v_mask
|
||||
// equals the MKB's v_mask and the calc_pk_from_dk loop is a
|
||||
// no-op — Kp = aesg3_step(dk, 1).
|
||||
let dk_bytes: [u8; 16] = [
|
||||
0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE,
|
||||
0xFF, 0x00,
|
||||
];
|
||||
let kp = aesg3_step(&dk_bytes, 1);
|
||||
|
||||
// Plant Kmp with chosen byte-15, then compute C such that
|
||||
// AES-D(Kp, C) ⊕ uv == Kmp. uv=2 → low-4 bytes XOR is 00 00 00 02.
|
||||
let mut kmp = [0x42u8; 16];
|
||||
kmp[15] = kmp15;
|
||||
let mut aes_d_result = kmp;
|
||||
aes_d_result[15] ^= 0x02;
|
||||
let c_block = aes_ecb_encrypt(&kp, &aes_d_result);
|
||||
|
||||
// cvalues record (0x07): one 16-byte cvalue. The walker
|
||||
// indexes it for the magic-check step; on a variant MKB the
|
||||
// magic check fails but `variant_present` is true so the
|
||||
// walker still returns the match. Content is don't-care.
|
||||
mkb.extend_from_slice(&[0x07, 0x00, 0x00, 0x14]);
|
||||
mkb.extend_from_slice(&[0xAB; 16]);
|
||||
|
||||
// Verify Media Key (0x86): body content is don't-care.
|
||||
mkb.extend_from_slice(&[0x86, 0x00, 0x00, 0x14]);
|
||||
mkb.extend_from_slice(&[0xCD; 16]);
|
||||
|
||||
// 0x82 record: holds C (Encrypted Media Key Variant Data) AND
|
||||
// doubles as the VKD table (single 16-byte entry → VKDidx must
|
||||
// resolve to 0 for `chain_surfaces_variants_table_gap` test —
|
||||
// but the test never reaches the VKD lookup since the
|
||||
// VARIANTS[uv] helper is not yet wired).
|
||||
mkb.extend_from_slice(&[0x82, 0x00, 0x00, 0x14]);
|
||||
mkb.extend_from_slice(&c_block);
|
||||
|
||||
// 0x83 record: 16-byte Nonce.
|
||||
mkb.extend_from_slice(&[0x83, 0x00, 0x00, 0x14]);
|
||||
mkb.extend_from_slice(&[0x77; 16]);
|
||||
|
||||
let recs = walk_mkb(&mkb);
|
||||
|
||||
let dk = DeviceKey {
|
||||
key: dk_bytes,
|
||||
node: 4,
|
||||
uv: 2,
|
||||
u_mask_shift: 3,
|
||||
};
|
||||
(recs, dk, kp, kmp)
|
||||
}
|
||||
}
|
||||
+158
-1
@@ -20,6 +20,35 @@ pub struct ClipInfo {
|
||||
pub ep_coarse: Vec<EpCoarse>,
|
||||
/// Fine EP entries for the primary video stream
|
||||
pub ep_fine: Vec<EpFine>,
|
||||
/// Per-stream metadata from the ProgramInfo section (BD spec).
|
||||
/// Cross-validates the MPLS STN view — see `labels/clpi.rs`.
|
||||
/// Empty when program_info is missing or malformed.
|
||||
pub streams: Vec<ClpiStream>,
|
||||
}
|
||||
|
||||
/// One stream descriptor from the CLPI ProgramInfo / stream_coding_info
|
||||
/// table. Mirrors the same fields the MPLS STN table carries — see
|
||||
/// `mpls::StreamEntry` for the playlist-side equivalent.
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct ClpiStream {
|
||||
/// PID of the stream in the MPEG-TS (matches MPLS).
|
||||
pub pid: u16,
|
||||
/// SCSI/BD coding type byte (0x80 LPCM, 0x83 TrueHD, 0x86 DTS-HD MA,
|
||||
/// 0x90 PG, etc.). See `labels::mpls_universal::coding_type_to_codec_hint`.
|
||||
pub coding_type: u8,
|
||||
/// ISO 639-2 3-char language code. Empty for video streams.
|
||||
pub language: String,
|
||||
/// Audio format byte (1=mono, 3=stereo, 6=5.1, 12=7.1).
|
||||
/// Zero for non-audio streams.
|
||||
pub audio_format: u8,
|
||||
/// Audio sample rate (1=48kHz, 4=96kHz, 5=192kHz). Zero for non-audio.
|
||||
pub audio_rate: u8,
|
||||
/// Video format byte (1=480i, 4=1080i, 5=720p, 6=1080p, 8=2160p).
|
||||
/// Zero for non-video.
|
||||
pub video_format: u8,
|
||||
/// Video rate (1=23.976, 2=24, 3=25, 4=29.97, 6=50, 7=59.94).
|
||||
pub video_rate: u8,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -128,7 +157,7 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
|
||||
|
||||
// Header offsets
|
||||
let _seq_info_start = u32::from_be_bytes([data[8], data[9], data[10], data[11]]) as usize;
|
||||
let _prog_info_start = u32::from_be_bytes([data[12], data[13], data[14], data[15]]) as usize;
|
||||
let prog_info_start = u32::from_be_bytes([data[12], data[13], data[14], data[15]]) as usize;
|
||||
let cpi_start = u32::from_be_bytes([data[16], data[17], data[18], data[19]]) as usize;
|
||||
|
||||
// ClipInfo section at offset 40
|
||||
@@ -139,6 +168,16 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
|
||||
0
|
||||
};
|
||||
|
||||
// Parse ProgramInfo (per-stream language + codec). Best-effort:
|
||||
// malformed program_info doesn't fail the parse, just gives an
|
||||
// empty streams list. EP map is unaffected — sector-range lookups
|
||||
// continue to work.
|
||||
let streams = if prog_info_start > 0 && prog_info_start + 6 < data.len() {
|
||||
parse_program_info(&data[prog_info_start..])
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
// Parse CPI / EP Map
|
||||
let (ep_coarse, ep_fine) = if cpi_start > 0 && cpi_start + 8 < data.len() {
|
||||
parse_cpi(&data[cpi_start..])?
|
||||
@@ -151,9 +190,127 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
|
||||
source_packet_count,
|
||||
ep_coarse,
|
||||
ep_fine,
|
||||
streams,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse the ProgramInfo section: per-stream (pid, coding_type,
|
||||
/// language, codec sub-fields). Layout per BD spec / libbluray
|
||||
/// clpi_parse.c:
|
||||
///
|
||||
/// ```text
|
||||
/// ProgramInfo:
|
||||
/// length: 4 bytes
|
||||
/// reserved: 1 byte
|
||||
/// num_programs: 1 byte
|
||||
/// for each program:
|
||||
/// spn_program_sequence_start: 4 bytes
|
||||
/// program_map_pid: 2 bytes
|
||||
/// num_streams: 1 byte
|
||||
/// num_groups: 1 byte
|
||||
/// for each stream:
|
||||
/// pid: 2 bytes
|
||||
/// stream_coding_info_length: 1 byte
|
||||
/// stream_coding_info: (varies by coding_type)
|
||||
/// coding_type: 1 byte
|
||||
/// per-type bytes (see match arms below)
|
||||
/// ```
|
||||
///
|
||||
/// Returns `Vec::new()` on any structural mismatch — we don't propagate
|
||||
/// errors because the EP map is the primary CLPI output, and a corrupt
|
||||
/// program_info shouldn't break sector-range lookups.
|
||||
fn parse_program_info(data: &[u8]) -> Vec<ClpiStream> {
|
||||
let mut out = Vec::new();
|
||||
if data.len() < 6 {
|
||||
return out;
|
||||
}
|
||||
// length: 4 bytes (skipped — we trust the section bounds in the
|
||||
// caller's slice and read the bytes that follow). Reserved 1 byte
|
||||
// at offset 4. num_programs at offset 5.
|
||||
let num_programs = data[5] as usize;
|
||||
let mut pos = 6usize;
|
||||
for _ in 0..num_programs {
|
||||
// Program header: 4 (spn) + 2 (pmt_pid) + 1 (num_streams) + 1 (num_groups) = 8 bytes
|
||||
if pos + 8 > data.len() {
|
||||
return out;
|
||||
}
|
||||
let num_streams = data[pos + 6] as usize;
|
||||
pos += 8;
|
||||
|
||||
for _ in 0..num_streams {
|
||||
// Stream header: 2 (pid) + 1 (sci_length) + sci bytes
|
||||
if pos + 3 > data.len() {
|
||||
return out;
|
||||
}
|
||||
let pid = u16::from_be_bytes([data[pos], data[pos + 1]]);
|
||||
let sci_len = data[pos + 2] as usize;
|
||||
let sci_end = pos + 3 + sci_len;
|
||||
if sci_end > data.len() || sci_len < 1 {
|
||||
return out;
|
||||
}
|
||||
let sci = &data[pos + 3..sci_end];
|
||||
let coding_type = sci[0];
|
||||
|
||||
let mut audio_format = 0u8;
|
||||
let mut audio_rate = 0u8;
|
||||
let mut video_format = 0u8;
|
||||
let mut video_rate = 0u8;
|
||||
let mut language = String::new();
|
||||
|
||||
match coding_type {
|
||||
// Video — MPEG-2 (0x02), H.264 (0x1B), HEVC (0x24)
|
||||
0x02 | 0x1B | 0x24 => {
|
||||
if sci.len() >= 2 {
|
||||
video_format = (sci[1] >> 4) & 0x0F;
|
||||
video_rate = sci[1] & 0x0F;
|
||||
}
|
||||
}
|
||||
// Primary audio — LPCM(0x80), AC-3(0x81), DTS(0x82),
|
||||
// TrueHD(0x83), AC-3+(0x84), DTS-HD(0x85), DTS-HD MA(0x86)
|
||||
0x80..=0x86 => {
|
||||
if sci.len() >= 2 {
|
||||
audio_format = (sci[1] >> 4) & 0x0F;
|
||||
audio_rate = sci[1] & 0x0F;
|
||||
}
|
||||
if sci.len() >= 5 {
|
||||
language = String::from_utf8_lossy(&sci[2..5]).to_string();
|
||||
}
|
||||
}
|
||||
// Secondary audio (0xA1 AC-3+, 0xA2 DTS-HD)
|
||||
0xA1 | 0xA2 => {
|
||||
if sci.len() >= 2 {
|
||||
audio_format = (sci[1] >> 4) & 0x0F;
|
||||
audio_rate = sci[1] & 0x0F;
|
||||
}
|
||||
if sci.len() >= 5 {
|
||||
language = String::from_utf8_lossy(&sci[2..5]).to_string();
|
||||
}
|
||||
}
|
||||
// PG (0x90), IG (0x91): coding_type + 3-byte language [+ char_code for PG]
|
||||
0x90 | 0x91 => {
|
||||
if sci.len() >= 4 {
|
||||
language = String::from_utf8_lossy(&sci[1..4]).to_string();
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
out.push(ClpiStream {
|
||||
pid,
|
||||
coding_type,
|
||||
language,
|
||||
audio_format,
|
||||
audio_rate,
|
||||
video_format,
|
||||
video_rate,
|
||||
});
|
||||
|
||||
pos = sci_end;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Parse the CPI section containing the EP map.
|
||||
fn parse_cpi(data: &[u8]) -> Result<(Vec<EpCoarse>, Vec<EpFine>)> {
|
||||
if data.len() < 8 {
|
||||
|
||||
+4
-3
@@ -4,13 +4,14 @@
|
||||
//! 1. Bus authentication (challenge-response) → bus key
|
||||
//! 2. Read disc key block (READ DVD STRUCTURE) → XOR with bus key → decrypt with player keys → disc key
|
||||
//! 3. Read title key (REPORT KEY format 0x04) → XOR with bus key → decrypt with disc key → title key
|
||||
//!
|
||||
//! Based on libdvdcss (VideoLAN) and Stevenson 1999 analysis.
|
||||
|
||||
use crate::drive::Drive;
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
// ── Player keys (from libdvdcss, Stevenson's PlayerKey cracker) ───────────
|
||||
// ── Built-in public DVD CSS player keys ────────────────────────────────────
|
||||
//
|
||||
// These 31 5-byte player keys are long-public CSS inputs. With them
|
||||
// compiled in, DVD ripping works with no external key file required.
|
||||
|
||||
const PLAYER_KEYS: [[u8; 5]; 31] = [
|
||||
[0x01, 0xaf, 0xe3, 0x12, 0x80],
|
||||
|
||||
+47
-2
@@ -16,7 +16,8 @@ pub mod lfsr;
|
||||
pub(crate) mod tables;
|
||||
|
||||
use crate::disc::Extent;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::drive::Drive;
|
||||
use crate::sector::SectorSource;
|
||||
|
||||
/// CSS decryption state for a DVD title.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -25,6 +26,50 @@ pub struct CssState {
|
||||
pub title_key: [u8; 5],
|
||||
}
|
||||
|
||||
/// Inputs for CSS key acquisition.
|
||||
///
|
||||
/// The acquisition path depends on which inputs the caller supplies:
|
||||
///
|
||||
/// - With `drive` + `auth_lba` set, [`resolve`] runs the full SCSI bus
|
||||
/// auth + title-key path (live BU40N / DVD drive).
|
||||
/// - With `reader` + `extents` set, [`resolve`] falls back to the
|
||||
/// crack path (Stevenson known-plaintext attack on encrypted PES
|
||||
/// headers; works on disc images and on drives whose CSS auth path
|
||||
/// is unavailable).
|
||||
///
|
||||
/// `live_drive` always wins when both modes are populated.
|
||||
pub struct CssContext<'a> {
|
||||
/// Live SCSI drive — when present, [`resolve`] tries the auth path.
|
||||
pub drive: Option<&'a mut Drive>,
|
||||
/// LBA of a known-scrambled sector for the auth path's title-key
|
||||
/// query. Required when `drive` is set.
|
||||
pub auth_lba: Option<u32>,
|
||||
/// Sector source for the crack path.
|
||||
pub reader: Option<&'a mut dyn SectorSource>,
|
||||
/// Extents to scan for the crack path. Required when `reader` is
|
||||
/// set.
|
||||
pub extents: Option<&'a [Extent]>,
|
||||
}
|
||||
|
||||
/// Acquire a CSS title key using whichever inputs the context provides.
|
||||
///
|
||||
/// Order of attempts:
|
||||
/// 1. SCSI auth path (when `drive` and `auth_lba` are set).
|
||||
/// 2. Crack path (when `reader` and `extents` are set).
|
||||
///
|
||||
/// Returns `None` if neither path is configured or both fail.
|
||||
pub fn resolve(ctx: &mut CssContext<'_>) -> Option<CssState> {
|
||||
if let (Some(drive), Some(lba)) = (ctx.drive.as_deref_mut(), ctx.auth_lba) {
|
||||
if let Ok(title_key) = auth::authenticate_and_read_title_key(drive, lba) {
|
||||
return Some(CssState { title_key });
|
||||
}
|
||||
}
|
||||
if let (Some(reader), Some(extents)) = (ctx.reader.as_deref_mut(), ctx.extents) {
|
||||
return crack_key(reader, extents);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Crack the CSS title key by reading encrypted sectors and applying
|
||||
/// a known-plaintext attack on MPEG-2 headers.
|
||||
///
|
||||
@@ -34,7 +79,7 @@ pub struct CssState {
|
||||
/// 0x80 (start of the encrypted region). This only happens when a new PES
|
||||
/// packet begins at exactly sector offset 128. We scan up to 50000
|
||||
/// scrambled sectors sequentially across all extents.
|
||||
pub fn crack_key(reader: &mut dyn SectorReader, extents: &[Extent]) -> Option<CssState> {
|
||||
pub fn crack_key(reader: &mut dyn SectorSource, extents: &[Extent]) -> Option<CssState> {
|
||||
let mut tried = 0u32;
|
||||
let max_tries = 50_000;
|
||||
|
||||
|
||||
+191
-4
@@ -3,9 +3,131 @@
|
||||
//! Decrypts sectors in-place using resolved keys from disc scanning.
|
||||
//! Handles AACS 1.0, AACS 2.0, and CSS transparently.
|
||||
//! The caller never sees encrypted data unless explicitly bypassed.
|
||||
//!
|
||||
//! ## Parallel AACS decrypt
|
||||
//!
|
||||
//! Each AACS aligned unit (6144 bytes) is decrypted INDEPENDENTLY of
|
||||
//! every other unit — per-unit key derivation from the unit_key plus
|
||||
//! the unit's own first-16-byte header. There is no cross-unit
|
||||
//! dependency, so a buffer of N units can be decrypted on N threads
|
||||
//! in parallel via a persistent rayon thread pool.
|
||||
//!
|
||||
//! Small buffers (< [`PARALLEL_MIN_UNITS`] units) fall through to the
|
||||
//! serial path to avoid pool dispatch overhead beating the per-unit
|
||||
//! AES work.
|
||||
//!
|
||||
//! ## Thread-count configuration — three layers
|
||||
//!
|
||||
//! Resolution order (highest wins):
|
||||
//! 1. The most recent [`set_decrypt_threads`] call with `n > 0`.
|
||||
//! Calling this *replaces* the live thread pool — useful for a
|
||||
//! settings-page slider in a long-running daemon.
|
||||
//! 2. `FREEMKV_THREADS` env var, if set and `> 0`. Single knob
|
||||
//! covering decrypt today, intended to also drive any future
|
||||
//! input-side / output-side worker pools.
|
||||
//! 3. Default: all available cores. Algorithm optimisation comes
|
||||
//! first — we measure single-thread performance to find serial
|
||||
//! bottlenecks before throwing parallelism at it — but once a
|
||||
//! pool is engaged we use the whole box. Hard cap at
|
||||
//! [`MAX_THREADS`] (rayon stack memory).
|
||||
|
||||
use crate::aacs;
|
||||
use crate::css;
|
||||
use rayon::prelude::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
/// Minimum units in a buffer before we pay the pool-dispatch cost of
|
||||
/// fanning out. Below this, serial is faster.
|
||||
const PARALLEL_MIN_UNITS: usize = 8;
|
||||
|
||||
/// Hard upper bound on configurable thread count. Anything larger is
|
||||
/// almost certainly a misconfiguration; rayon would happily allocate
|
||||
/// thousands of worker stacks otherwise.
|
||||
pub const MAX_THREADS: usize = 64;
|
||||
|
||||
/// Process-wide decrypt thread count override. `0` means "use env
|
||||
/// var, else default" — see [`decrypt_threads`] for the resolution
|
||||
/// order.
|
||||
static DECRYPT_THREADS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
/// Current rayon pool. `RwLock<Option<Arc<...>>>` so that
|
||||
/// [`set_decrypt_threads`] can swap the pool out without leaking the
|
||||
/// old one and without blocking ongoing decrypt work (in-flight calls
|
||||
/// hold an `Arc` clone via [`decrypt_pool`] and finish on the old
|
||||
/// pool; new calls pick up the new pool).
|
||||
static DECRYPT_POOL: RwLock<Option<Arc<rayon::ThreadPool>>> = RwLock::new(None);
|
||||
|
||||
/// Configure how many threads to use for AACS unit decryption. A value
|
||||
/// of `0` resets to the env / default resolution. `1` forces serial.
|
||||
/// `N > 1` builds a new rayon pool of size N and atomically replaces
|
||||
/// the live pool.
|
||||
///
|
||||
/// Thread-safe. Live decrypt calls keep their previously-acquired
|
||||
/// pool reference for the rest of the call — no mid-call pool
|
||||
/// switch. Subsequent calls see the new pool.
|
||||
///
|
||||
/// Pool construction is ~ms-scale; safe to call from a settings POST
|
||||
/// handler.
|
||||
pub fn set_decrypt_threads(n: usize) {
|
||||
let clamped = n.min(MAX_THREADS);
|
||||
DECRYPT_THREADS.store(clamped, Ordering::Relaxed);
|
||||
// Drop the existing pool. Next decrypt_pool() call rebuilds with
|
||||
// the new resolved thread count.
|
||||
if let Ok(mut guard) = DECRYPT_POOL.write() {
|
||||
*guard = None;
|
||||
}
|
||||
}
|
||||
|
||||
/// Get (or lazily build) the active rayon thread pool. Returns an
|
||||
/// `Arc` so in-flight work survives a concurrent
|
||||
/// [`set_decrypt_threads`] swap.
|
||||
fn decrypt_pool() -> Arc<rayon::ThreadPool> {
|
||||
// Fast path: pool already built.
|
||||
if let Ok(guard) = DECRYPT_POOL.read() {
|
||||
if let Some(pool) = guard.as_ref() {
|
||||
return Arc::clone(pool);
|
||||
}
|
||||
}
|
||||
// Slow path: build a new one under the write lock. Double-check
|
||||
// after acquiring in case another caller built it first.
|
||||
let mut guard = DECRYPT_POOL.write().expect("DECRYPT_POOL RwLock poisoned");
|
||||
if let Some(pool) = guard.as_ref() {
|
||||
return Arc::clone(pool);
|
||||
}
|
||||
let n = decrypt_threads();
|
||||
let pool = Arc::new(
|
||||
rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(n)
|
||||
.thread_name(|i| format!("freemkv-decrypt-{i}"))
|
||||
.build()
|
||||
.expect("rayon decrypt pool build failed"),
|
||||
);
|
||||
*guard = Some(Arc::clone(&pool));
|
||||
pool
|
||||
}
|
||||
|
||||
/// Current effective decrypt thread count. Resolution order:
|
||||
/// 1. Most recent [`set_decrypt_threads`] value (if > 0)
|
||||
/// 2. `FREEMKV_THREADS` env var (if set and > 0)
|
||||
/// 3. Default: all available cores, capped at [`MAX_THREADS`].
|
||||
pub fn decrypt_threads() -> usize {
|
||||
let explicit = DECRYPT_THREADS.load(Ordering::Relaxed);
|
||||
if explicit > 0 {
|
||||
return explicit;
|
||||
}
|
||||
let env = std::env::var("FREEMKV_THREADS")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<usize>().ok())
|
||||
.unwrap_or(0);
|
||||
if env > 0 {
|
||||
return env.min(MAX_THREADS);
|
||||
}
|
||||
let cores = std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(2);
|
||||
cores.clamp(1, MAX_THREADS)
|
||||
}
|
||||
|
||||
/// Resolved decryption state from disc scanning.
|
||||
/// Passed to `decrypt_sectors()` — the caller doesn't need to know
|
||||
@@ -57,13 +179,45 @@ pub fn decrypt_sectors(
|
||||
return Err(crate::error::Error::DecryptFailed);
|
||||
}
|
||||
};
|
||||
let rdk = read_data_key.as_ref();
|
||||
let rdk: Option<[u8; 16]> = *read_data_key;
|
||||
let unit_len = aacs::ALIGNED_UNIT_LEN;
|
||||
let nthreads = decrypt_threads();
|
||||
let chunks: Vec<&mut [u8]> = buf.chunks_mut(unit_len).collect();
|
||||
let nunits = chunks.len();
|
||||
|
||||
for chunk in buf.chunks_mut(unit_len) {
|
||||
if chunk.len() == unit_len && aacs::is_unit_encrypted(chunk) {
|
||||
aacs::decrypt_unit_full(chunk, &uk, rdk);
|
||||
// Per-unit decrypt closure. The is_aacs_scrambled check reads the
|
||||
// raw TS syncs; a non-m2ts unit (e.g. MPLS/CLPI nav file) can look
|
||||
// scrambled and trigger a decrypt attempt, so on a verify miss we
|
||||
// snapshot+restore the original bytes so it survives. See test
|
||||
// `nav_file_unit_survives_decrypt_attempt`.
|
||||
let decrypt_one = |chunk: &mut [u8]| {
|
||||
if chunk.len() == unit_len && aacs::is_aacs_scrambled(chunk) {
|
||||
let original: Vec<u8> = chunk.to_vec();
|
||||
if !aacs::decrypt_unit_full(chunk, &uk, rdk.as_ref()) {
|
||||
chunk.copy_from_slice(&original);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if nthreads <= 1 || nunits < PARALLEL_MIN_UNITS {
|
||||
// Serial path: avoids thread-pool overhead for tiny
|
||||
// buffers; also the only path when caller pinned
|
||||
// single-threaded via FREEMKV_THREADS=1.
|
||||
for chunk in chunks {
|
||||
decrypt_one(chunk);
|
||||
}
|
||||
} else {
|
||||
// Parallel path via rayon's persistent global pool.
|
||||
// The pool is built once on first use (lazy_static-style)
|
||||
// and reused across every decrypt_sectors call — no
|
||||
// per-call OS thread spawn, no thread-creation latency
|
||||
// amortised per batch. Each unit decrypts independently
|
||||
// (own key derivation), so par_iter is sound.
|
||||
decrypt_pool().install(|| {
|
||||
chunks.into_par_iter().for_each(|chunk| {
|
||||
decrypt_one(chunk);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
DecryptKeys::Css { title_key } => {
|
||||
@@ -74,3 +228,36 @@ pub fn decrypt_sectors(
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Regression for the 0.18.1 nav-file scramble bug. A non-m2ts unit (here
|
||||
/// an MPLS file: starts "MPLS", carries no TS syncs) reads as scrambled
|
||||
/// under `is_aacs_scrambled`, gets AES-decrypted with the unit key, fails
|
||||
/// the TS-sync verification, and must be restored to its original bytes —
|
||||
/// not left scrambled.
|
||||
#[test]
|
||||
fn nav_file_unit_survives_decrypt_attempt() {
|
||||
let mut unit = vec![0u8; aacs::ALIGNED_UNIT_LEN];
|
||||
unit[0] = b'M';
|
||||
unit[1] = b'P';
|
||||
unit[2] = b'L';
|
||||
unit[3] = b'S';
|
||||
for (i, b) in unit.iter_mut().enumerate().skip(4) {
|
||||
*b = (i as u8).wrapping_mul(31);
|
||||
}
|
||||
let snapshot = unit.clone();
|
||||
|
||||
let keys = DecryptKeys::Aacs {
|
||||
unit_keys: vec![(0, [0xAB; 16])],
|
||||
read_data_key: None,
|
||||
};
|
||||
decrypt_sectors(&mut unit, &keys, 0).unwrap();
|
||||
assert_eq!(
|
||||
unit, snapshot,
|
||||
"non-m2ts unit must be restored after failed decrypt"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -3,13 +3,13 @@
|
||||
use super::*;
|
||||
use crate::clpi;
|
||||
use crate::mpls;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf;
|
||||
|
||||
impl Disc {
|
||||
/// Scan Blu-ray titles from MPLS playlists.
|
||||
pub(super) fn scan_bluray_titles(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
) -> Vec<DiscTitle> {
|
||||
let mut titles = Vec::new();
|
||||
@@ -31,7 +31,7 @@ impl Disc {
|
||||
}
|
||||
|
||||
pub(super) fn parse_playlist(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
filename: &str,
|
||||
data: &[u8],
|
||||
@@ -199,7 +199,7 @@ impl Disc {
|
||||
/// Prefers English, falls back to first available language.
|
||||
/// Returns None if META directory is empty or XML has no usable title.
|
||||
pub(super) fn read_meta_title(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
) -> Option<String> {
|
||||
let meta_dir = udf_fs.find_dir("/BDMV/META")?;
|
||||
|
||||
+2
-2
@@ -2,13 +2,13 @@
|
||||
|
||||
use super::*;
|
||||
use crate::ifo;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf;
|
||||
|
||||
impl Disc {
|
||||
/// Scan DVD titles from IFO files (VIDEO_TS.IFO + VTS_XX_0.IFO).
|
||||
pub(super) fn scan_dvd_titles(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
) -> Vec<DiscTitle> {
|
||||
let dvd_info = match ifo::parse_vmg(reader, udf_fs) {
|
||||
|
||||
+319
-67
@@ -2,7 +2,7 @@
|
||||
|
||||
use super::*;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf;
|
||||
|
||||
/// Result of SCSI AACS handshake (ECDH authentication).
|
||||
@@ -14,107 +14,359 @@ pub(super) struct HandshakeResult {
|
||||
}
|
||||
|
||||
impl Disc {
|
||||
/// SCSI handshake result — volume ID and bus keys from ECDH authentication.
|
||||
/// Only available when scanning from a real drive (not ISO images).
|
||||
/// Acquire the Volume ID. Tries the per-drive OEM CDB path first
|
||||
/// when the drive reports `is_unlocked()` (extended-access state),
|
||||
/// and falls back to the cert-based AACS mutual-auth handshake
|
||||
/// otherwise.
|
||||
///
|
||||
/// The OEM path is a single READ_BUFFER CDB built from the drive
|
||||
/// profile's `read_vid_cdb` template. The response carries a 3-byte
|
||||
/// header (validated against `00 22 00`) followed by the 16-byte
|
||||
/// VID at bytes [4..20]. Crucially, no AGID setup is required —
|
||||
/// the drive's runtime firmware serves the VID directly when in
|
||||
/// extended-access state.
|
||||
///
|
||||
/// The cert path is the standard AACS spec flow: ECDH key
|
||||
/// agreement, bus-key derivation, then `REPORT_DISC_STRUCTURE`
|
||||
/// format 0x80 to retrieve VID under bus-key MAC.
|
||||
pub(super) fn read_vid(
|
||||
session: &mut crate::drive::Drive,
|
||||
opts: &ScanOptions,
|
||||
) -> Result<[u8; 16]> {
|
||||
if session.is_unlocked() {
|
||||
let profile = session
|
||||
.drive_profile()
|
||||
.ok_or(Error::DriveProfileMissing)?
|
||||
.clone();
|
||||
return Self::read_vid_oem(session, &profile);
|
||||
}
|
||||
Self::read_vid_cert(session, opts)
|
||||
}
|
||||
|
||||
/// OEM VID retrieval — issues the per-drive READ_BUFFER CDB and
|
||||
/// parses the response.
|
||||
///
|
||||
/// Response layout (36 bytes):
|
||||
/// * [0..3] 3-byte response signature; expected `00 22 00`
|
||||
/// * [3] reserved
|
||||
/// * [4..20] 16-byte Volume ID
|
||||
/// * [20..36] reserved / per-drive padding
|
||||
fn read_vid_oem(
|
||||
session: &mut crate::drive::Drive,
|
||||
profile: &crate::profile::DriveProfile,
|
||||
) -> Result<[u8; 16]> {
|
||||
const RESPONSE_LEN: usize = 36;
|
||||
const EXPECTED_HEADER: [u8; 3] = [0x00, 0x22, 0x00];
|
||||
|
||||
let cdb = profile.read_vid_cdb.ok_or(Error::VidCdbUnavailable)?;
|
||||
let mut buf = vec![0u8; RESPONSE_LEN];
|
||||
let result = session.scsi_execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
)?;
|
||||
if result.bytes_transferred < RESPONSE_LEN {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "oem_vid_short_response",
|
||||
bytes_transferred = result.bytes_transferred,
|
||||
"OEM VID CDB returned short response"
|
||||
);
|
||||
return Err(Error::AacsVidRead);
|
||||
}
|
||||
if buf[0..3] != EXPECTED_HEADER {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "oem_vid_bad_header",
|
||||
header_0 = buf[0],
|
||||
header_1 = buf[1],
|
||||
header_2 = buf[2],
|
||||
"OEM VID response header mismatch"
|
||||
);
|
||||
return Err(Error::AacsVidRead);
|
||||
}
|
||||
let mut vid = [0u8; 16];
|
||||
vid.copy_from_slice(&buf[4..20]);
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "oem_vid_ok",
|
||||
"OEM VID retrieved"
|
||||
);
|
||||
Ok(vid)
|
||||
}
|
||||
|
||||
/// Cert-based VID retrieval — runs the full AACS mutual-auth
|
||||
/// handshake and extracts VID from the bus-key-MAC'd
|
||||
/// `REPORT_DISC_STRUCTURE` response.
|
||||
fn read_vid_cert(session: &mut crate::drive::Drive, opts: &ScanOptions) -> Result<[u8; 16]> {
|
||||
match Self::do_handshake_cert(session, opts) {
|
||||
(Some(h), _) => Ok(h.volume_id),
|
||||
(None, Some(e)) => Err(e),
|
||||
(None, None) => Err(Error::AacsVidUnavailable),
|
||||
}
|
||||
}
|
||||
|
||||
/// SCSI handshake — drives the VID-acquisition flow and returns
|
||||
/// a structured `HandshakeResult` for downstream key resolution.
|
||||
/// Prefers the OEM path when `Drive::is_unlocked()` is true and
|
||||
/// falls back to cert-based mutual auth otherwise.
|
||||
///
|
||||
/// The OEM path produces only VID (no bus-key, so no
|
||||
/// `read_data_key`); the cert path can produce both. AACS 2.0
|
||||
/// content that needs read_data_key for bus decryption requires
|
||||
/// the cert path.
|
||||
///
|
||||
/// Returns `(handshake, error)`:
|
||||
/// * `(Some(_), None)` — VID acquired
|
||||
/// * `(None, Some(_))` — specific failure mode (see
|
||||
/// `AacsHostCertRejected` / `AacsRawReadUnsupported` /
|
||||
/// `AacsVidUnavailable` / `DriveProfileMissing` /
|
||||
/// `VidCdbUnavailable` variants in `error.rs`)
|
||||
/// * `(None, None)` — handshake not attempted (no keydb;
|
||||
/// resolution will proceed with VID=zero and rely on path 1
|
||||
/// disc-hash → VUK lookup)
|
||||
pub(super) fn do_handshake(
|
||||
session: &mut crate::drive::Drive,
|
||||
opts: &ScanOptions,
|
||||
) -> Option<HandshakeResult> {
|
||||
use crate::aacs::{self, KeyDb};
|
||||
) -> (Option<HandshakeResult>, Option<Error>) {
|
||||
let unlocked = session.is_unlocked();
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_entry",
|
||||
unlocked,
|
||||
"do_handshake entered"
|
||||
);
|
||||
|
||||
let keydb_path = opts.resolve_keydb()?;
|
||||
let keydb = KeyDb::load(&keydb_path).ok()?;
|
||||
if unlocked {
|
||||
// Try OEM VID retrieval first. If the drive's profile
|
||||
// doesn't carry the CDB template, or the response is
|
||||
// malformed, fall through to cert-based auth.
|
||||
match Self::read_vid(session, opts) {
|
||||
Ok(volume_id) => {
|
||||
return (
|
||||
Some(HandshakeResult {
|
||||
volume_id,
|
||||
read_data_key: None,
|
||||
}),
|
||||
None,
|
||||
);
|
||||
}
|
||||
Err(Error::DriveProfileMissing) | Err(Error::VidCdbUnavailable) => {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_oem_unavailable",
|
||||
"OEM VID path unavailable for this drive; trying cert handshake"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_oem_failed",
|
||||
error_code = e.code(),
|
||||
"OEM VID retrieval failed; trying cert handshake"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_CERT_ATTEMPTS: usize = 16;
|
||||
for hc in keydb.host_certs.iter().take(MAX_CERT_ATTEMPTS) {
|
||||
Self::do_handshake_cert(session, opts)
|
||||
}
|
||||
|
||||
/// Cert-based AACS handshake. The legacy auth path; still used as
|
||||
/// the fallback when the OEM VID path isn't available or fails.
|
||||
fn do_handshake_cert(
|
||||
session: &mut crate::drive::Drive,
|
||||
opts: &ScanOptions,
|
||||
) -> (Option<HandshakeResult>, Option<Error>) {
|
||||
use crate::aacs;
|
||||
|
||||
// Host certs come from the caller's DriveCredentials (e.g. the keydb's
|
||||
// host_certs(), sourced app-side) — the library does not load a keydb.
|
||||
// Absent ⇒ no cert auth: an unlocked / LibreDrive drive already returned
|
||||
// a Volume ID via the OEM path before reaching here, so this is the
|
||||
// locked-drive-without-credentials case.
|
||||
let host_certs: &[aacs::HostCert] = match &opts.credentials {
|
||||
Some(c) if !c.host_certs.is_empty() => &c.host_certs,
|
||||
_ => {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_no_credentials",
|
||||
"no drive credentials supplied; cert handshake skipped"
|
||||
);
|
||||
return (None, None);
|
||||
}
|
||||
};
|
||||
|
||||
let host_cert_count = host_certs.len();
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_start",
|
||||
host_cert_count,
|
||||
"handshake starting"
|
||||
);
|
||||
|
||||
// v0.25.7 wedge fix. Pre-0.25.7 this loop fired up to 16 AACS
|
||||
// authenticate attempts back-to-back with no pause. Each attempt
|
||||
// is 5-10 SCSI REPORT_KEY/SEND_KEY exchanges. On a disc whose
|
||||
// host cert isn't in our KEYDB (or one the drive rejects),
|
||||
// that's 80-160 SCSI commands hammered at the drive in a
|
||||
// few hundred milliseconds — and the BU40N (and most consumer
|
||||
// optical drives) responds by entering a fast-fail firmware
|
||||
// wedge state where every subsequent CDB returns
|
||||
// ILLEGAL_REQUEST/INVALID_FIELD_IN_CDB (sense 05/24) until
|
||||
// power-cycled. Hit live on rip1 2026-05-20 during a MOVIE
|
||||
// UHD scan: KEYDB miss → 16 cert attempts in a tight loop →
|
||||
// wedge → forced host reboot + drive disconnect to recover.
|
||||
//
|
||||
// Defense-in-depth: cap attempts, sleep between, and bail
|
||||
// early on the drive's wedge sense so any later regression
|
||||
// can't undo the protection silently.
|
||||
const MAX_CERT_ATTEMPTS: usize = 3;
|
||||
const PER_CERT_BACKOFF_MS: u64 = 1000;
|
||||
let mut last_err_code: Option<u16> = None;
|
||||
for (idx, hc) in host_certs.iter().take(MAX_CERT_ATTEMPTS).enumerate() {
|
||||
if idx > 0 {
|
||||
std::thread::sleep(std::time::Duration::from_millis(PER_CERT_BACKOFF_MS));
|
||||
}
|
||||
match aacs::handshake::aacs_authenticate(session, &hc.private_key, &hc.certificate) {
|
||||
Ok(mut auth) => {
|
||||
let volume_id = match aacs::handshake::read_volume_id(session, &mut auth) {
|
||||
Ok(vid) => vid,
|
||||
Err(_) => return None, // handshake succeeded but can't read VID
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_vid_read_failed",
|
||||
cert_index = idx,
|
||||
error_code = e.code(),
|
||||
"auth ok but volume ID read failed"
|
||||
);
|
||||
return (None, Some(Error::AacsVidUnavailable));
|
||||
}
|
||||
};
|
||||
let read_data_key = aacs::handshake::read_data_keys(session, &mut auth)
|
||||
.ok()
|
||||
.map(|(rdk, _)| rdk);
|
||||
return Some(HandshakeResult {
|
||||
volume_id,
|
||||
read_data_key,
|
||||
});
|
||||
tracing::debug!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_ok",
|
||||
cert_index = idx,
|
||||
has_read_data_key = read_data_key.is_some(),
|
||||
);
|
||||
return (
|
||||
Some(HandshakeResult {
|
||||
volume_id,
|
||||
read_data_key,
|
||||
}),
|
||||
None,
|
||||
);
|
||||
}
|
||||
Err(_) => {
|
||||
Err(e) => {
|
||||
let code = e.code();
|
||||
last_err_code = Some(code);
|
||||
// Drive wedge senses (any with high byte 0x05 =
|
||||
// ILLEGAL_REQUEST). The drive isn't merely
|
||||
// rejecting our cert — it's saying "I won't talk
|
||||
// to you anymore." Trying more certs makes the
|
||||
// wedge worse. Bail out immediately.
|
||||
let sense_key = ((code >> 8) & 0xFF) as u8;
|
||||
if sense_key == 0x05 {
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_wedge_detected",
|
||||
cert_index = idx,
|
||||
error_code = code,
|
||||
"drive returned ILLEGAL_REQUEST during auth; bailing out to avoid wedge"
|
||||
);
|
||||
return (None, Some(Error::AacsHostCertRejected));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
// All host certs failed — return None, not a fake success
|
||||
None
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "handshake_all_certs_failed",
|
||||
host_cert_count,
|
||||
tried = host_cert_count.min(MAX_CERT_ATTEMPTS),
|
||||
last_error_code = last_err_code,
|
||||
"all host certs in KEYDB rejected by drive (capped at {} attempts to prevent firmware wedge)",
|
||||
MAX_CERT_ATTEMPTS
|
||||
);
|
||||
(None, Some(Error::AacsHostCertRejected))
|
||||
}
|
||||
|
||||
/// Resolve disc encryption — AACS 1.0, AACS 2.0, CSS, or none.
|
||||
///
|
||||
/// Reads AACS files from UDF (via SectorReader), resolves keys through
|
||||
/// whatever path works: KEYDB VUK lookup, media key derivation, processing
|
||||
/// keys, device keys. Uses handshake result (volume ID, bus key) if available.
|
||||
pub(super) fn resolve_encryption(
|
||||
/// Build a keys-free AACS state that carries only the Volume ID (+ version
|
||||
/// metadata), for callers that resolve Unit Keys out-of-band and have
|
||||
/// disabled the local keydb. The VID is on-disc content read during the
|
||||
/// handshake; preserving it here lets the out-of-band path use it. No keys
|
||||
/// are present (`unit_keys` empty, `vuk` None), so the disc reports as
|
||||
/// "encrypted, no keys" until the caller re-scans with a resolved Unit Key.
|
||||
pub(super) fn resolve_vid_only(
|
||||
udf_fs: &udf::UdfFs,
|
||||
reader: &mut dyn SectorReader,
|
||||
keydb_path: &std::path::Path,
|
||||
reader: &mut dyn SectorSource,
|
||||
handshake: Option<&HandshakeResult>,
|
||||
) -> Result<AacsState> {
|
||||
use crate::aacs::{self, KeyDb};
|
||||
use crate::aacs;
|
||||
|
||||
let keydb = KeyDb::load(keydb_path).map_err(|_| Error::KeydbLoad {
|
||||
path: keydb_path.display().to_string(),
|
||||
})?;
|
||||
|
||||
// Read AACS files from disc/image via UDF
|
||||
let uk_ro_data = udf_fs
|
||||
.read_file(reader, "/AACS/Unit_Key_RO.inf")
|
||||
.or_else(|_| udf_fs.read_file(reader, "/AACS/DUPLICATE/Unit_Key_RO.inf"))
|
||||
.map_err(|_| Error::AacsNoKeys)?;
|
||||
let dh = aacs::disc_hash(&uk_ro_data);
|
||||
|
||||
let cc_data = udf_fs
|
||||
let cc = udf_fs
|
||||
.read_file(reader, "/AACS/Content000.cer")
|
||||
.or_else(|_| udf_fs.read_file(reader, "/AACS/Content001.cer"))
|
||||
.ok();
|
||||
.ok()
|
||||
.as_deref()
|
||||
.and_then(aacs::parse_content_cert);
|
||||
let bus_encryption = cc.as_ref().map(|c| c.bus_encryption).unwrap_or(false);
|
||||
let version = match cc.as_ref().map(|c| c.version) {
|
||||
Some(aacs::AacsVersion::V10) => 1,
|
||||
Some(_) => 2,
|
||||
None if bus_encryption => 2,
|
||||
None => 1,
|
||||
};
|
||||
// MKB_RO/RW are allocated to a fixed ~128 MiB and zero-padded; trim to
|
||||
// the real record length (same as `read_aacs_inputs`). Without this the
|
||||
// MKB stashed on `AacsState` — which `Disc::inputs()` and the device/
|
||||
// processing-key `decrypt_with` derivation consume, and which a key
|
||||
// source ships to an online service — is the full 128 MiB pad, not the
|
||||
// ~few-MB record stream.
|
||||
let mut mkb_bytes = udf_fs
|
||||
.read_file(reader, "/AACS/MKB_RO.inf")
|
||||
.or_else(|_| udf_fs.read_file(reader, "/AACS/MKB_RW.inf"))
|
||||
.ok()
|
||||
.unwrap_or_default();
|
||||
let n = aacs::mkb_content_len(&mkb_bytes);
|
||||
if n > 0 && n < mkb_bytes.len() {
|
||||
mkb_bytes.truncate(n);
|
||||
}
|
||||
let mkb_ver = aacs::mkb_version(&mkb_bytes);
|
||||
|
||||
let mkb_data = udf_fs
|
||||
.read_file(reader, "/AACS/MKB_RW.inf")
|
||||
.or_else(|_| udf_fs.read_file(reader, "/AACS/MKB_RO.inf"))
|
||||
.ok();
|
||||
let mkb_ver = mkb_data.as_deref().and_then(aacs::mkb_version);
|
||||
|
||||
// Use handshake volume ID if available, otherwise zeros
|
||||
// (KEYDB VUK lookup by disc hash works without volume ID)
|
||||
let volume_id = handshake.map(|h| h.volume_id).unwrap_or([0u8; 16]);
|
||||
let read_data_key = handshake.and_then(|h| h.read_data_key);
|
||||
|
||||
// Resolve: tries all available paths — KEYDB VUK, media key, processing key, device key
|
||||
let resolved = aacs::resolve_keys(
|
||||
&uk_ro_data,
|
||||
cc_data.as_deref(),
|
||||
&volume_id,
|
||||
&keydb,
|
||||
mkb_data.as_deref(),
|
||||
)
|
||||
.ok_or(Error::AacsNoKeys)?;
|
||||
tracing::warn!(
|
||||
target: "freemkv::disc",
|
||||
phase = "scan_aacs_vid_only",
|
||||
disc_hash = %aacs::disc_hash_hex(&dh),
|
||||
version,
|
||||
bus_encryption,
|
||||
has_vid = handshake.is_some(),
|
||||
"keydb disabled — carrying VID only, keys resolved out-of-band"
|
||||
);
|
||||
|
||||
Ok(AacsState {
|
||||
version: if resolved.aacs2 { 2 } else { 1 },
|
||||
bus_encryption: resolved.bus_encryption,
|
||||
version,
|
||||
bus_encryption,
|
||||
mkb_version: mkb_ver,
|
||||
disc_hash: aacs::disc_hash_hex(&resolved.disc_hash),
|
||||
key_source: match resolved.key_source {
|
||||
1 => KeySource::KeyDb,
|
||||
2 => KeySource::KeyDbDerived,
|
||||
3 => KeySource::ProcessingKey,
|
||||
4 => KeySource::DeviceKey,
|
||||
_ => KeySource::KeyDb,
|
||||
},
|
||||
vuk: resolved.vuk,
|
||||
unit_keys: resolved.unit_keys,
|
||||
read_data_key,
|
||||
volume_id,
|
||||
disc_hash: aacs::disc_hash_hex(&dh),
|
||||
key_source: KeyOrigin::ExternalUk,
|
||||
vuk: None,
|
||||
unit_keys: vec![],
|
||||
read_data_key: handshake.and_then(|h| h.read_data_key),
|
||||
volume_id: handshake.map(|h| h.volume_id).unwrap_or([0u8; 16]),
|
||||
uk_ro: uk_ro_data,
|
||||
mkb: mkb_bytes,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+396
-8
@@ -17,11 +17,22 @@
|
||||
//!
|
||||
//! Status chars: `?` non-tried · `*` non-trimmed · `/` non-scraped · `-` unreadable · `+` finished.
|
||||
//!
|
||||
//! The mapfile is flushed to disk on every `record()` call so a crashed
|
||||
//! rip loses at most one block of recorded state.
|
||||
//! The mapfile is flushed to disk at most once per `FLUSH_INTERVAL`
|
||||
//! during `record()` calls, plus on explicit `flush()` and on `Drop`.
|
||||
//! This bounds atomic-rename RPC rate on networked staging (e.g. NFS)
|
||||
//! where per-record persists otherwise serialize the rip pipeline.
|
||||
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Minimum interval between mapfile persists. `record()` updates in-memory
|
||||
/// state every call but only writes to disk when this interval has elapsed
|
||||
/// since the last persist (or when `flush()` is called explicitly, or on
|
||||
/// `Drop`). Bounds RPC rate on NFS staging where atomic-rename per record
|
||||
/// otherwise dominates throughput. On crash the worst-case progress loss
|
||||
/// is one interval's worth of records.
|
||||
const FLUSH_INTERVAL: Duration = Duration::from_millis(1000);
|
||||
|
||||
/// Status of a byte range in the mapfile. ddrescue-compatible.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -91,15 +102,49 @@ pub struct MapStats {
|
||||
/// retry" UI bucket; `bytes_pending` over-counts because it folds
|
||||
/// in `bytes_nontried`.
|
||||
pub bytes_retryable: u64,
|
||||
/// Number of unreadable ranges (for UI display). Computed from
|
||||
/// `ranges_with(&[Unreadable])`.
|
||||
pub num_bad_ranges: u32,
|
||||
/// Largest gap among unreadable ranges in milliseconds. Computed as
|
||||
/// largest range size / bytes_per_sec * 1000. Set by caller (autorip)
|
||||
/// since bytes_per_sec is application-specific.
|
||||
pub main_lost_ms: f64,
|
||||
}
|
||||
|
||||
/// Write-through mapfile. Every `record()` persists to disk immediately
|
||||
/// so a crash during rip loses at most one block.
|
||||
/// Time-batched mapfile. `record()` keeps in-memory state up-to-date on
|
||||
/// every call; persists to disk at most once per `FLUSH_INTERVAL`.
|
||||
/// Explicit `flush()` and `Drop` guarantee state is on disk after a sweep
|
||||
/// or patch finishes. On hard crash the worst-case loss is one flush
|
||||
/// interval of records — the file's payload bytes are unaffected.
|
||||
pub struct Mapfile {
|
||||
path: PathBuf,
|
||||
entries: Vec<MapEntry>,
|
||||
total_size: u64,
|
||||
version: String,
|
||||
/// Incrementally maintained stats — updated on every `record()` call
|
||||
/// so `stats()` is O(1) instead of O(n).
|
||||
stats: MapStats,
|
||||
/// True when in-memory state has changed but `write_to_disk` has not
|
||||
/// yet captured it.
|
||||
dirty: bool,
|
||||
/// Wall-clock timestamp of the last successful `write_to_disk` (or
|
||||
/// the moment the mapfile was constructed, whichever is later).
|
||||
last_flushed: Instant,
|
||||
/// AACS Volume ID (16 bytes) for the disc, persisted as a
|
||||
/// `# freemkv-vid:` comment header so it survives to deferred-mux /
|
||||
/// resume without altering the ISO payload or breaking ddrescue
|
||||
/// data-line parsing. `None` for unencrypted / non-AACS discs.
|
||||
///
|
||||
/// MUTUALLY EXCLUSIVE with `unit_keys`: a disc whose keys were resolved
|
||||
/// persists the keys (`unit_keys`) and NOT the VID — the keys are the final
|
||||
/// answer, so deferred-mux/resume decrypts directly with no key service. A
|
||||
/// disc that did NOT resolve persists only the VID, the retry-able "still
|
||||
/// need a key" marker (a future mux can re-ask the key service with it).
|
||||
vid: Option<[u8; 16]>,
|
||||
/// Decrypted AACS unit keys `(CPS unit, key)`, persisted as `# freemkv-uk:`
|
||||
/// comment headers when the disc was successfully keyed. Mutually exclusive
|
||||
/// with `vid` (see above). Empty when unresolved.
|
||||
unit_keys: Vec<(u32, [u8; 16])>,
|
||||
}
|
||||
|
||||
impl Mapfile {
|
||||
@@ -107,7 +152,7 @@ impl Mapfile {
|
||||
/// Writes to disk immediately so a resume can pick up even if the caller
|
||||
/// never records anything.
|
||||
pub fn create(path: &Path, total_size: u64, version: &str) -> io::Result<Self> {
|
||||
let mf = Self {
|
||||
let mut mf = Self {
|
||||
path: path.to_path_buf(),
|
||||
entries: vec![MapEntry {
|
||||
pos: 0,
|
||||
@@ -116,8 +161,21 @@ impl Mapfile {
|
||||
}],
|
||||
total_size,
|
||||
version: version.to_string(),
|
||||
stats: MapStats {
|
||||
bytes_total: total_size,
|
||||
bytes_pending: total_size,
|
||||
bytes_nontried: total_size,
|
||||
..Default::default()
|
||||
},
|
||||
dirty: false,
|
||||
last_flushed: Instant::now(),
|
||||
vid: None,
|
||||
unit_keys: Vec::new(),
|
||||
};
|
||||
// Eager initial persist so a resume can pick this up even if
|
||||
// `record()` is never called.
|
||||
mf.write_to_disk()?;
|
||||
mf.last_flushed = Instant::now();
|
||||
Ok(mf)
|
||||
}
|
||||
|
||||
@@ -127,6 +185,8 @@ impl Mapfile {
|
||||
let mut entries = Vec::new();
|
||||
let mut saw_current_line = false;
|
||||
let mut version = String::from("unknown");
|
||||
let mut vid: Option<[u8; 16]> = None;
|
||||
let mut unit_keys: Vec<(u32, [u8; 16])> = Vec::new();
|
||||
for line in text.lines() {
|
||||
let t = line.trim();
|
||||
if t.is_empty() {
|
||||
@@ -137,6 +197,17 @@ impl Mapfile {
|
||||
if let Some(v) = rest.strip_prefix("Rescue Logfile. Created by ") {
|
||||
version = v.to_string();
|
||||
}
|
||||
if let Some(hex) = rest.strip_prefix("freemkv-vid:") {
|
||||
// Best-effort: a malformed or short VID comment is
|
||||
// ignored rather than failing the whole load.
|
||||
vid = parse_vid_hex(hex.trim());
|
||||
}
|
||||
if let Some(uk) = rest.strip_prefix("freemkv-uk:") {
|
||||
// `<cps>:<32hex>`. Best-effort: a malformed line is skipped.
|
||||
if let Some(entry) = parse_uk_line(uk.trim()) {
|
||||
unit_keys.push(entry);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// First non-comment line is the "current" state line (pos status [pass] [pass_time]).
|
||||
@@ -177,11 +248,17 @@ impl Mapfile {
|
||||
}
|
||||
entries.sort_by_key(|e| e.pos);
|
||||
let total_size = entries.last().map(|e| e.pos + e.size).unwrap_or(0);
|
||||
let stats = Self::compute_stats(&entries, total_size);
|
||||
Ok(Self {
|
||||
path: path.to_path_buf(),
|
||||
entries,
|
||||
total_size,
|
||||
version,
|
||||
stats,
|
||||
dirty: false,
|
||||
last_flushed: Instant::now(),
|
||||
vid,
|
||||
unit_keys,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -244,11 +321,68 @@ impl Mapfile {
|
||||
merged.push(e);
|
||||
}
|
||||
|
||||
// Recompute stats from merged entries. record() is already O(n) due to
|
||||
// drain-and-rebuild, so this is a constant-factor overhead. The critical
|
||||
// win is that stats() is now O(1) — called millions of times in the hot
|
||||
// path during sweep/patch, it just returns the cached value.
|
||||
self.stats = Self::compute_stats(&merged, self.total_size);
|
||||
self.entries = merged;
|
||||
self.write_to_disk()?;
|
||||
self.dirty = true;
|
||||
if self.last_flushed.elapsed() >= FLUSH_INTERVAL {
|
||||
self.write_to_disk()?;
|
||||
self.dirty = false;
|
||||
self.last_flushed = Instant::now();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist any pending in-memory changes to disk. No-op if clean.
|
||||
/// Callers (sweep/patch finalisation) invoke this after their last
|
||||
/// `record()` to guarantee state is durable before returning.
|
||||
pub fn flush(&mut self) -> io::Result<()> {
|
||||
if self.dirty {
|
||||
self.write_to_disk()?;
|
||||
self.dirty = false;
|
||||
self.last_flushed = Instant::now();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Record the disc's 16-byte AACS Volume ID so it persists in the
|
||||
/// mapfile's comment header. Marks the mapfile dirty; the next
|
||||
/// `flush()` / `Drop` writes the `# freemkv-vid:` line. Does not
|
||||
/// touch the ISO payload or the ddrescue data lines.
|
||||
pub fn set_vid(&mut self, vid: [u8; 16]) {
|
||||
self.vid = Some(vid);
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
/// The disc's AACS Volume ID, if one was set or parsed from a
|
||||
/// `# freemkv-vid:` comment on load. `None` for unencrypted /
|
||||
/// non-AACS discs.
|
||||
pub fn vid(&self) -> Option<[u8; 16]> {
|
||||
self.vid
|
||||
}
|
||||
|
||||
/// Record the disc's decrypted AACS unit keys so they persist in the
|
||||
/// mapfile header (`# freemkv-uk:` lines). The KEYED state: a deferred-mux /
|
||||
/// resume decrypts directly from these with no key-service round-trip.
|
||||
/// Setting keys clears any VID — the mapfile holds keys XOR VID, never both
|
||||
/// (keys are the final answer; VID is only the "still unresolved" marker).
|
||||
pub fn set_unit_keys(&mut self, keys: &[(u32, [u8; 16])]) {
|
||||
self.unit_keys = keys.to_vec();
|
||||
if !self.unit_keys.is_empty() {
|
||||
self.vid = None;
|
||||
}
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
/// The disc's decrypted AACS unit keys, if the disc was keyed (parsed from
|
||||
/// `# freemkv-uk:` comments on load). Empty = unresolved (check `vid()`).
|
||||
pub fn unit_keys(&self) -> &[(u32, [u8; 16])] {
|
||||
&self.unit_keys
|
||||
}
|
||||
|
||||
pub fn entries(&self) -> &[MapEntry] {
|
||||
&self.entries
|
||||
}
|
||||
@@ -283,11 +417,15 @@ impl Mapfile {
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> MapStats {
|
||||
self.stats
|
||||
}
|
||||
|
||||
fn compute_stats(entries: &[MapEntry], total_size: u64) -> MapStats {
|
||||
let mut s = MapStats {
|
||||
bytes_total: self.total_size,
|
||||
bytes_total: total_size,
|
||||
..Default::default()
|
||||
};
|
||||
for e in &self.entries {
|
||||
for e in entries {
|
||||
match e.status {
|
||||
SectorStatus::Finished => s.bytes_good += e.size,
|
||||
SectorStatus::Unreadable => s.bytes_unreadable += e.size,
|
||||
@@ -317,6 +455,30 @@ impl Mapfile {
|
||||
let file = std::fs::File::create(&tmp)?;
|
||||
let mut w = std::io::BufWriter::new(file);
|
||||
writeln!(w, "# Rescue Logfile. Created by {}", self.version)?;
|
||||
// VID comment lives in the header block. ddrescue treats any
|
||||
// `#`-prefixed line as a comment, so this round-trips through
|
||||
// our `load()` without affecting the `pos size status` data
|
||||
// parser. 16 bytes → 32 lowercase hex chars.
|
||||
// KEYS XOR VID: a keyed disc persists its unit keys (the final
|
||||
// answer — deferred-mux decrypts directly); an unresolved disc
|
||||
// persists only the VID (the retry marker, so a future mux can
|
||||
// re-ask the key service). Never both.
|
||||
use std::fmt::Write as _;
|
||||
if !self.unit_keys.is_empty() {
|
||||
for (cps, key) in &self.unit_keys {
|
||||
let mut hex = String::with_capacity(32);
|
||||
for b in key {
|
||||
let _ = write!(hex, "{b:02x}");
|
||||
}
|
||||
writeln!(w, "# freemkv-uk: {cps}:{hex}")?;
|
||||
}
|
||||
} else if let Some(vid) = self.vid {
|
||||
let mut hex = String::with_capacity(32);
|
||||
for b in vid {
|
||||
let _ = write!(hex, "{b:02x}");
|
||||
}
|
||||
writeln!(w, "# freemkv-vid: {hex}")?;
|
||||
}
|
||||
writeln!(w, "# Current pos / status / pass / pass_time")?;
|
||||
writeln!(w, "0x000000000 ? 1 0")?;
|
||||
writeln!(w, "# pos size status")?;
|
||||
@@ -336,6 +498,41 @@ impl Mapfile {
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Mapfile {
|
||||
/// Best-effort flush on drop so a sweep / patch that returns early
|
||||
/// (or unwinds) doesn't lose its in-memory state. Errors here are
|
||||
/// swallowed because Drop has no way to surface them; explicit
|
||||
/// `flush()` on the success path gives callers proper error handling.
|
||||
fn drop(&mut self) {
|
||||
let _ = self.flush();
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a 32-char lowercase/uppercase hex string into a 16-byte VID.
|
||||
/// Returns `None` on any malformation (wrong length, non-hex) — the
|
||||
/// caller treats a bad VID comment as simply absent rather than an
|
||||
/// error, so a corrupt header never fails a mapfile load.
|
||||
fn parse_vid_hex(s: &str) -> Option<[u8; 16]> {
|
||||
let s = s.strip_prefix("0x").unwrap_or(s);
|
||||
if s.len() != 32 {
|
||||
return None;
|
||||
}
|
||||
let mut out = [0u8; 16];
|
||||
for (i, b) in out.iter_mut().enumerate() {
|
||||
*b = u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).ok()?;
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Parse a `# freemkv-uk:` value `<cps>:<32hex>` into `(cps_unit, key)`. Returns
|
||||
/// `None` on any malformation so a corrupt line is ignored, never fatal.
|
||||
fn parse_uk_line(s: &str) -> Option<(u32, [u8; 16])> {
|
||||
let (cps, hex) = s.split_once(':')?;
|
||||
let cps: u32 = cps.trim().parse().ok()?;
|
||||
let key = parse_vid_hex(hex.trim())?; // 32-hex → [u8; 16], shared parser
|
||||
Some((cps, key))
|
||||
}
|
||||
|
||||
fn parse_hex(s: &str) -> io::Result<u64> {
|
||||
let s = s.strip_prefix("0x").unwrap_or(s);
|
||||
u64::from_str_radix(s, 16).map_err(|_| {
|
||||
@@ -436,6 +633,8 @@ mod tests {
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
mf.record(100, 200, SectorStatus::Finished).unwrap();
|
||||
mf.record(500, 100, SectorStatus::Unreadable).unwrap();
|
||||
// record() batches; explicit flush before reading back from disk.
|
||||
mf.flush().unwrap();
|
||||
let loaded = Mapfile::load(&p).unwrap();
|
||||
assert_eq!(loaded.entries(), mf.entries());
|
||||
let _ = std::fs::remove_file(&p);
|
||||
@@ -467,4 +666,193 @@ mod tests {
|
||||
assert_eq!(bad, vec![(100, 50), (300, 50)]);
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_consistent_after_overlapping_records() {
|
||||
let p = tmpfile("stats_consistent_after_overlapping");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
// Record some finished, some unreadable, some nontrimmed
|
||||
mf.record(0, 300, SectorStatus::Finished).unwrap();
|
||||
mf.record(300, 200, SectorStatus::NonTrimmed).unwrap();
|
||||
mf.record(500, 100, SectorStatus::Unreadable).unwrap();
|
||||
mf.record(600, 400, SectorStatus::Finished).unwrap();
|
||||
|
||||
// Final entries: [0..300 Finished, 300..500 NonTrimmed, 500..600 Unreadable, 600..1000 Finished]
|
||||
let s = mf.stats();
|
||||
assert_eq!(s.bytes_good, 700); // 300 + 400
|
||||
assert_eq!(s.bytes_unreadable, 100); // 100
|
||||
assert_eq!(s.bytes_pending, 200); // NonTrimmed only (NonTried=0)
|
||||
assert_eq!(s.bytes_nontried, 0);
|
||||
assert_eq!(s.bytes_retryable, 200); // NonTrimmed
|
||||
assert_eq!(s.bytes_total, 1000);
|
||||
|
||||
// Overwrite a NonTrimmed range with Finished
|
||||
mf.record(300, 100, SectorStatus::Finished).unwrap();
|
||||
// Entries: [0..400 Finished, 400..500 NonTrimmed, 500..600 Unreadable, 600..1000 Finished]
|
||||
let s2 = mf.stats();
|
||||
assert_eq!(s2.bytes_good, 800); // 400 + 400
|
||||
assert_eq!(s2.bytes_unreadable, 100);
|
||||
assert_eq!(s2.bytes_pending, 100); // NonTrimmed only
|
||||
assert_eq!(s2.bytes_retryable, 100);
|
||||
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unit_keys_round_trip_and_are_mutually_exclusive_with_vid() {
|
||||
let p = tmpfile("uk_round_trips");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
mf.record(0, 500, SectorStatus::Finished).unwrap();
|
||||
// Set a VID first, then unit keys: keys must WIN and clear the VID.
|
||||
mf.set_vid([0xAA; 16]);
|
||||
let keys: Vec<(u32, [u8; 16])> = vec![
|
||||
(
|
||||
0,
|
||||
[
|
||||
0x57, 0x60, 0xcc, 0x83, 0x3d, 0x86, 0x0e, 0x48, 0x92, 0x1f, 0x88, 0x16, 0xe1,
|
||||
0x35, 0x9b, 0xad,
|
||||
],
|
||||
),
|
||||
(1, [0x11; 16]),
|
||||
];
|
||||
mf.set_unit_keys(&keys);
|
||||
assert_eq!(
|
||||
mf.vid(),
|
||||
None,
|
||||
"set_unit_keys must clear vid (keys XOR vid)"
|
||||
);
|
||||
mf.flush().unwrap();
|
||||
|
||||
let text = std::fs::read_to_string(&p).unwrap();
|
||||
assert!(
|
||||
text.contains("# freemkv-uk: 0:5760cc833d860e48921f8816e1359bad"),
|
||||
"uk comment format mismatch: {text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains("# freemkv-uk: 1:11111111111111111111111111111111"),
|
||||
"second uk missing: {text}"
|
||||
);
|
||||
assert!(
|
||||
!text.contains("# freemkv-vid:"),
|
||||
"VID must NOT be written when keys are present: {text}"
|
||||
);
|
||||
|
||||
// load() recovers the unit keys (and no VID).
|
||||
let loaded = Mapfile::load(&p).unwrap();
|
||||
assert_eq!(loaded.unit_keys(), keys.as_slice());
|
||||
assert_eq!(loaded.vid(), None);
|
||||
assert_eq!(loaded.entries(), mf.entries());
|
||||
|
||||
// VID-only path (no keys) still persists the VID as the retry marker.
|
||||
let p2 = tmpfile("uk_vid_only");
|
||||
let _ = std::fs::remove_file(&p2);
|
||||
let mut mf2 = Mapfile::create(&p2, 1000, "test").unwrap();
|
||||
mf2.set_vid([0xBB; 16]);
|
||||
mf2.flush().unwrap();
|
||||
let loaded2 = Mapfile::load(&p2).unwrap();
|
||||
assert_eq!(loaded2.vid(), Some([0xBB; 16]));
|
||||
assert!(loaded2.unit_keys().is_empty());
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let _ = std::fs::remove_file(&p2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vid_round_trips_and_data_lines_unaffected() {
|
||||
let p = tmpfile("vid_round_trips");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
|
||||
// Build a mapfile with some data ranges, set a VID, persist.
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
mf.record(100, 200, SectorStatus::Finished).unwrap();
|
||||
mf.record(500, 100, SectorStatus::Unreadable).unwrap();
|
||||
mf.record(700, 50, SectorStatus::NonTrimmed).unwrap();
|
||||
let vid: [u8; 16] = [
|
||||
0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd,
|
||||
0xee, 0xff,
|
||||
];
|
||||
mf.set_vid(vid);
|
||||
mf.flush().unwrap();
|
||||
|
||||
// The saved file must contain the VID comment in lowercase hex.
|
||||
let text = std::fs::read_to_string(&p).unwrap();
|
||||
assert!(
|
||||
text.contains("# freemkv-vid:"),
|
||||
"saved mapfile missing VID comment: {text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains("# freemkv-vid: 00112233445566778899aabbccddeeff"),
|
||||
"VID comment format mismatch: {text}"
|
||||
);
|
||||
|
||||
// load() recovers the VID and the identical data ranges.
|
||||
let loaded = Mapfile::load(&p).unwrap();
|
||||
assert_eq!(loaded.vid(), Some(vid));
|
||||
assert_eq!(loaded.entries(), mf.entries());
|
||||
|
||||
// A mapfile WITHOUT the VID comment must parse the same +/-/?
|
||||
// data ranges as the one WITH it (comment ignored by parser).
|
||||
let p2 = tmpfile("vid_round_trips_novid");
|
||||
let _ = std::fs::remove_file(&p2);
|
||||
let mut mf2 = Mapfile::create(&p2, 1000, "test").unwrap();
|
||||
mf2.record(100, 200, SectorStatus::Finished).unwrap();
|
||||
mf2.record(500, 100, SectorStatus::Unreadable).unwrap();
|
||||
mf2.record(700, 50, SectorStatus::NonTrimmed).unwrap();
|
||||
mf2.flush().unwrap();
|
||||
let loaded_novid = Mapfile::load(&p2).unwrap();
|
||||
assert_eq!(loaded_novid.vid(), None);
|
||||
assert_eq!(loaded_novid.entries(), loaded.entries());
|
||||
|
||||
// Malformed VID comments must not error the load (treated absent).
|
||||
let mut bad = text.replace("00112233445566778899aabbccddeeff", "zzzz");
|
||||
let pbad = tmpfile("vid_round_trips_bad");
|
||||
let _ = std::fs::remove_file(&pbad);
|
||||
std::fs::write(&pbad, &bad).unwrap();
|
||||
let loaded_bad = Mapfile::load(&pbad).unwrap();
|
||||
assert_eq!(loaded_bad.vid(), None);
|
||||
assert_eq!(loaded_bad.entries(), loaded.entries());
|
||||
|
||||
// A load->save cycle preserves the VID (the patch-pass path).
|
||||
bad.clear();
|
||||
let resaved = tmpfile("vid_round_trips_resave");
|
||||
let _ = std::fs::remove_file(&resaved);
|
||||
let mut reloaded = Mapfile::load(&p).unwrap();
|
||||
// Repoint at a fresh path and flush; mark dirty via a no-op record.
|
||||
reloaded.path = resaved.clone();
|
||||
reloaded.dirty = true;
|
||||
reloaded.flush().unwrap();
|
||||
let again = Mapfile::load(&resaved).unwrap();
|
||||
assert_eq!(again.vid(), Some(vid));
|
||||
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let _ = std::fs::remove_file(&p2);
|
||||
let _ = std::fs::remove_file(&pbad);
|
||||
let _ = std::fs::remove_file(&resaved);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stats_consistent_after_split_record() {
|
||||
let p = tmpfile("stats_consistent_after_split");
|
||||
let _ = std::fs::remove_file(&p);
|
||||
let mut mf = Mapfile::create(&p, 1000, "test").unwrap();
|
||||
// Mark middle as NonTrimmed
|
||||
mf.record(200, 400, SectorStatus::NonTrimmed).unwrap();
|
||||
// Entries: [0..200 NonTried, 200..600 NonTrimmed, 600..1000 NonTried]
|
||||
let s = mf.stats();
|
||||
assert_eq!(s.bytes_pending, 1000); // NonTried(600) + NonTrimmed(400)
|
||||
assert_eq!(s.bytes_retryable, 400); // NonTrimmed only
|
||||
assert_eq!(s.bytes_nontried, 600); // 200 + 400
|
||||
|
||||
// Overwrite the NonTrimmed with Finished (splitting the remaining NonTried)
|
||||
mf.record(200, 400, SectorStatus::Finished).unwrap();
|
||||
// Entries: [0..200 NonTried, 200..600 Finished, 600..1000 NonTried]
|
||||
let s2 = mf.stats();
|
||||
assert_eq!(s2.bytes_good, 400);
|
||||
assert_eq!(s2.bytes_pending, 600); // NonTried(200 + 400)
|
||||
assert_eq!(s2.bytes_nontried, 600);
|
||||
assert_eq!(s2.bytes_retryable, 0);
|
||||
|
||||
let _ = std::fs::remove_file(&p);
|
||||
}
|
||||
}
|
||||
|
||||
+1665
-806
File diff suppressed because it is too large
Load Diff
+1787
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,239 @@
|
||||
//! `Disc::sweep`'s consumer-side `Sink<WorkItem>`.
|
||||
//!
|
||||
//! Background: the original sweep loop runs strictly serialised —
|
||||
//! SCSI read → decrypt → seek + write → mapfile.record → next iter.
|
||||
//! On a healthy disc the SCSI read costs ~5-12 ms per 64 KB batch and
|
||||
//! the post-read work (decrypt 1-3 ms + file write + mapfile fsync
|
||||
//! 5-15 ms) adds another batch's worth of latency. The drive idles
|
||||
//! during the post-read work; throughput tops out at the *sum* of
|
||||
//! both costs.
|
||||
//!
|
||||
//! 0.17.11 introduced a bespoke producer/consumer split (the now-
|
||||
//! removed `disc/sweep_pipeline.rs`) to overlap the two stages. 0.18
|
||||
//! collapses that split — together with the analogous splits patch
|
||||
//! and mux need — onto the generic [`crate::io::Pipeline`] +
|
||||
//! [`crate::io::Sink`] primitive. This module is the sweep-specific
|
||||
//! `Sink` impl; the producer-side state machine (read_error context,
|
||||
//! decrypt, set_speed, halt) stays in `Disc::sweep` in `disc/mod.rs`.
|
||||
//!
|
||||
//! Correctness invariants preserved (same as 0.17.11):
|
||||
//! - Mapfile is single-writer (consumer-only). No locking.
|
||||
//! - All `read_error::ReadCtx` state stays on the producer thread.
|
||||
//! - `set_speed` calls happen on the producer thread (same thread that
|
||||
//! owns the `SectorSource`). No new SCSI concurrency.
|
||||
//! - Per-iteration ordering of file-write → mapfile-record is kept
|
||||
//! intact in the consumer (write before record), so the on-disk
|
||||
//! invariant "mapfile only marks Finished what the file has
|
||||
//! received" survives a crash mid-pass.
|
||||
//! - The BU40N+Initio bridge wedge concern is unchanged: only one
|
||||
//! SCSI command in flight at a time, error-path timing identical,
|
||||
//! no new retry logic.
|
||||
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
use std::sync::mpsc::{Receiver, SyncSender, sync_channel};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::io::{Flow, Sink};
|
||||
|
||||
use super::mapfile::{MapStats, Mapfile, SectorStatus};
|
||||
|
||||
/// Reusable zero buffer for SkipFill / GapFill / BisectBad. 64 KB
|
||||
/// matches the existing zero_gap chunk size used by the pre-split
|
||||
/// sweep loop.
|
||||
const ZERO_CHUNK: usize = 65 * 1024;
|
||||
|
||||
/// Producer → Consumer messages. The consumer applies these in FIFO
|
||||
/// order; ordering of file writes and mapfile records across items is
|
||||
/// preserved.
|
||||
pub(super) enum WorkItem {
|
||||
/// Successful batch read. Producer has already decrypted `buf` if
|
||||
/// `opts.decrypt` was set. Consumer writes `buf` at `pos` and
|
||||
/// records the range as `Finished`.
|
||||
Good { pos: u64, buf: Vec<u8> },
|
||||
|
||||
/// Bisect inner-loop good single sector (already decrypted by the
|
||||
/// producer). 2048 bytes.
|
||||
BisectGood { pos: u64, buf: Box<[u8; 2048]> },
|
||||
|
||||
/// Bisect inner-loop bad single sector. Consumer writes 2048
|
||||
/// zeros at `pos` and records the sector as `NonTrimmed`.
|
||||
BisectBad { pos: u64 },
|
||||
|
||||
/// Whole-batch zero-fill (failed batch on `SkipBlock`, or the
|
||||
/// failed batch portion of `JumpAhead`). Consumer streams zeros
|
||||
/// across `[pos, pos+len)` and records the range as `NonTrimmed`.
|
||||
SkipFill { pos: u64, len: u64 },
|
||||
|
||||
/// Gap fill following a `JumpAhead`. Same effect as `SkipFill`;
|
||||
/// distinguished only so future logging / instrumentation can
|
||||
/// tell them apart without parsing a flag.
|
||||
GapFill { pos: u64, len: u64 },
|
||||
|
||||
/// Producer wants the latest mapfile stats for the progress
|
||||
/// callback. Consumer responds on `prog_tx` with a fresh
|
||||
/// [`ProgressSnapshot`]. Best-effort: if the producer hasn't
|
||||
/// drained the previous snapshot, the new one is silently
|
||||
/// dropped — the producer's local cache stays current enough.
|
||||
StatsRequest,
|
||||
}
|
||||
|
||||
/// Snapshot the consumer sends back to the producer for the progress
|
||||
/// callback.
|
||||
pub(super) struct ProgressSnapshot {
|
||||
pub stats: MapStats,
|
||||
pub bad_ranges: Vec<(u64, u64)>,
|
||||
}
|
||||
|
||||
/// Final summary returned by the consumer thread on shutdown — what
|
||||
/// `SweepSink::close` produces, surfaced to the producer via
|
||||
/// `Pipeline::finish`.
|
||||
pub(super) struct ConsumerSummary {
|
||||
pub stats: MapStats,
|
||||
}
|
||||
|
||||
/// Drain any pending progress snapshots from the consumer. Returns
|
||||
/// the most recent one, if any. The producer caches it and uses it
|
||||
/// for subsequent progress callbacks until a fresh one arrives.
|
||||
pub(super) fn try_recv_progress(rx: &Receiver<ProgressSnapshot>) -> Option<ProgressSnapshot> {
|
||||
let mut latest = None;
|
||||
while let Ok(snap) = rx.try_recv() {
|
||||
latest = Some(snap);
|
||||
}
|
||||
latest
|
||||
}
|
||||
|
||||
/// `Sink<WorkItem>` for sweep. Owns the writeback file + mapfile +
|
||||
/// progress back-channel. `apply` carries the file-write +
|
||||
/// mapfile.record per item; `close` drains the writeback pipeline,
|
||||
/// fsyncs the ISO, and flushes the mapfile.
|
||||
pub(super) struct SweepSink {
|
||||
file: crate::io::WritebackFile,
|
||||
map: Mapfile,
|
||||
/// `sync_all`-on-failure-is-an-error iff the output is a regular
|
||||
/// file. `/dev/null` and pipes always fail `sync_all`; that's not
|
||||
/// a real error.
|
||||
is_regular: bool,
|
||||
/// Back-channel for `StatsRequest` responses. The producer caches
|
||||
/// the latest snapshot and uses it for the progress callback;
|
||||
/// dropped sends on a full channel are by design.
|
||||
prog_tx: SyncSender<ProgressSnapshot>,
|
||||
/// Reusable zero buffer for SkipFill / GapFill / BisectBad. Held
|
||||
/// in the sink so each apply call doesn't reallocate.
|
||||
zero: Box<[u8; ZERO_CHUNK]>,
|
||||
}
|
||||
|
||||
impl SweepSink {
|
||||
/// Construct a new `SweepSink` plus the matching progress
|
||||
/// receiver. Channel depth on the back-channel is `1` — the
|
||||
/// producer's cache is the source of truth between snapshots.
|
||||
pub(super) fn new(
|
||||
file: crate::io::WritebackFile,
|
||||
map: Mapfile,
|
||||
is_regular: bool,
|
||||
) -> (Self, Receiver<ProgressSnapshot>) {
|
||||
let (prog_tx, prog_rx) = sync_channel::<ProgressSnapshot>(1);
|
||||
let sink = SweepSink {
|
||||
file,
|
||||
map,
|
||||
is_regular,
|
||||
prog_tx,
|
||||
zero: Box::new([0u8; ZERO_CHUNK]),
|
||||
};
|
||||
(sink, prog_rx)
|
||||
}
|
||||
}
|
||||
|
||||
impl Sink<WorkItem> for SweepSink {
|
||||
type Output = ConsumerSummary;
|
||||
|
||||
fn apply(&mut self, item: WorkItem) -> Result<Flow, Error> {
|
||||
match item {
|
||||
WorkItem::Good { pos, buf } => {
|
||||
// Decrypt is on the producer; consumer assumes plaintext.
|
||||
let len = buf.len() as u64;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&buf)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, len, SectorStatus::Finished)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::BisectGood { pos, buf } => {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&buf[..])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, 2048, SectorStatus::Finished)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::BisectBad { pos } => {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.write_all(&self.zero[..2048])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.map
|
||||
.record(pos, 2048, SectorStatus::NonTrimmed)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::SkipFill { pos, len } | WorkItem::GapFill { pos, len } => {
|
||||
self.file
|
||||
.seek(SeekFrom::Start(pos))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
// Subsequent writes are sequential; `WritebackFile`'s
|
||||
// seek-elision keeps them on the writeback pipeline path.
|
||||
let mut filled = 0u64;
|
||||
while filled < len {
|
||||
let chunk = (len - filled).min(self.zero.len() as u64) as usize;
|
||||
self.file
|
||||
.write_all(&self.zero[..chunk])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
filled += chunk as u64;
|
||||
}
|
||||
self.map
|
||||
.record(pos, len, SectorStatus::NonTrimmed)
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
}
|
||||
WorkItem::StatsRequest => {
|
||||
let stats = self.map.stats();
|
||||
let bad_ranges = self.map.ranges_with(&[
|
||||
SectorStatus::NonTrimmed,
|
||||
SectorStatus::Unreadable,
|
||||
SectorStatus::NonScraped,
|
||||
SectorStatus::NonTried,
|
||||
]);
|
||||
// Best-effort: drop on backpressure; producer's cache
|
||||
// stays current enough.
|
||||
let _ = self
|
||||
.prog_tx
|
||||
.try_send(ProgressSnapshot { stats, bad_ranges });
|
||||
}
|
||||
}
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(mut self) -> Result<Self::Output, Error> {
|
||||
// Drain the writeback pipeline + fsync the ISO, then persist
|
||||
// any pending mapfile state. Same finalisation order as the
|
||||
// pre-Pipeline consumer loop.
|
||||
if let Err(e) = self.file.sync_all() {
|
||||
if self.is_regular {
|
||||
return Err(Error::IoError { source: e });
|
||||
}
|
||||
// Non-regular outputs (/dev/null, pipes) always fail
|
||||
// sync_all; that's not a real error.
|
||||
}
|
||||
self.map.flush().map_err(|e| Error::IoError { source: e })?;
|
||||
|
||||
Ok(ConsumerSummary {
|
||||
stats: self.map.stats(),
|
||||
})
|
||||
}
|
||||
}
|
||||
+148
-2
@@ -29,7 +29,7 @@ use crate::platform::PlatformDriver;
|
||||
use crate::platform::mt1959::Mt1959;
|
||||
use crate::profile::{self, DriveProfile};
|
||||
use crate::scsi::ScsiTransport;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
@@ -69,6 +69,16 @@ pub struct Drive {
|
||||
halt: Arc<AtomicBool>,
|
||||
/// Event handler — fires for read errors and library-level state changes.
|
||||
event_fn: Option<Box<dyn Fn(Event) + Send>>,
|
||||
/// Linux only: raw fd for the corresponding block device (`/dev/sr*`)
|
||||
/// used as a recovery fallback when SCSI READ via `/dev/sg*` returns
|
||||
/// an error. The kernel `sr_mod` driver auto-retries failed reads
|
||||
/// (~5× per command) — historically the reason `dd if=/dev/sr0`
|
||||
/// recovers ~50% of bad sectors that single-shot `SG_IO` READ
|
||||
/// misses on the same drive. `None` when the block device couldn't
|
||||
/// be resolved or opened (no fallback in that case; SCSI read
|
||||
/// errors propagate as before).
|
||||
#[cfg(target_os = "linux")]
|
||||
block_dev_fd: Option<std::os::unix::io::RawFd>,
|
||||
}
|
||||
|
||||
impl Drive {
|
||||
@@ -87,6 +97,9 @@ impl Drive {
|
||||
None => (None, None, None),
|
||||
};
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let block_dev_fd = open_block_device_for_sg(device);
|
||||
|
||||
Ok(Drive {
|
||||
scsi: transport,
|
||||
driver,
|
||||
@@ -96,6 +109,8 @@ impl Drive {
|
||||
device_path: device.to_string_lossy().to_string(),
|
||||
halt: Arc::new(AtomicBool::new(false)),
|
||||
event_fn: None,
|
||||
#[cfg(target_os = "linux")]
|
||||
block_dev_fd,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -169,6 +184,13 @@ impl Drive {
|
||||
self.profile.is_some()
|
||||
}
|
||||
|
||||
/// Borrow the matched drive profile, if any. Used by callers that
|
||||
/// need to issue per-drive OEM CDB templates (e.g. the OEM VID
|
||||
/// retrieval path in `disc::encrypt`).
|
||||
pub fn drive_profile(&self) -> Option<&DriveProfile> {
|
||||
self.profile.as_ref()
|
||||
}
|
||||
|
||||
/// Access the SCSI transport for direct commands (used by CSS/AACS auth).
|
||||
pub fn scsi_mut(&mut self) -> &mut dyn ScsiTransport {
|
||||
self.scsi.as_mut()
|
||||
@@ -417,6 +439,27 @@ impl Drive {
|
||||
}
|
||||
}
|
||||
|
||||
/// True if the drive is currently in the extended-access state.
|
||||
///
|
||||
/// Detected by the platform driver during `init()` from the unlock
|
||||
/// response's mode markers. When true:
|
||||
/// - SCSI READ_10 returns plaintext sectors (no AACS bus
|
||||
/// encryption applied)
|
||||
/// - VID retrieval works via the per-drive OEM CDB in
|
||||
/// [`DriveProfile`] without the cert-based AACS handshake
|
||||
/// - Disc-side Host Revocation List enforcement is effectively
|
||||
/// bypassed by the alternate data path
|
||||
///
|
||||
/// AACS layer code branches on this: if true, issue the OEM
|
||||
/// `read_vid_cdb` to retrieve VID directly; if false, fall back
|
||||
/// to the cert-based mutual-auth handshake.
|
||||
pub fn is_unlocked(&self) -> bool {
|
||||
match self.driver {
|
||||
Some(ref d) => d.is_unlocked(),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read sectors from the disc. Single-shot — no inline retries, no
|
||||
/// SCSI reset.
|
||||
///
|
||||
@@ -483,6 +526,54 @@ impl Drive {
|
||||
scsi_status = status,
|
||||
"Drive::read checked_exec failed"
|
||||
);
|
||||
|
||||
// /dev/sr0 pread fallback (Linux only). The kernel
|
||||
// sr_mod driver auto-retries failed reads (~5× per
|
||||
// command). Empirically (BU40N + a UHD disc,
|
||||
// 2026-05-08) dd via /dev/sr0 recovers ~50% of bad
|
||||
// sectors that a single-shot SG_IO READ misses.
|
||||
#[cfg(target_os = "linux")]
|
||||
if recovery {
|
||||
if let Some(fd) = self.block_dev_fd {
|
||||
let len = count as usize * 2048;
|
||||
if buf.len() >= len {
|
||||
let offset = lba as i64 * 2048;
|
||||
// Drop kernel cache for this region so we get
|
||||
// a fresh device read, not stale page-cache
|
||||
// data from a prior successful neighbour read.
|
||||
let _ = unsafe {
|
||||
libc::posix_fadvise(
|
||||
fd,
|
||||
offset,
|
||||
len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
)
|
||||
};
|
||||
let n = unsafe {
|
||||
libc::pread(fd, buf.as_mut_ptr() as *mut libc::c_void, len, offset)
|
||||
};
|
||||
if n == len as isize {
|
||||
tracing::info!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
bytes = len,
|
||||
"Drive::read recovered via /dev/sr0 pread fallback"
|
||||
);
|
||||
return Ok(len);
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "freemkv::drive",
|
||||
lba,
|
||||
count,
|
||||
pread_ret = n as i64,
|
||||
errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0),
|
||||
"/dev/sr0 pread fallback also failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::DiscRead {
|
||||
sector: lba as u64,
|
||||
status: Some(status),
|
||||
@@ -586,10 +677,65 @@ impl Drop for Drive {
|
||||
fn drop(&mut self) {
|
||||
self.cleanup();
|
||||
// SgIoTransport::drop() runs next, calling libc::close(fd)
|
||||
#[cfg(target_os = "linux")]
|
||||
if let Some(fd) = self.block_dev_fd.take() {
|
||||
unsafe { libc::close(fd) };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for Drive {
|
||||
/// Resolve a `/dev/sg*` path to the corresponding `/dev/sr*` block
|
||||
/// device by walking sysfs, then open it for read (no `O_DIRECT` —
|
||||
/// `posix_fadvise(POSIX_FADV_DONTNEED)` flushes the cache before each
|
||||
/// pread, which avoids buffer-alignment requirements while still
|
||||
/// forcing fresh device reads).
|
||||
///
|
||||
/// Returns `None` on any error (sysfs not present, no matching block
|
||||
/// device, open failed). Callers treat that as "no fallback available"
|
||||
/// and propagate the original SCSI READ error.
|
||||
#[cfg(target_os = "linux")]
|
||||
fn open_block_device_for_sg(sg_path: &Path) -> Option<std::os::unix::io::RawFd> {
|
||||
let basename = sg_path.file_name()?.to_str()?;
|
||||
if !basename.starts_with("sg") {
|
||||
return None;
|
||||
}
|
||||
let sysfs_dir = format!("/sys/class/scsi_generic/{}/device/block", basename);
|
||||
let entries = std::fs::read_dir(&sysfs_dir).ok()?;
|
||||
let block_name = entries
|
||||
.flatten()
|
||||
.find_map(|e| e.file_name().into_string().ok())?;
|
||||
let block_path = format!("/dev/{}", block_name);
|
||||
|
||||
let mut bytes = block_path.as_bytes().to_vec();
|
||||
bytes.push(0);
|
||||
let fd = unsafe {
|
||||
libc::open(
|
||||
bytes.as_ptr() as *const libc::c_char,
|
||||
libc::O_RDONLY | libc::O_CLOEXEC,
|
||||
)
|
||||
};
|
||||
if fd < 0 {
|
||||
tracing::debug!(
|
||||
target: "freemkv::drive",
|
||||
sg = basename,
|
||||
block_path,
|
||||
errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0),
|
||||
"Failed to open block device for fallback; sr0 fallback disabled"
|
||||
);
|
||||
None
|
||||
} else {
|
||||
tracing::info!(
|
||||
target: "freemkv::drive",
|
||||
sg = basename,
|
||||
block_path,
|
||||
fd,
|
||||
"Opened /dev/sr* as recovery fallback for failed SCSI reads"
|
||||
);
|
||||
Some(fd)
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for Drive {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
|
||||
+289
@@ -0,0 +1,289 @@
|
||||
//! Top-level DRM scheme dispatch.
|
||||
//!
|
||||
//! Four content-protection schemes ride through a single
|
||||
//! detect-then-load pipeline:
|
||||
//!
|
||||
//! | Scheme | Discriminator |
|
||||
//! |---------------------|------------------------------------------------|
|
||||
//! | [`DrmScheme::Css`] | DVD probe sector flagged scrambled |
|
||||
//! | [`DrmScheme::Aacs10`] | Content cert type byte `0x00` |
|
||||
//! | [`DrmScheme::Aacs20`] | Content cert type byte `!= 0x00`, no Variant |
|
||||
//! | [`DrmScheme::Aacs21`] | Content cert + MKB records `0x82` / `0x83` |
|
||||
//!
|
||||
//! Detection happens from a [`DrmProbe`] (raw inputs the caller has
|
||||
//! already extracted from the disc); resolution runs through a
|
||||
//! [`DrmContext`] (the full set of inputs the loaders need).
|
||||
//!
|
||||
//! The AACS 2.1 arm is wired but disabled. The dispatcher leaves
|
||||
//! [`crate::aacs::resolve_keys_v21`] reachable as a library entry point
|
||||
//! for fixture-driven validation, but production consumers go through
|
||||
//! [`DrmScheme::load`], which short-circuits V21 to `None` until the
|
||||
//! Variant chain has a real Variant-scheme disc to validate against.
|
||||
|
||||
use crate::aacs;
|
||||
use crate::css;
|
||||
|
||||
/// Which content-protection scheme governs a disc.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum DrmScheme {
|
||||
/// DVD Content Scramble System.
|
||||
Css,
|
||||
/// AACS 1.0 — original BD-ROM.
|
||||
Aacs10,
|
||||
/// AACS 2.0 — UHD-BD, classical Media Key chain.
|
||||
Aacs20,
|
||||
/// AACS 2.1 — UHD-BD with Media Key Variant chain.
|
||||
Aacs21,
|
||||
}
|
||||
|
||||
/// Inputs to [`DrmScheme::detect`]. All borrows — caller retains
|
||||
/// ownership.
|
||||
pub struct DrmProbe<'a> {
|
||||
/// 2048-byte sample sector from inside a DVD title's extents. Used
|
||||
/// only for CSS scramble-flag detection. `None` for non-DVD discs.
|
||||
pub dvd_sample_sector: Option<&'a [u8]>,
|
||||
/// Content Certificate file bytes (typically `/AACS/Content000.cer`).
|
||||
/// `None` when the disc has no AACS directory.
|
||||
pub content_cert: Option<&'a [u8]>,
|
||||
/// MKB file bytes (typically `/AACS/MKB_RW.inf`). Required to
|
||||
/// distinguish AACS 2.0 from AACS 2.1.
|
||||
pub mkb: Option<&'a [u8]>,
|
||||
}
|
||||
|
||||
/// Inputs to [`DrmScheme::load`]. Carries everything needed by either
|
||||
/// the AACS or CSS loader.
|
||||
pub struct DrmContext<'a> {
|
||||
/// AACS resolver inputs — required when the scheme is any AACS
|
||||
/// variant.
|
||||
pub aacs: Option<aacs::ResolveContext<'a>>,
|
||||
/// CSS resolver inputs — required when the scheme is [`DrmScheme::Css`].
|
||||
pub css: Option<css::CssContext<'a>>,
|
||||
}
|
||||
|
||||
/// Resolved key material, tagged by scheme.
|
||||
#[derive(Debug)]
|
||||
pub enum ResolvedScheme {
|
||||
Css(css::CssState),
|
||||
Aacs(aacs::ResolvedKeys),
|
||||
}
|
||||
|
||||
impl DrmScheme {
|
||||
/// Detect which DRM scheme protects the disc described by `probe`.
|
||||
///
|
||||
/// Returns `None` for unencrypted media. The order is intentional:
|
||||
/// CSS is checked first (DVD-format probe), then AACS (Blu-ray
|
||||
/// format).
|
||||
pub fn detect(probe: &DrmProbe<'_>) -> Option<DrmScheme> {
|
||||
// CSS — DVD probe sector carries the scramble flag.
|
||||
if let Some(sector) = probe.dvd_sample_sector {
|
||||
if css::is_scrambled(sector) {
|
||||
return Some(DrmScheme::Css);
|
||||
}
|
||||
}
|
||||
|
||||
// AACS — content cert type byte distinguishes V10 from V20+.
|
||||
// V21 promotion requires MKB Variant records.
|
||||
let cc = probe.content_cert.and_then(aacs::parse_content_cert)?;
|
||||
match cc.version {
|
||||
aacs::AacsVersion::V10 => Some(DrmScheme::Aacs10),
|
||||
aacs::AacsVersion::V20 | aacs::AacsVersion::V21 => {
|
||||
if let Some(mkb) = probe.mkb {
|
||||
let recs = aacs::variants::walk_mkb(mkb);
|
||||
if aacs::variants::is_variant_mkb(&recs) {
|
||||
return Some(DrmScheme::Aacs21);
|
||||
}
|
||||
}
|
||||
Some(DrmScheme::Aacs20)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run key resolution for this scheme against `ctx`.
|
||||
///
|
||||
/// Returns `None` when the scheme's resolver could not produce keys
|
||||
/// (missing context, KEYDB miss, failed crypto walk, etc.) or when
|
||||
/// the scheme itself is gated off (see the inline comment on the
|
||||
/// `Aacs21` arm).
|
||||
pub fn load(self, ctx: &mut DrmContext<'_>) -> Option<ResolvedScheme> {
|
||||
match self {
|
||||
DrmScheme::Css => ctx
|
||||
.css
|
||||
.as_mut()
|
||||
.and_then(css::resolve)
|
||||
.map(ResolvedScheme::Css),
|
||||
DrmScheme::Aacs10 => ctx
|
||||
.aacs
|
||||
.as_ref()
|
||||
.and_then(aacs::resolve_keys_v1)
|
||||
.map(ResolvedScheme::Aacs),
|
||||
DrmScheme::Aacs20 => ctx
|
||||
.aacs
|
||||
.as_ref()
|
||||
.and_then(aacs::resolve_keys_v2)
|
||||
.map(ResolvedScheme::Aacs),
|
||||
// AACS 2.1 derivation is wired but disabled. KCD validation
|
||||
// against a Variant-scheme disc is pending. To enable,
|
||||
// uncomment the line below.
|
||||
// DrmScheme::Aacs21 => ctx
|
||||
// .aacs
|
||||
// .as_ref()
|
||||
// .and_then(aacs::resolve_keys_v21)
|
||||
// .map(ResolvedScheme::Aacs),
|
||||
DrmScheme::Aacs21 => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// Build a minimal cert: type byte + bus-encryption byte + 6 zero
|
||||
// cc_id bytes.
|
||||
fn cert(type_byte: u8) -> Vec<u8> {
|
||||
let mut v = vec![0u8; 8];
|
||||
v[0] = type_byte;
|
||||
v
|
||||
}
|
||||
|
||||
// Synthetic AACS 2.x MKB with no Variant records.
|
||||
fn mkb_classical() -> Vec<u8> {
|
||||
vec![
|
||||
0x10, 0x00, 0x00, 0x0C, 0x48, 0x14, 0x10, 0x03, 0x00, 0x00, 0x00, 0x4D,
|
||||
]
|
||||
}
|
||||
|
||||
// Synthetic AACS 2.x MKB with a 0x82 + 0x83 record pair.
|
||||
fn mkb_with_variant() -> Vec<u8> {
|
||||
let mut m = mkb_classical();
|
||||
m.extend_from_slice(&[0x82, 0x00, 0x00, 0x14]);
|
||||
m.extend_from_slice(&[0xEE; 16]);
|
||||
m.extend_from_slice(&[0x83, 0x00, 0x00, 0x14]);
|
||||
m.extend_from_slice(&[0x55; 16]);
|
||||
m
|
||||
}
|
||||
|
||||
// Synthetic scrambled DVD sector — byte 0x14 carries the CSS
|
||||
// scramble flag in bits 4-5.
|
||||
fn scrambled_dvd_sector() -> Vec<u8> {
|
||||
let mut s = vec![0u8; 2048];
|
||||
s[0x14] = 0x30;
|
||||
s
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_returns_none_for_unencrypted() {
|
||||
let probe = DrmProbe {
|
||||
dvd_sample_sector: None,
|
||||
content_cert: None,
|
||||
mkb: None,
|
||||
};
|
||||
assert_eq!(DrmScheme::detect(&probe), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_returns_css_for_scrambled_dvd() {
|
||||
let sector = scrambled_dvd_sector();
|
||||
let probe = DrmProbe {
|
||||
dvd_sample_sector: Some(§or),
|
||||
content_cert: None,
|
||||
mkb: None,
|
||||
};
|
||||
assert_eq!(DrmScheme::detect(&probe), Some(DrmScheme::Css));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_returns_aacs10_for_type0_cert() {
|
||||
let c = cert(0x00);
|
||||
let probe = DrmProbe {
|
||||
dvd_sample_sector: None,
|
||||
content_cert: Some(&c),
|
||||
mkb: None,
|
||||
};
|
||||
assert_eq!(DrmScheme::detect(&probe), Some(DrmScheme::Aacs10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_returns_aacs20_for_type1_cert_no_variant() {
|
||||
let c = cert(0x01);
|
||||
let mkb = mkb_classical();
|
||||
let probe = DrmProbe {
|
||||
dvd_sample_sector: None,
|
||||
content_cert: Some(&c),
|
||||
mkb: Some(&mkb),
|
||||
};
|
||||
assert_eq!(DrmScheme::detect(&probe), Some(DrmScheme::Aacs20));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_returns_aacs21_for_type1_cert_with_variant() {
|
||||
let c = cert(0x01);
|
||||
let mkb = mkb_with_variant();
|
||||
let probe = DrmProbe {
|
||||
dvd_sample_sector: None,
|
||||
content_cert: Some(&c),
|
||||
mkb: Some(&mkb),
|
||||
};
|
||||
assert_eq!(DrmScheme::detect(&probe), Some(DrmScheme::Aacs21));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_returns_aacs20_when_mkb_absent() {
|
||||
// Type-1 cert but no MKB to upgrade with -> Aacs20.
|
||||
let c = cert(0x01);
|
||||
let probe = DrmProbe {
|
||||
dvd_sample_sector: None,
|
||||
content_cert: Some(&c),
|
||||
mkb: None,
|
||||
};
|
||||
assert_eq!(DrmScheme::detect(&probe), Some(DrmScheme::Aacs20));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_aacs21_returns_none() {
|
||||
// The Aacs21 dispatch arm is commented out; load() must
|
||||
// return None until KCD validation lands.
|
||||
let uk_ro = vec![0u8; 256];
|
||||
let vid = [0u8; 16];
|
||||
let keydb = aacs::KeyDb::empty();
|
||||
let providers: &[&dyn aacs::KeyProvider] = &[&keydb];
|
||||
let ctx_aacs = aacs::ResolveContext {
|
||||
unit_key_ro: &uk_ro,
|
||||
content_cert: None,
|
||||
volume_id: &vid,
|
||||
providers,
|
||||
mkb: None,
|
||||
};
|
||||
let mut ctx = DrmContext {
|
||||
aacs: Some(ctx_aacs),
|
||||
css: None,
|
||||
};
|
||||
assert!(DrmScheme::Aacs21.load(&mut ctx).is_none());
|
||||
}
|
||||
|
||||
/// Exercises the V21 helper directly. Gated `#[ignore]` because
|
||||
/// the chain reaches `MediaKeyVariantError::VariantsTableUnavailable`
|
||||
/// without a real Variant-scheme disc to fix the per-uv table
|
||||
/// layout against — running it here would assert only the
|
||||
/// not-yet-wired error code. Kept as a wiring smoke-test for
|
||||
/// future enablement.
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn resolve_keys_v21_helper_exists() {
|
||||
let uk_ro = vec![0u8; 256];
|
||||
let vid = [0xAAu8; 16];
|
||||
let keydb = aacs::KeyDb::empty();
|
||||
let providers: &[&dyn aacs::KeyProvider] = &[&keydb];
|
||||
let mkb = mkb_with_variant();
|
||||
let ctx = aacs::ResolveContext {
|
||||
unit_key_ro: &uk_ro,
|
||||
content_cert: None,
|
||||
volume_id: &vid,
|
||||
providers,
|
||||
mkb: Some(&mkb),
|
||||
};
|
||||
// Just confirm the symbol is callable; we don't assert on the
|
||||
// result.
|
||||
let _ = aacs::resolve_keys_v21(&ctx);
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,14 @@ pub const E_AACS_VID_MAC: u16 = 7010;
|
||||
pub const E_AACS_DATA_KEY: u16 = 7011;
|
||||
pub const E_DECRYPT_FAILED: u16 = 7013;
|
||||
pub const E_CSS_AUTH_FAILED: u16 = 7014;
|
||||
pub const E_AACS_HOST_CERT_REJECTED: u16 = 7015;
|
||||
pub const E_AACS_RAW_READ_UNSUPPORTED: u16 = 7016;
|
||||
pub const E_AACS_VID_UNAVAILABLE: u16 = 7017;
|
||||
pub const E_AACS_MK_UNAVAILABLE: u16 = 7018;
|
||||
pub const E_AACS_VUK_NOT_IN_KEYDB: u16 = 7019;
|
||||
pub const E_DRIVE_PROFILE_MISSING: u16 = 7020;
|
||||
pub const E_VID_CDB_UNAVAILABLE: u16 = 7021;
|
||||
pub const E_NO_DISC_KEY: u16 = 7022;
|
||||
|
||||
// Keydb (8xxx)
|
||||
pub const E_KEYDB_CONNECT: u16 = 8000;
|
||||
@@ -222,6 +230,40 @@ pub enum Error {
|
||||
AacsDataKey,
|
||||
DecryptFailed,
|
||||
CssAuthFailed,
|
||||
/// Host certificate rejected by the drive's revocation list (HRL hit).
|
||||
/// All available host certs failed mutual auth on this drive.
|
||||
AacsHostCertRejected,
|
||||
/// Drive cannot be put into raw-read mode and standard AACS cert
|
||||
/// auth failed. No path to decryption remains.
|
||||
AacsRawReadUnsupported,
|
||||
/// Volume ID could not be retrieved from the drive (neither via cert
|
||||
/// auth nor via the alternate VID read path). Downstream of step 1
|
||||
/// of the AACS chain.
|
||||
AacsVidUnavailable,
|
||||
/// No available path produced a Media Key (no MK+VID in keydb, no
|
||||
/// PK match, no DK derivation).
|
||||
AacsMkUnavailable,
|
||||
/// Disc-hash lookup in the keydb missed and no other path is
|
||||
/// available (typically because VID is missing).
|
||||
AacsVukNotInKeydb,
|
||||
/// Drive identity did not match any bundled profile; per-drive CDB
|
||||
/// templates aren't available so the OEM VID retrieval path can't
|
||||
/// run.
|
||||
DriveProfileMissing,
|
||||
/// Drive's profile is present but doesn't carry a VID-retrieval CDB
|
||||
/// template (older profile blob, or a drive class without an OEM
|
||||
/// VID path).
|
||||
VidCdbUnavailable,
|
||||
/// The disc is AACS-encrypted and decryption was requested, but key
|
||||
/// resolution produced no usable key for it — so muxing would emit
|
||||
/// undecryptable garbage. Distinct from [`Error::KeydbLoad`] (no keydb
|
||||
/// file at all): a keydb may be present but lack an entry for this disc.
|
||||
/// `disc_hash` is the 40-hex SHA1 of `Unit_Key_RO.inf` (no `0x` prefix)
|
||||
/// so the application can name the disc; empty if the hash wasn't
|
||||
/// captured at scan.
|
||||
NoDiscKey {
|
||||
disc_hash: String,
|
||||
},
|
||||
|
||||
// Keydb (8xxx)
|
||||
KeydbConnect {
|
||||
@@ -307,6 +349,14 @@ impl Error {
|
||||
Error::AacsDataKey => E_AACS_DATA_KEY,
|
||||
Error::DecryptFailed => E_DECRYPT_FAILED,
|
||||
Error::CssAuthFailed => E_CSS_AUTH_FAILED,
|
||||
Error::AacsHostCertRejected => E_AACS_HOST_CERT_REJECTED,
|
||||
Error::AacsRawReadUnsupported => E_AACS_RAW_READ_UNSUPPORTED,
|
||||
Error::AacsVidUnavailable => E_AACS_VID_UNAVAILABLE,
|
||||
Error::AacsMkUnavailable => E_AACS_MK_UNAVAILABLE,
|
||||
Error::AacsVukNotInKeydb => E_AACS_VUK_NOT_IN_KEYDB,
|
||||
Error::DriveProfileMissing => E_DRIVE_PROFILE_MISSING,
|
||||
Error::VidCdbUnavailable => E_VID_CDB_UNAVAILABLE,
|
||||
Error::NoDiscKey { .. } => E_NO_DISC_KEY,
|
||||
Error::KeydbConnect { .. } => E_KEYDB_CONNECT,
|
||||
Error::KeydbHttp { .. } => E_KEYDB_HTTP,
|
||||
Error::KeydbInvalid => E_KEYDB_INVALID,
|
||||
@@ -433,6 +483,7 @@ impl std::fmt::Display for Error {
|
||||
Error::StreamUrlMissingPort { addr } => write!(f, "E{}: {}", self.code(), addr),
|
||||
Error::PesFrameTooLarge { size } => write!(f, "E{}: {}", self.code(), size),
|
||||
Error::IsoTooLarge { path } => write!(f, "E{}: {}", self.code(), path),
|
||||
Error::NoDiscKey { disc_hash } => write!(f, "E{}: {}", self.code(), disc_hash),
|
||||
_ => write!(f, "E{}", self.code()),
|
||||
}
|
||||
}
|
||||
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
//! One-bit cooperative cancellation flag.
|
||||
//!
|
||||
//! `Halt` is a clonable token wrapping `Arc<AtomicBool>`. Pass clones into
|
||||
//! every long-running loop; the loop polls `is_cancelled()` and bails out
|
||||
//! cleanly. Calling `cancel()` from any clone flips the shared flag, and
|
||||
//! every other clone observes it on its next poll.
|
||||
//!
|
||||
//! Why: `Ordering::Relaxed` is sufficient on both load and store because
|
||||
//! this flag is purely advisory — no other memory operations piggyback on
|
||||
//! it for happens-before ordering. Callers that need to publish data
|
||||
//! across threads do so via channels or other synchronization, not via
|
||||
//! this bit.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
/// Clonable, infallible cooperative-cancellation token.
|
||||
///
|
||||
/// Clones share the same underlying flag. `cancel()` is one-way; there is
|
||||
/// no `reset()` by design — construct a fresh `Halt` for a fresh
|
||||
/// operation.
|
||||
///
|
||||
/// Construct with [`Halt::new`] (or [`Halt::default`]). The `Default`
|
||||
/// impl forwards to `new()` — both produce a fresh, uncancelled token.
|
||||
/// The pair exists because clippy's `new_without_default` lint requires
|
||||
/// `Default` whenever a public `new()` is present, even when the two
|
||||
/// would do exactly the same thing.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Halt(Arc<AtomicBool>);
|
||||
|
||||
impl Halt {
|
||||
/// Construct a fresh, uncancelled token.
|
||||
pub fn new() -> Self {
|
||||
Self(Arc::new(AtomicBool::new(false)))
|
||||
}
|
||||
|
||||
/// Wrap an existing `Arc<AtomicBool>` as a `Halt`. Useful as a
|
||||
/// bridge during the 0.18 deprecation window: callers that already
|
||||
/// hold an `Arc<AtomicBool>` (e.g. `Drive::halt_flag()`, the
|
||||
/// deprecated `DiscStream::set_halt`) can adopt the new token API
|
||||
/// without changing the underlying flag.
|
||||
///
|
||||
/// Cancelling either side flips the same bit — the wrapping `Halt`
|
||||
/// and the original `Arc` are two views over one shared flag.
|
||||
pub fn from_arc(flag: Arc<AtomicBool>) -> Self {
|
||||
Self(flag)
|
||||
}
|
||||
|
||||
/// Borrow the underlying `Arc<AtomicBool>`. Used at boundaries with
|
||||
/// pre-`Halt` APIs that still take an `Arc<AtomicBool>` directly
|
||||
/// (`CopyOptions::halt`, the deprecated `DiscStream::set_halt`).
|
||||
/// Round 3 deletes those boundaries and this accessor with them.
|
||||
pub fn as_arc(&self) -> &Arc<AtomicBool> {
|
||||
&self.0
|
||||
}
|
||||
|
||||
/// Flip the shared flag to cancelled. Idempotent.
|
||||
pub fn cancel(&self) {
|
||||
self.0.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Read the shared flag.
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.0.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Halt {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared poll interval for halt-aware loops.
|
||||
///
|
||||
/// `bounded_syscall` checks the cancellation flag and the deadline
|
||||
/// every [`POLL_INTERVAL`] while blocked on a worker; the same
|
||||
/// cadence governs `Pipeline::send_with_halt`'s `try_send` retry.
|
||||
/// 250 ms is the sweet spot between responsiveness (operator presses
|
||||
/// Stop, sees it take effect within ~quarter-second) and waste
|
||||
/// (atomic load + clock read is cheap but not free at thousands of
|
||||
/// hertz).
|
||||
///
|
||||
/// Centralised here so the half-dozen halt-polling loops across `io`
|
||||
/// can't drift apart silently.
|
||||
pub const POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(250);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fresh_is_not_cancelled() {
|
||||
let h = Halt::new();
|
||||
assert!(!h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_flips_state() {
|
||||
let h = Halt::new();
|
||||
assert!(!h.is_cancelled());
|
||||
h.cancel();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_is_idempotent() {
|
||||
let h = Halt::new();
|
||||
h.cancel();
|
||||
h.cancel();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state() {
|
||||
let original = Halt::new();
|
||||
let cloned = original.clone();
|
||||
assert!(!original.is_cancelled());
|
||||
assert!(!cloned.is_cancelled());
|
||||
|
||||
// Cancel via the clone; the original observes it.
|
||||
cloned.cancel();
|
||||
assert!(original.is_cancelled());
|
||||
assert!(cloned.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state_reverse_direction() {
|
||||
let original = Halt::new();
|
||||
let cloned = original.clone();
|
||||
|
||||
// Cancel via the original; the clone observes it.
|
||||
original.cancel();
|
||||
assert!(cloned.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state_across_threads() {
|
||||
let h = Halt::new();
|
||||
let h2 = h.clone();
|
||||
let handle = std::thread::spawn(move || {
|
||||
h2.cancel();
|
||||
});
|
||||
handle.join().unwrap();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_arc_shares_state() {
|
||||
// The 0.18 deprecation-window bridge: a Halt built from an
|
||||
// existing Arc<AtomicBool> must be a *view* over the same bit,
|
||||
// not a fresh copy. Cancelling either side flips both.
|
||||
let arc = Arc::new(AtomicBool::new(false));
|
||||
let halt = Halt::from_arc(arc.clone());
|
||||
assert!(!halt.is_cancelled());
|
||||
assert!(!arc.load(Ordering::Relaxed));
|
||||
|
||||
// Cancel via the wrapping Halt; the original Arc observes it.
|
||||
halt.cancel();
|
||||
assert!(arc.load(Ordering::Relaxed));
|
||||
|
||||
// Conversely: flip the Arc directly; the Halt view observes it.
|
||||
let arc2 = Arc::new(AtomicBool::new(false));
|
||||
let halt2 = Halt::from_arc(arc2.clone());
|
||||
arc2.store(true, Ordering::Relaxed);
|
||||
assert!(halt2.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn as_arc_returns_backing_flag() {
|
||||
// `as_arc()` must hand back the *same* Arc, not a clone of a
|
||||
// different bit. Verified by writing through the borrowed Arc
|
||||
// and observing through the Halt.
|
||||
let halt = Halt::new();
|
||||
let arc = halt.as_arc().clone();
|
||||
assert!(!halt.is_cancelled());
|
||||
arc.store(true, Ordering::Relaxed);
|
||||
assert!(halt.is_cancelled());
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -9,7 +9,7 @@
|
||||
|
||||
use crate::disc::{Codec, Resolution};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
// ── Public types ────────────────────────────────────────────────────────────
|
||||
@@ -184,7 +184,7 @@ fn bcd_byte(b: u8) -> u32 {
|
||||
///
|
||||
/// Reads the VMG (Video Manager) to discover title sets, then reads each
|
||||
/// VTS IFO to extract PGC chains, cell addresses, and stream attributes.
|
||||
pub fn parse_vmg(reader: &mut dyn SectorReader, udf: &UdfFs) -> Result<DvdInfo> {
|
||||
pub fn parse_vmg(reader: &mut dyn SectorSource, udf: &UdfFs) -> Result<DvdInfo> {
|
||||
let vmg_data = udf.read_file(reader, "/VIDEO_TS/VIDEO_TS.IFO")?;
|
||||
|
||||
// Validate VMG magic
|
||||
@@ -263,7 +263,7 @@ pub fn parse_vmg(reader: &mut dyn SectorReader, udf: &UdfFs) -> Result<DvdInfo>
|
||||
///
|
||||
/// `titles_info` is a list of (chapter_count, vts_title_number) from TT_SRPT.
|
||||
fn parse_vts(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf: &UdfFs,
|
||||
vts_number: u8,
|
||||
titles_info: &[(u16, u8)],
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
//! Bounded-syscall primitive: run a (potentially-blocking) operation
|
||||
//! on a worker thread, with a hard wall-clock deadline and an optional
|
||||
//! cooperative [`Halt`] poll. The calling thread is never trapped
|
||||
//! inside a kernel call.
|
||||
//!
|
||||
//! ## Why this exists
|
||||
//!
|
||||
//! [`crate::halt::Halt`] is cooperative: callers poll
|
||||
//! `is_cancelled()`. It cannot reach inside a syscall the kernel
|
||||
//! currently owns the thread for — `libc::sync_file_range`,
|
||||
//! `libc::fsync`, `File::write` on NFS, and so on. `/api/stop` from
|
||||
//! autorip therefore can't unstick a thread sitting in such a syscall.
|
||||
//!
|
||||
//! [`bounded_syscall`] is the escape hatch: it runs `op` on a fresh
|
||||
//! worker thread, then `recv_timeout`s on a rendezvous channel for the
|
||||
//! result. The wait is broken into ~250 ms slices so the calling
|
||||
//! thread can poll the supplied [`Halt`] in between. If the deadline
|
||||
//! elapses or the halt fires, the worker is intentionally leaked — the
|
||||
//! syscall will unwind whenever the kernel decides, or at process
|
||||
//! exit, but the caller is free to fall back to a degraded code path
|
||||
//! (skip the sync, log loudly, etc.).
|
||||
//!
|
||||
//! ## Trade-offs
|
||||
//!
|
||||
//! - **Thread per call.** Cheap (`std::thread::spawn` is < 100 µs on
|
||||
//! Linux/macOS), but not free. Use on coarse-grained finalisation
|
||||
//! syscalls (`sync_all`, `sync_file_range(WAIT_AFTER)`), not on hot
|
||||
//! inner-loop writes.
|
||||
//! - **Leak on timeout.** A wedged syscall keeps a kernel slot and a
|
||||
//! user-space thread around for the rest of the process's life.
|
||||
//! Bounded by the number of independent rip/mux sessions, which is
|
||||
//! one per disc. The alternative — trapping the caller forever —
|
||||
//! defeats the entire purpose of `/api/stop`.
|
||||
//! - **Halt granularity ~250 ms.** Halt observation is not instant;
|
||||
//! it's the worst-case latency of the `recv_timeout` slice. Good
|
||||
//! enough for human-driven stop requests; not suitable for hard
|
||||
//! real-time deadlines.
|
||||
//!
|
||||
//! ## Single source of truth
|
||||
//!
|
||||
//! Do NOT inline this pattern. Every blocking-syscall wrapper in the
|
||||
//! rip + mux pipeline calls this helper, so changes (e.g. swapping the
|
||||
//! channel impl, adjusting the poll slice, adding metrics) land in one
|
||||
//! place.
|
||||
//!
|
||||
//! ## Platform
|
||||
//!
|
||||
//! Pure `std::thread` + `std::sync::mpsc`. No `cfg(target_os)` needed
|
||||
//! here — the helper itself is platform-agnostic. Callers that wrap
|
||||
//! Linux-only syscalls (`sync_file_range`) still need their own
|
||||
//! `#[cfg(target_os = "linux")]` gates; this helper does not.
|
||||
|
||||
use std::sync::mpsc::{RecvTimeoutError, sync_channel};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::halt::{Halt, POLL_INTERVAL};
|
||||
|
||||
/// Failure outcome from a bounded syscall wrapper.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum BoundedError {
|
||||
/// The user-visible halt token fired during the wait. The worker
|
||||
/// thread is intentionally leaked — the caller should fall back to
|
||||
/// a degraded code path rather than waiting on the syscall to
|
||||
/// return.
|
||||
Halted,
|
||||
/// The deadline elapsed before the syscall returned. Same leak
|
||||
/// semantics as `Halted`.
|
||||
Timeout,
|
||||
/// The worker thread panicked, or its sender disconnected before
|
||||
/// sending a result. Treat as a benign no-op (callers usually
|
||||
/// log and continue) rather than a hard error — by definition no
|
||||
/// syscall observably ran to completion in this case.
|
||||
WorkerLost,
|
||||
}
|
||||
|
||||
/// Run a (potentially-blocking) operation on a worker thread with a
|
||||
/// deadline and an optional cooperative halt-token poll. Returns the
|
||||
/// operation's result if it completes within `timeout`; otherwise one
|
||||
/// of [`BoundedError::Halted`] / [`BoundedError::Timeout`] /
|
||||
/// [`BoundedError::WorkerLost`].
|
||||
///
|
||||
/// On `Halted` / `Timeout` the worker thread is intentionally leaked:
|
||||
/// the syscall will unwind whenever the kernel decides, or when the
|
||||
/// process exits. The calling thread is never trapped inside a kernel
|
||||
/// call.
|
||||
///
|
||||
/// `halt` is polled at [`POLL_INTERVAL`] granularity. Pass `None` for
|
||||
/// callers that don't (yet) have a halt token plumbed through —
|
||||
/// behaviour degrades to deadline-only, matching the 0.20.5
|
||||
/// `wait_after_with_timeout` shape this helper generalises.
|
||||
///
|
||||
/// `op` returns `R: Send + 'static`. The closure must own everything
|
||||
/// it touches because it may outlive this call (timeout / halt cases).
|
||||
pub(crate) fn bounded_syscall<F, R>(
|
||||
halt: Option<&Halt>,
|
||||
timeout: Duration,
|
||||
op: F,
|
||||
) -> Result<R, BoundedError>
|
||||
where
|
||||
F: FnOnce() -> R + Send + 'static,
|
||||
R: Send + 'static,
|
||||
{
|
||||
// Rendezvous channel: the worker sends exactly one value (the
|
||||
// op's return) and then exits. Capacity-0 means the send blocks
|
||||
// until we receive — fine on the happy path; on the timeout /
|
||||
// halt path the receiver is dropped and the worker's send
|
||||
// returns Err, which the worker ignores.
|
||||
let (tx, rx) = sync_channel::<R>(0);
|
||||
let _ = thread::Builder::new()
|
||||
.name("freemkv-bounded-syscall".into())
|
||||
.spawn(move || {
|
||||
// Ignore the send error: if we time out (or get halted)
|
||||
// before the worker finishes, the receiver is dropped
|
||||
// and `tx.send` returns Err. Either way, the worker has
|
||||
// nothing more to do.
|
||||
let _ = tx.send(op());
|
||||
});
|
||||
|
||||
let deadline = Instant::now() + timeout;
|
||||
loop {
|
||||
let now = Instant::now();
|
||||
let remaining = deadline.saturating_duration_since(now);
|
||||
let slice = remaining.min(POLL_INTERVAL);
|
||||
match rx.recv_timeout(slice) {
|
||||
Ok(v) => return Ok(v),
|
||||
Err(RecvTimeoutError::Timeout) => {
|
||||
if let Some(h) = halt {
|
||||
if h.is_cancelled() {
|
||||
return Err(BoundedError::Halted);
|
||||
}
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(BoundedError::Timeout);
|
||||
}
|
||||
// Otherwise: another slice.
|
||||
}
|
||||
Err(RecvTimeoutError::Disconnected) => {
|
||||
// Worker thread spawn failed, or it panicked before
|
||||
// sending. Caller treats this as "no syscall ran" —
|
||||
// typically a no-op + log.
|
||||
return Err(BoundedError::WorkerLost);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
#[test]
|
||||
fn op_completes_quickly() {
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), || 42u32);
|
||||
assert!(matches!(r, Ok(42)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn op_exceeds_timeout() {
|
||||
// Op sleeps longer than the deadline → Timeout.
|
||||
let r = bounded_syscall(None, Duration::from_millis(300), || {
|
||||
thread::sleep(Duration::from_secs(2));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Timeout)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_fires_during_wait() {
|
||||
let halt = Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
// Flip the halt from a side thread after ~300 ms — long
|
||||
// enough that the receive loop has rolled at least one
|
||||
// 250 ms slice and is sitting in `recv_timeout` again when
|
||||
// the bit flips.
|
||||
thread::spawn(move || {
|
||||
thread::sleep(Duration::from_millis(300));
|
||||
halt2.cancel();
|
||||
});
|
||||
let r = bounded_syscall(Some(&halt), Duration::from_secs(5), || {
|
||||
thread::sleep(Duration::from_secs(5));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Halted)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_panics() {
|
||||
// Worker panics → sender drops without sending → recv sees
|
||||
// Disconnected → WorkerLost. We use an explicit panic in the
|
||||
// op closure rather than `panic!()` from inside the channel
|
||||
// machinery; the spawned thread's panic is contained (no
|
||||
// process abort) because we don't `.join()` it.
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), || -> u32 {
|
||||
panic!("intentional test panic");
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::WorkerLost)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_already_set_before_call_still_returns_halted() {
|
||||
// Halt observed on the very first poll slice. The op blocks
|
||||
// forever; we must not wait the full timeout to notice the
|
||||
// halt is already set.
|
||||
let halt = Halt::new();
|
||||
halt.cancel();
|
||||
let started = Instant::now();
|
||||
let r = bounded_syscall(Some(&halt), Duration::from_secs(10), || {
|
||||
thread::sleep(Duration::from_secs(10));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Halted)));
|
||||
// Should bail out within ~1 s; allow 2 s of slack for slow
|
||||
// CI hosts.
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(2),
|
||||
"halt-already-set took {:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ok_path_takes_no_halt_token() {
|
||||
// Sanity: the `None` halt path is the documented zero-config
|
||||
// form (matches the 0.20.5 `wait_after_with_timeout`
|
||||
// behaviour). Op returns immediately; we must observe Ok.
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let f2 = flag.clone();
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), move || {
|
||||
f2.store(true, Ordering::Relaxed);
|
||||
"ok"
|
||||
});
|
||||
assert!(matches!(r, Ok("ok")));
|
||||
assert!(flag.load(Ordering::Relaxed));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
//! Byte-sized bounded producer/consumer channel.
|
||||
//!
|
||||
//! Wraps `std::sync::mpsc::sync_channel` with a byte-accounting
|
||||
//! `Mutex<usize> + Condvar` cap. Sender blocks (cooperatively) when
|
||||
//! `used_bytes + item.byte_size() > capacity_bytes`. Receiver
|
||||
//! decrements `used_bytes` when it takes the item.
|
||||
//!
|
||||
//! Why: the existing producer→consumer channel between `DiscStream`
|
||||
//! (PES producer) and `MuxSink` (PES consumer) is bounded by frame
|
||||
//! count. Frame sizes vary 100× between metadata and keyframes, so a
|
||||
//! count-based cap either starves on small frames or buffers far too
|
||||
//! much memory on big ones. Byte-sized accounting sizes the buffer for
|
||||
//! the worst-case input stall (NFS read p99 ≈ 1–2 s × ~15 MB/s peak
|
||||
//! compressed bitrate ≈ ~30 MB) directly.
|
||||
//!
|
||||
//! The underlying mpsc channel is created with a very large slot count
|
||||
//! so the byte cap (not the slot count) is the real backpressure. Slot
|
||||
//! count is only there to give the kernel a small chunk to wake on.
|
||||
|
||||
use std::sync::mpsc::{Receiver as MpscReceiver, RecvError, SendError, SyncSender, sync_channel};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
|
||||
/// Default byte cap for the muxer's input channel. Sized to hide a
|
||||
/// worst-case ~2 s NFS read refill at UHD peak compressed bitrate
|
||||
/// (~15 MB/s); 64 MiB gives headroom. Tweakable; not magic.
|
||||
pub const BYTE_CHANNEL_DEFAULT_CAPACITY: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// Slot capacity of the inner `sync_channel`. Large so the byte cap is
|
||||
/// the real backpressure mechanism — the mpsc slot count only exists
|
||||
/// to give the kernel a chunk to wake on. PES frames are typically
|
||||
/// ~700 B each, so 64 MiB ≈ 90 k frames; 200 k is comfortable headroom.
|
||||
const INNER_SLOT_CAPACITY: usize = 200_000;
|
||||
|
||||
/// Anything whose in-memory cost can be accounted by a single
|
||||
/// `usize`. Implement on the item type sent through [`Sender`].
|
||||
pub trait HasByteSize {
|
||||
/// Bytes this item contributes to the channel's used budget.
|
||||
/// Must be > 0 to make progress (a 0-byte item would never
|
||||
/// block the sender no matter the cap; see send_blocks_at_capacity
|
||||
/// test).
|
||||
fn byte_size(&self) -> usize;
|
||||
}
|
||||
|
||||
impl HasByteSize for crate::pes::PesFrame {
|
||||
fn byte_size(&self) -> usize {
|
||||
// Frame data + the fixed header overhead the serializer
|
||||
// writes (track + pts + keyframe + len). The `Vec<u8>` heap
|
||||
// allocation also has alloc-header overhead but that's
|
||||
// <0.1 % at typical frame sizes — folding it in would just
|
||||
// add noise to the budget.
|
||||
self.data.len() + 14
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared book-keeping between [`Sender`] and [`Receiver`]. Wrapped in
|
||||
/// an `Arc` because both halves hold it independently.
|
||||
struct Accounting {
|
||||
used: Mutex<usize>,
|
||||
cv: Condvar,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
/// Send half of the byte-bounded channel.
|
||||
///
|
||||
/// `send` blocks (on a `Condvar`) when adding the item would push
|
||||
/// `used_bytes` past `capacity_bytes`. Unblocks when the receiver
|
||||
/// `recv`s items out and notifies. Returns `Err(item)` if the
|
||||
/// receiver has been dropped — mirrors `mpsc::SyncSender::send`.
|
||||
pub struct Sender<T: HasByteSize> {
|
||||
tx: SyncSender<T>,
|
||||
acct: Arc<Accounting>,
|
||||
}
|
||||
|
||||
impl<T: HasByteSize> Clone for Sender<T> {
|
||||
fn clone(&self) -> Self {
|
||||
Sender {
|
||||
tx: self.tx.clone(),
|
||||
acct: self.acct.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: HasByteSize> Sender<T> {
|
||||
/// Push one item. Blocks until adding it would not exceed the
|
||||
/// capacity, then sends through the inner mpsc channel.
|
||||
pub fn send(&self, item: T) -> Result<(), SendError<T>> {
|
||||
let sz = item.byte_size();
|
||||
// Reserve capacity first. The reservation is observable to
|
||||
// other senders via `used`; only after we win the slot do we
|
||||
// hand the item to the inner mpsc channel. That ordering means
|
||||
// `used` is always a conservative upper bound on what's in the
|
||||
// mpsc queue + about-to-be-sent.
|
||||
{
|
||||
let mut used = self.acct.used.lock().expect("byte_channel poisoned");
|
||||
// An item bigger than the whole capacity will never fit; let
|
||||
// it through anyway as a one-shot reservation, otherwise the
|
||||
// sender deadlocks forever waiting for `used == 0` AND
|
||||
// nothing in flight. The receiver will drain it on the
|
||||
// other side. Same behaviour as `std::sync::mpsc` for
|
||||
// arbitrarily large messages.
|
||||
while *used + sz > self.acct.capacity && *used > 0 {
|
||||
used = self.acct.cv.wait(used).expect("byte_channel cv poisoned");
|
||||
}
|
||||
*used += sz;
|
||||
}
|
||||
match self.tx.send(item) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(SendError(returned)) => {
|
||||
// Receiver dropped — refund the reservation so a later
|
||||
// sender on a clone doesn't observe phantom used bytes
|
||||
// (the receiver is gone so nobody will decrement).
|
||||
let mut used = self.acct.used.lock().expect("byte_channel poisoned");
|
||||
*used = used.saturating_sub(sz);
|
||||
self.acct.cv.notify_all();
|
||||
Err(SendError(returned))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Receive half of the byte-bounded channel.
|
||||
///
|
||||
/// `recv` blocks on the inner mpsc until an item is available, then
|
||||
/// decrements the byte-accounting and wakes any sender waiting on
|
||||
/// capacity.
|
||||
pub struct Receiver<T: HasByteSize> {
|
||||
rx: MpscReceiver<T>,
|
||||
acct: Arc<Accounting>,
|
||||
}
|
||||
|
||||
impl<T: HasByteSize> Receiver<T> {
|
||||
/// Take the next item. Returns `Err(RecvError)` when all senders
|
||||
/// have been dropped and the channel is empty.
|
||||
pub fn recv(&self) -> Result<T, RecvError> {
|
||||
let item = self.rx.recv()?;
|
||||
let sz = item.byte_size();
|
||||
let mut used = self.acct.used.lock().expect("byte_channel poisoned");
|
||||
*used = used.saturating_sub(sz);
|
||||
// Notify all so multi-sender setups wake every blocked sender,
|
||||
// not just one. Wasted wakeups are cheap; missed wakeups would
|
||||
// be a deadlock.
|
||||
self.acct.cv.notify_all();
|
||||
Ok(item)
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a byte-bounded channel with the given capacity in bytes.
|
||||
/// Returns a `(Sender, Receiver)` pair; clone the `Sender` for
|
||||
/// multi-producer setups.
|
||||
pub fn channel<T: HasByteSize>(capacity_bytes: usize) -> (Sender<T>, Receiver<T>) {
|
||||
let (tx, rx) = sync_channel::<T>(INNER_SLOT_CAPACITY);
|
||||
let acct = Arc::new(Accounting {
|
||||
used: Mutex::new(0),
|
||||
cv: Condvar::new(),
|
||||
capacity: capacity_bytes,
|
||||
});
|
||||
(
|
||||
Sender {
|
||||
tx,
|
||||
acct: acct.clone(),
|
||||
},
|
||||
Receiver { rx, acct },
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Test payload — its `byte_size` returns whatever we passed at
|
||||
/// construction so capacity math is exact and predictable.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
struct Item {
|
||||
sz: usize,
|
||||
tag: u32,
|
||||
}
|
||||
|
||||
impl HasByteSize for Item {
|
||||
fn byte_size(&self) -> usize {
|
||||
self.sz
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_recv_round_trip() {
|
||||
let (tx, rx) = channel::<Item>(1024);
|
||||
for i in 0..5 {
|
||||
tx.send(Item { sz: 100, tag: i }).unwrap();
|
||||
}
|
||||
for i in 0..5 {
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got, Item { sz: 100, tag: i });
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn byte_accounting_decrements_on_recv() {
|
||||
// Internal book-keeping check via observable side-effect: after
|
||||
// sending K items totalling N bytes and receiving them all, a
|
||||
// subsequent send of an N-byte item must NOT block (no items
|
||||
// in flight, all capacity refunded).
|
||||
let (tx, rx) = channel::<Item>(1024);
|
||||
for _ in 0..4 {
|
||||
tx.send(Item { sz: 256, tag: 0 }).unwrap();
|
||||
}
|
||||
for _ in 0..4 {
|
||||
rx.recv().unwrap();
|
||||
}
|
||||
// Cap is now fully available again. Send a 1024-byte item; the
|
||||
// `used > 0` guard means it goes through alone (no wait).
|
||||
let start = Instant::now();
|
||||
tx.send(Item { sz: 1024, tag: 99 }).unwrap();
|
||||
assert!(start.elapsed() < Duration::from_millis(100));
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got.tag, 99);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_blocks_at_capacity_unblocks_on_recv() {
|
||||
// Cap = 200 bytes, item = 100 bytes. First two sends fit
|
||||
// exactly; the third must block until a recv frees capacity.
|
||||
let (tx, rx) = channel::<Item>(200);
|
||||
tx.send(Item { sz: 100, tag: 0 }).unwrap();
|
||||
tx.send(Item { sz: 100, tag: 1 }).unwrap();
|
||||
|
||||
let tx2 = tx.clone();
|
||||
let sent_at = Arc::new(Mutex::new(None::<Instant>));
|
||||
let sent_at2 = sent_at.clone();
|
||||
let h = thread::spawn(move || {
|
||||
tx2.send(Item { sz: 100, tag: 2 }).unwrap();
|
||||
*sent_at2.lock().unwrap() = Some(Instant::now());
|
||||
});
|
||||
|
||||
// Give the sender thread a head start; it should be parked in
|
||||
// `cv.wait` because used (200) + 100 > capacity (200).
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
assert!(
|
||||
sent_at.lock().unwrap().is_none(),
|
||||
"third send should be blocked at capacity"
|
||||
);
|
||||
|
||||
// Drain one. Sender wakes and completes.
|
||||
let recv_at = Instant::now();
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got.tag, 0);
|
||||
h.join().unwrap();
|
||||
|
||||
let sent_when = sent_at.lock().unwrap().unwrap();
|
||||
assert!(
|
||||
sent_when >= recv_at,
|
||||
"sender must complete AFTER receiver freed capacity"
|
||||
);
|
||||
|
||||
// Drain the remaining two.
|
||||
assert_eq!(rx.recv().unwrap().tag, 1);
|
||||
assert_eq!(rx.recv().unwrap().tag, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_larger_than_capacity_still_goes_through() {
|
||||
// Pathological case: a single item bigger than the capacity.
|
||||
// The guard `*used > 0` lets it through when the channel is
|
||||
// empty (otherwise the sender deadlocks forever). Matches
|
||||
// `mpsc::SyncSender` semantics for oversize messages.
|
||||
let (tx, rx) = channel::<Item>(100);
|
||||
tx.send(Item { sz: 1000, tag: 7 }).unwrap();
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got, Item { sz: 1000, tag: 7 });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_send_recv_stress() {
|
||||
// 4 sender threads × 1k items each, 1 receiver. Verify byte
|
||||
// accounting stays sane (channel never deadlocks, every item
|
||||
// arrives exactly once) under contention.
|
||||
const SENDERS: u32 = 4;
|
||||
const PER_SENDER: u32 = 1000;
|
||||
const TOTAL: u32 = SENDERS * PER_SENDER;
|
||||
|
||||
let (tx, rx) = channel::<Item>(8 * 1024);
|
||||
let sent = Arc::new(AtomicUsize::new(0));
|
||||
let mut handles = Vec::new();
|
||||
for s in 0..SENDERS {
|
||||
let tx = tx.clone();
|
||||
let sent = sent.clone();
|
||||
handles.push(thread::spawn(move || {
|
||||
for i in 0..PER_SENDER {
|
||||
// Vary item size so accounting actually has to
|
||||
// multiplex differently-sized blockers. 1B → 256B.
|
||||
let sz = 1 + ((i as usize) % 256);
|
||||
tx.send(Item {
|
||||
sz,
|
||||
tag: s * PER_SENDER + i,
|
||||
})
|
||||
.unwrap();
|
||||
sent.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}));
|
||||
}
|
||||
// Drop our local sender so the receiver can eventually see
|
||||
// RecvError once all sender clones are done. Cloning the
|
||||
// sender into each producer means each clone Drop'd separately.
|
||||
drop(tx);
|
||||
|
||||
let mut received = 0u32;
|
||||
while let Ok(_item) = rx.recv() {
|
||||
received += 1;
|
||||
}
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
assert_eq!(received, TOTAL);
|
||||
assert_eq!(sent.load(Ordering::SeqCst) as u32, TOTAL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_after_recv_dropped_returns_err() {
|
||||
let (tx, rx) = channel::<Item>(1024);
|
||||
drop(rx);
|
||||
let r = tx.send(Item { sz: 10, tag: 0 });
|
||||
assert!(r.is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
//! `BytePrefetcher` — `std::io::Read` analogue of
|
||||
//! [`crate::sector::PrefetchedSectorSource`].
|
||||
//!
|
||||
//! Spawns a producer thread that fills a bounded pool of `Vec<u8>`
|
||||
//! chunks from the underlying reader and ships them through a
|
||||
//! channel; the consumer pulls filled chunks, uses them, and sends
|
||||
//! the empty `Vec<u8>` back through a recycle channel so the
|
||||
//! producer can re-fill in place. Result: zero allocations and zero
|
||||
//! cross-thread frees in the steady-state hot loop.
|
||||
//!
|
||||
//! This is the byte-stream half of the freemkv mux highway —
|
||||
//! `BytePrefetcher` feeds [`crate::mux::demux_thread::DemuxThread`]
|
||||
//! for `m2ts://`, `network://`, `stdio://`, and any other stream
|
||||
//! whose source is an `io::Read` rather than a `SectorSource`.
|
||||
|
||||
use crate::halt::Halt;
|
||||
use crossbeam_channel::{Receiver, Sender, bounded};
|
||||
use std::io::Read;
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// Items flowing through the forward channel.
|
||||
pub type Batch = std::io::Result<Vec<u8>>;
|
||||
|
||||
/// Forward channel depth — how many filled buffers the producer can
|
||||
/// stay ahead by. Two is enough to absorb a moderate consumer stall
|
||||
/// without piling up bytes.
|
||||
const FORWARD_DEPTH: usize = 2;
|
||||
|
||||
/// Recycle channel depth = forward + 1 so the producer always has at
|
||||
/// least one buffer to fill while the consumer holds one.
|
||||
const RECYCLE_DEPTH: usize = FORWARD_DEPTH + 1;
|
||||
|
||||
/// Default chunk size — 16 MiB matches the ISO-mux sector batch and
|
||||
/// is large enough that per-chunk overhead is amortised; small
|
||||
/// enough that the in-flight memory footprint stays bounded.
|
||||
pub const DEFAULT_CHUNK_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// Returned from [`BytePrefetcher::into_channels`]. Owns the
|
||||
/// producer-thread join handle so dropping the shell joins the
|
||||
/// producer.
|
||||
pub struct PrefetchShell {
|
||||
producer: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl Drop for PrefetchShell {
|
||||
fn drop(&mut self) {
|
||||
if let Some(h) = self.producer.take() {
|
||||
let _ = h.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawned byte prefetcher. Drop joins the producer thread.
|
||||
pub struct BytePrefetcher {
|
||||
rx: Receiver<Batch>,
|
||||
recycle_tx: Sender<Vec<u8>>,
|
||||
producer: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl BytePrefetcher {
|
||||
/// Spawn the producer thread. `reader` must be `Send` because it
|
||||
/// moves into the thread. `chunk_bytes` is the size of each
|
||||
/// recycled buffer; pick the natural batch size of the
|
||||
/// downstream demuxer (16 MiB for the BD-TS mux pipeline).
|
||||
pub fn new<R: Read + Send + 'static>(
|
||||
mut reader: R,
|
||||
chunk_bytes: usize,
|
||||
halt: Option<Halt>,
|
||||
) -> Self {
|
||||
let (tx, rx) = bounded::<Batch>(FORWARD_DEPTH);
|
||||
let (recycle_tx, recycle_rx) = bounded::<Vec<u8>>(RECYCLE_DEPTH);
|
||||
|
||||
// Seed the recycle pool. Without these the first
|
||||
// `recycle_rx.recv()` would block forever (no consumer has
|
||||
// returned a buffer yet).
|
||||
for _ in 0..RECYCLE_DEPTH {
|
||||
let _ = recycle_tx.send(vec![0u8; chunk_bytes]);
|
||||
}
|
||||
|
||||
let producer = std::thread::Builder::new()
|
||||
.name("freemkv-byte-prefetch".into())
|
||||
.spawn(move || {
|
||||
loop {
|
||||
if halt.as_ref().map(|h| h.is_cancelled()).unwrap_or(false) {
|
||||
return;
|
||||
}
|
||||
let mut buf = match recycle_rx.recv() {
|
||||
Ok(b) => b,
|
||||
Err(_) => return, // consumer dropped both channels
|
||||
};
|
||||
// Re-expose the full extent (previous iteration
|
||||
// may have truncated after a short read).
|
||||
if buf.len() < chunk_bytes {
|
||||
buf.resize(chunk_bytes, 0);
|
||||
} else {
|
||||
// SAFETY: capacity is at least chunk_bytes
|
||||
// after construction.
|
||||
unsafe { buf.set_len(chunk_bytes) };
|
||||
}
|
||||
// Read up to one full chunk. Short reads are
|
||||
// valid and common — pipe `truncate` so the
|
||||
// consumer sees only the bytes that arrived.
|
||||
let n = match reader.read(&mut buf[..]) {
|
||||
Ok(0) => return, // EOF — drop tx, consumer sees RecvError
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
let _ = tx.send(Err(e));
|
||||
return;
|
||||
}
|
||||
};
|
||||
buf.truncate(n);
|
||||
if tx.send(Ok(buf)).is_err() {
|
||||
return; // consumer dropped
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("freemkv-byte-prefetch thread spawn failed");
|
||||
|
||||
Self {
|
||||
rx,
|
||||
recycle_tx,
|
||||
producer: Some(producer),
|
||||
}
|
||||
}
|
||||
|
||||
/// Peel off the channels for zero-copy pipeline consumption. The
|
||||
/// caller (typically [`crate::mux::demux_thread::DemuxThread`])
|
||||
/// drains `rx`, runs the demuxer in place on each filled buffer,
|
||||
/// and recycles back through `recycle_tx`.
|
||||
pub fn into_channels(self) -> (Receiver<Batch>, Sender<Vec<u8>>, PrefetchShell) {
|
||||
let mut me = self;
|
||||
let producer = me.producer.take();
|
||||
let rx = me.rx.clone();
|
||||
let recycle = me.recycle_tx.clone();
|
||||
std::mem::forget(me);
|
||||
(rx, recycle, PrefetchShell { producer })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BytePrefetcher {
|
||||
fn drop(&mut self) {
|
||||
if let Some(h) = self.producer.take() {
|
||||
let _ = h.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
//! Linux read-side platform hooks: sequential-access hint at open +
|
||||
//! periodic page-cache eviction during streaming reads.
|
||||
//!
|
||||
//! ## Why both
|
||||
//!
|
||||
//! `POSIX_FADV_SEQUENTIAL` at open widens the kernel's readahead window
|
||||
//! so each pread aggregates into fewer NFS round-trips. `DONTNEED` on
|
||||
//! the consumed window (called periodically by the caller) drops the
|
||||
//! already-read pages from the page cache so an 85 GB streaming ISO
|
||||
//! read doesn't fill memory and starve concurrent writes (the MKV
|
||||
//! output during mux). Together they mirror the write-side
|
||||
//! WritebackPipeline's policy.
|
||||
//!
|
||||
//! ## History
|
||||
//!
|
||||
//! Pre-Phase-1 (0.20.7 baseline) had both. Phase 1's introduction of
|
||||
//! `FileSectorSource` silently dropped the read-side DONTNEED, and
|
||||
//! 0.21.2's revert of `SEQUENTIAL` (mistakenly attributing a regression
|
||||
//! to it) removed the hint. Net effect: 85 GB of ISO reads pinned in
|
||||
//! the page cache + no readahead widening → mux throughput collapse
|
||||
//! from 18 MB/s historical to 2.7-8 MB/s on 0.21.x. Restored in 0.21.6.
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
pub(super) fn hint_sequential(file: &File, _len_bytes: u64) {
|
||||
// Best-effort: return value ignored. A fadvise failure has no
|
||||
// user-observable consequence.
|
||||
unsafe {
|
||||
libc::posix_fadvise(file.as_raw_fd(), 0, 0, libc::POSIX_FADV_SEQUENTIAL);
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop pages in the half-open byte range `[start, start+len)` from
|
||||
/// the page cache. Called periodically by `read_sectors` to bound the
|
||||
/// read-side page cache pressure.
|
||||
pub(super) fn drop_window(file: &File, start: u64, len: u64) {
|
||||
unsafe {
|
||||
libc::posix_fadvise(
|
||||
file.as_raw_fd(),
|
||||
start as i64,
|
||||
len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Async-prefetch `len` bytes at `offset` into the page cache. The
|
||||
/// kernel `readahead(2)` syscall queues the I/O and returns
|
||||
/// immediately — it does NOT wait for completion. Called right after
|
||||
/// each consumed read so the next batch's I/O overlaps with the
|
||||
/// caller's processing of the current batch (decrypt + demux + mux).
|
||||
///
|
||||
/// Without this hint, with a synchronous demux consumer running at
|
||||
/// ~50 MB/s and a single-spindle disk capable of ~150 MB/s, the disk
|
||||
/// sits idle ~70% of each iteration because kernel readahead alone
|
||||
/// (capped at `/sys/block/<dev>/queue/read_ahead_kb`, default 128 KB)
|
||||
/// can only pre-stage a tiny slice of the next batch. An explicit
|
||||
/// `readahead()` of the same size as the current batch tells the
|
||||
/// kernel to queue the full next-batch read now.
|
||||
pub(super) fn prefetch(file: &File, offset: u64, len: u64) {
|
||||
unsafe {
|
||||
libc::readahead(file.as_raw_fd(), offset as i64, len as usize);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
//! macOS: hint the kernel to prefetch a generous chunk. macOS has no
|
||||
//! direct `POSIX_FADV_SEQUENTIAL` equivalent; the idiomatic hint is
|
||||
//! `fcntl(F_RDADVISE, &radvisory)` describing the byte range you
|
||||
//! intend to read soon. We point it at the whole file (clamped to a
|
||||
//! ceiling so a multi-TB ISO doesn't ask the kernel to prefetch
|
||||
//! everything at once).
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
/// `F_RDADVISE` opcode — not in libc's named constants on all SDKs.
|
||||
const F_RDADVISE: libc::c_int = 44;
|
||||
|
||||
/// Cap on the byte length we pass to `F_RDADVISE`. Asking for a
|
||||
/// multi-GB readahead window is counterproductive — the OS doesn't
|
||||
/// have that much cache to throw at one fd. 64 MiB is generous for
|
||||
/// our use case (sweep, mux) and matches the byte-channel cap so the
|
||||
/// kernel's prefetch ≥ our app-level pipeline depth.
|
||||
const RDADVISE_MAX_BYTES: i64 = 64 * 1024 * 1024;
|
||||
|
||||
/// `radvisory` per `<sys/fcntl.h>`. repr(C) layout is stable.
|
||||
#[repr(C)]
|
||||
struct RadAdvisory {
|
||||
ra_offset: libc::off_t,
|
||||
ra_count: libc::c_int,
|
||||
}
|
||||
|
||||
pub(super) fn hint_sequential(file: &File, len_bytes: u64) {
|
||||
let bytes = (len_bytes as i64).min(RDADVISE_MAX_BYTES);
|
||||
let mut ra = RadAdvisory {
|
||||
ra_offset: 0,
|
||||
ra_count: bytes as libc::c_int,
|
||||
};
|
||||
// Best-effort.
|
||||
unsafe {
|
||||
libc::fcntl(file.as_raw_fd(), F_RDADVISE, &mut ra);
|
||||
}
|
||||
}
|
||||
|
||||
/// macOS has no direct `POSIX_FADV_DONTNEED` equivalent for a byte
|
||||
/// range. `fcntl(F_NOCACHE)` would disable caching globally on the fd
|
||||
/// (too coarse — we want the unread region to still benefit). Best
|
||||
/// approximation: no-op. macOS's unified buffer cache is generally
|
||||
/// less prone to the pin-everything pathology that triggers the
|
||||
/// regression on Linux NFS clients.
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
|
||||
/// Async-prefetch the byte range `[offset, offset+len)`. macOS uses
|
||||
/// the same `fcntl(F_RDADVISE, &radvisory)` primitive as the open-
|
||||
/// time sequential hint, just targeted at a moving window instead of
|
||||
/// the whole file. The kernel queues I/O for the requested range and
|
||||
/// returns immediately.
|
||||
pub(super) fn prefetch(file: &File, offset: u64, len: u64) {
|
||||
let bytes = (len as i64).min(RDADVISE_MAX_BYTES);
|
||||
let mut ra = RadAdvisory {
|
||||
ra_offset: offset as libc::off_t,
|
||||
ra_count: bytes as libc::c_int,
|
||||
};
|
||||
unsafe {
|
||||
libc::fcntl(file.as_raw_fd(), F_RDADVISE, &mut ra);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
//! [`FileSectorSource`] — read 2048-byte sectors from an ISO file on
|
||||
//! disk via direct `seek + read_exact` (`pread`-equivalent) calls,
|
||||
//! letting the kernel's own readahead policy manage prefetch.
|
||||
//!
|
||||
//! ## Why no app-level buffer
|
||||
//!
|
||||
//! Pre-0.21.3 this source held a 32 MiB (later 4 MiB) read-ahead
|
||||
//! buffer to amortise per-sector NFS round-trips. Empirically that
|
||||
//! buffer hurt: 32 MiB refills bursted the NFS TCP connection hard
|
||||
//! enough to starve the concurrent writer, and even a 4 MiB window
|
||||
//! gave the kernel less freedom to pipeline reads with writes. Direct
|
||||
//! pread per call lets Linux's readahead widen as it detects the
|
||||
//! sequential pattern, and naturally interleaves with writeback.
|
||||
//!
|
||||
//! ## DONTNEED on the consumed window
|
||||
//!
|
||||
//! Without page-cache eviction an 85 GB streaming ISO read pins the
|
||||
//! entire file in memory, starves the concurrent writer, and collapses
|
||||
//! mux throughput (observed: 2.7 MB/s mux on 0.21.5 vs. 70 MB/s
|
||||
//! isolated NFS reads). Every [`READ_DROP_CHUNK_BYTES`] of consumed
|
||||
//! bytes we call `posix_fadvise(DONTNEED)` over that window, mirroring
|
||||
//! the write-side [`crate::io::writeback::WritebackPipeline`] policy.
|
||||
//!
|
||||
//! ## Platform open hint
|
||||
//!
|
||||
//! On `open()` each platform issues its "sequential access expected"
|
||||
//! hint so OS-level readahead widens. The hint and the DONTNEED call
|
||||
//! live in per-OS sibling modules ([`linux::hint_sequential`] et al.)
|
||||
//! — no inline `#[cfg]` in this file.
|
||||
//!
|
||||
//! ## Read-ahead prefetch
|
||||
//!
|
||||
//! After every consumed read we issue an OS-level prefetch hint for
|
||||
//! the next equivalent-sized window (`platform::prefetch`). The
|
||||
//! kernel queues that I/O asynchronously and returns immediately, so
|
||||
//! the next batch's read overlaps with the caller's processing of
|
||||
//! the current batch (decrypt + demux + mux). Without this the disk
|
||||
//! sits idle ~70% of each iteration because kernel SEQUENTIAL
|
||||
//! readahead alone (capped at `read_ahead_kb`, default 128 KB) is
|
||||
//! far smaller than our 16 MiB app-level batch.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
mod other;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux as platform;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos as platform;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
use other as platform;
|
||||
#[cfg(target_os = "windows")]
|
||||
use windows as platform;
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::Path;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorSource;
|
||||
|
||||
const SECTOR_SIZE: usize = 2048;
|
||||
|
||||
/// Bytes-read threshold per `posix_fadvise(DONTNEED)` drop on the
|
||||
/// read side. Mirrors `WRITEBACK_CHUNK_BYTES` so the read-side page
|
||||
/// cache stays bounded the same way the write side does.
|
||||
///
|
||||
/// 32 MiB is the empirically tuned value on the rip1 test bed (single
|
||||
/// 7200rpm HDD via SATA): smaller windows (8 / 16 MiB) shorten the
|
||||
/// kernel-readahead overlap and slow the producer; larger windows
|
||||
/// (64 / 128 MiB) let the page cache pin enough of the ISO to
|
||||
/// pressure concurrent writes. Override via `FREEMKV_READ_DROP_CHUNK_MIB`.
|
||||
const READ_DROP_CHUNK_BYTES_DEFAULT: u64 = 32 * 1024 * 1024;
|
||||
|
||||
fn read_drop_chunk_bytes() -> u64 {
|
||||
std::env::var("FREEMKV_READ_DROP_CHUNK_MIB")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.filter(|&n| n > 0)
|
||||
.map(|n| n * 1024 * 1024)
|
||||
.unwrap_or(READ_DROP_CHUNK_BYTES_DEFAULT)
|
||||
}
|
||||
|
||||
/// SectorSource backed by a file (ISO image). Every `read_sectors`
|
||||
/// call is a direct `seek + read_exact` against the underlying file
|
||||
/// — kernel readahead handles prefetch, and every
|
||||
/// [`READ_DROP_CHUNK_BYTES_DEFAULT`] bytes of consumed data the
|
||||
/// platform's `DONTNEED` hook drops the consumed window from the
|
||||
/// page cache to bound memory pressure.
|
||||
pub struct FileSectorSource {
|
||||
file: File,
|
||||
/// Total file size in sectors. Constant after construction;
|
||||
/// surfaced via [`SectorSource::capacity_sectors`].
|
||||
capacity: u32,
|
||||
/// Bytes read since the last DONTNEED drop. Drives the per-
|
||||
/// [`read_drop_chunk_bytes`] page-cache eviction in read_sectors.
|
||||
bytes_read_since_drop: u64,
|
||||
/// File offset at which the current drop window starts. The next
|
||||
/// DONTNEED drops from `drop_window_start` for
|
||||
/// `bytes_read_since_drop` bytes.
|
||||
drop_window_start: u64,
|
||||
/// Cached drop chunk size (resolved from env once at open).
|
||||
drop_chunk_bytes: u64,
|
||||
}
|
||||
|
||||
impl FileSectorSource {
|
||||
/// Open an existing ISO file for reading. Capacity is derived
|
||||
/// from `metadata().len() / 2048`. Returns
|
||||
/// [`Error::IsoTooLarge`] if the file would exceed the 32-bit
|
||||
/// LBA address space (~8 TB).
|
||||
///
|
||||
/// Issues the platform's "sequential access expected" hint on the
|
||||
/// fd (Linux `posix_fadvise(SEQUENTIAL)`, macOS `fcntl(F_RDADVISE)`,
|
||||
/// Windows TODO stub) so the kernel's readahead widens.
|
||||
pub fn open(path: &Path) -> std::io::Result<Self> {
|
||||
let file = File::open(path)?;
|
||||
let len = file.metadata()?.len();
|
||||
let sectors = len / SECTOR_SIZE as u64;
|
||||
if sectors > u32::MAX as u64 {
|
||||
return Err(Error::IsoTooLarge {
|
||||
path: path.to_string_lossy().into_owned(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let capacity = sectors as u32;
|
||||
|
||||
// Best-effort sequential hint. Ignored on platforms without
|
||||
// an equivalent primitive (or where the API exists but the
|
||||
// FS doesn't honour it).
|
||||
platform::hint_sequential(&file, len);
|
||||
|
||||
Ok(Self {
|
||||
file,
|
||||
capacity,
|
||||
bytes_read_since_drop: 0,
|
||||
drop_window_start: 0,
|
||||
drop_chunk_bytes: read_drop_chunk_bytes(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for FileSectorSource {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
out: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
let count = count as u32;
|
||||
let bytes = count as usize * SECTOR_SIZE;
|
||||
debug_assert!(
|
||||
out.len() >= bytes,
|
||||
"FileSectorSource::read_sectors: out len {} < requested {}",
|
||||
out.len(),
|
||||
bytes
|
||||
);
|
||||
if count == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let offset = lba as u64 * SECTOR_SIZE as u64;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.read_exact(&mut out[..bytes])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
|
||||
// Queue the next batch's read with the kernel before the
|
||||
// caller starts processing what we just returned. readahead()
|
||||
// is non-blocking — it queues I/O and returns, so the kernel
|
||||
// pulls those pages into cache while the consumer (decrypt +
|
||||
// demux + mux) runs. Next read_sectors call hits a warm cache.
|
||||
platform::prefetch(&self.file, offset + bytes as u64, bytes as u64);
|
||||
|
||||
// Periodic page-cache eviction on the read side. Without
|
||||
// this, an 85 GB streaming ISO read pins the entire file in
|
||||
// the kernel page cache, which starves concurrent writes and
|
||||
// collapses mux throughput. Mirrors the write-side
|
||||
// WritebackPipeline's DONTNEED policy.
|
||||
self.bytes_read_since_drop += bytes as u64;
|
||||
if self.bytes_read_since_drop >= self.drop_chunk_bytes {
|
||||
let drop_start = self.drop_window_start;
|
||||
let drop_len = self.bytes_read_since_drop;
|
||||
platform::drop_window(&self.file, drop_start, drop_len);
|
||||
self.drop_window_start = drop_start + drop_len;
|
||||
self.bytes_read_since_drop = 0;
|
||||
}
|
||||
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
use tempfile::tempdir;
|
||||
|
||||
/// Build a deterministic ISO of `sectors` sectors where sector `n`
|
||||
/// is filled with the byte pattern `((n & 0xff) as u8)`. Lets us
|
||||
/// verify any sector by content alone.
|
||||
fn make_iso(path: &std::path::Path, sectors: u32) {
|
||||
let mut f = std::fs::File::create(path).unwrap();
|
||||
let mut chunk = vec![0u8; SECTOR_SIZE];
|
||||
for n in 0..sectors {
|
||||
let b = (n & 0xff) as u8;
|
||||
chunk.iter_mut().for_each(|c| *c = b);
|
||||
f.write_all(&chunk).unwrap();
|
||||
}
|
||||
f.flush().unwrap();
|
||||
}
|
||||
|
||||
/// Sectors used by spanning-boundary tests. Pick something that
|
||||
/// exercises multi-megabyte reads without making test ISOs huge.
|
||||
/// 8192 sectors = 16 MiB — large enough to cross any readahead
|
||||
/// chunk size we set the kernel hint to.
|
||||
const TEST_SPAN_SECTORS: u32 = 8192;
|
||||
|
||||
#[test]
|
||||
fn sequential_reads_match_file() {
|
||||
let total = TEST_SPAN_SECTORS * 2 + 17;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("seq.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), total);
|
||||
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
for lba in 0..total {
|
||||
src.read_sectors(lba, 1, &mut got, false).unwrap();
|
||||
let expected = (lba & 0xff) as u8;
|
||||
assert!(
|
||||
got.iter().all(|b| *b == expected),
|
||||
"sector {lba} content mismatch: expected 0x{expected:02x}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_sector_read_across_chunk_boundary() {
|
||||
let total = TEST_SPAN_SECTORS * 2;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("span.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
|
||||
let span_lba = TEST_SPAN_SECTORS - 2;
|
||||
let mut buf4 = vec![0u8; SECTOR_SIZE * 4];
|
||||
src.read_sectors(span_lba, 4, &mut buf4, false).unwrap();
|
||||
for i in 0..4 {
|
||||
let lba = span_lba + i as u32;
|
||||
let expected = (lba & 0xff) as u8;
|
||||
for b in &buf4[i * SECTOR_SIZE..(i + 1) * SECTOR_SIZE] {
|
||||
assert_eq!(*b, expected, "byte mismatch at sub-sector {i}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backward_seek_reads_correct_bytes() {
|
||||
// Read forward then jump back: the SectorSource contract is
|
||||
// byte-correctness regardless of access pattern.
|
||||
let total = TEST_SPAN_SECTORS * 2 + 5;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("back.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
|
||||
src.read_sectors(TEST_SPAN_SECTORS + 1, 1, &mut got, false)
|
||||
.unwrap();
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
assert!(got.iter().all(|b| *b == 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_at_eof_returns_correct_bytes() {
|
||||
// File smaller than the readahead chunk — reads near EOF must
|
||||
// still return correct bytes.
|
||||
let total: u32 = 100;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("small.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), total);
|
||||
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
src.read_sectors(total - 1, 1, &mut got, false).unwrap();
|
||||
let expected = ((total - 1) & 0xff) as u8;
|
||||
assert!(got.iter().all(|b| *b == expected));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_single_read() {
|
||||
// A multi-MB single read must work — the implementation has
|
||||
// no app-level chunking, so this just exercises the direct
|
||||
// pread path on a larger request.
|
||||
let total = TEST_SPAN_SECTORS + 100;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("big.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
let req = (TEST_SPAN_SECTORS + 1) as u16;
|
||||
let req_bytes = req as usize * SECTOR_SIZE;
|
||||
let mut big = vec![0u8; req_bytes];
|
||||
src.read_sectors(0, req, &mut big, false).unwrap();
|
||||
assert!(big[..SECTOR_SIZE].iter().all(|b| *b == 0));
|
||||
let last_lba = req as u32 - 1;
|
||||
let exp = (last_lba & 0xff) as u8;
|
||||
let last_off = (req as usize - 1) * SECTOR_SIZE;
|
||||
assert!(
|
||||
big[last_off..last_off + SECTOR_SIZE]
|
||||
.iter()
|
||||
.all(|b| *b == exp)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_chunk_size_env_override() {
|
||||
// Explicit 8 MiB via env var.
|
||||
// SAFETY: tests in this crate are single-threaded per the
|
||||
// default cargo test harness, but std::env::set_var is
|
||||
// declared `unsafe` since Rust 2024 (it can race with other
|
||||
// threads / TLS). For a test that runs in-process before any
|
||||
// FileSectorSource construction this is safe in practice.
|
||||
unsafe {
|
||||
std::env::set_var("FREEMKV_READ_DROP_CHUNK_MIB", "8");
|
||||
}
|
||||
assert_eq!(read_drop_chunk_bytes(), 8 * 1024 * 1024);
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("FREEMKV_READ_DROP_CHUNK_MIB");
|
||||
}
|
||||
assert_eq!(read_drop_chunk_bytes(), READ_DROP_CHUNK_BYTES_DEFAULT);
|
||||
|
||||
// Garbage env value falls back to default.
|
||||
unsafe {
|
||||
std::env::set_var("FREEMKV_READ_DROP_CHUNK_MIB", "not-a-number");
|
||||
}
|
||||
assert_eq!(read_drop_chunk_bytes(), READ_DROP_CHUNK_BYTES_DEFAULT);
|
||||
unsafe {
|
||||
std::env::remove_var("FREEMKV_READ_DROP_CHUNK_MIB");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
//! Fallback for targets without a known sequential-readahead hint
|
||||
//! (BSDs, illumos, etc.). No-op — reads still work, they just don't
|
||||
//! get the OS-level prefetch widening.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn hint_sequential(_file: &File, _len_bytes: u64) {}
|
||||
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
|
||||
pub(super) fn prefetch(_file: &File, _offset: u64, _len: u64) {}
|
||||
@@ -0,0 +1,30 @@
|
||||
//! Windows: the canonical sequential-access hint is
|
||||
//! `FILE_FLAG_SEQUENTIAL_SCAN` passed to `CreateFile` at open time —
|
||||
//! it cannot be set after the fact via `SetFileInformationByHandle`.
|
||||
//! Routing the open call through this module would mean a custom
|
||||
//! `File::from_raw_handle` plumb for every `FileSectorSource::open`
|
||||
//! caller, which is more invasive than the Phase 1 scope.
|
||||
//!
|
||||
//! TODO: replumb `FileSectorSource::open` to take an
|
||||
//! `OpenOptions`-style builder so the Windows path can flip the flag
|
||||
//! at open time. For now this is a no-op stub.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn hint_sequential(_file: &File, _len_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"FileSectorSource hint_sequential: windows stub (TODO: FILE_FLAG_SEQUENTIAL_SCAN at open)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Windows page-cache eviction is not exposed via a posix_fadvise
|
||||
/// equivalent. The kernel does its own working-set management. No-op
|
||||
/// for now.
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
|
||||
/// Windows async-prefetch hint. With FILE_FLAG_SEQUENTIAL_SCAN at
|
||||
/// open the kernel already prefetches aggressively, so there's no
|
||||
/// per-range hint we'd add on top. No-op stub for parity with the
|
||||
/// posix platforms.
|
||||
pub(super) fn prefetch(_file: &File, _offset: u64, _len: u64) {}
|
||||
@@ -0,0 +1,48 @@
|
||||
//! File I/O helpers that bound kernel cache pressure on big writes.
|
||||
//!
|
||||
//! `WritebackFile` is a drop-in wrapper around `std::fs::File` for any
|
||||
//! call site that performs large sequential writes (sweep, patch, mux,
|
||||
//! etc.). It implements `Write` and `Seek` so existing code paths can
|
||||
//! swap `File` for `WritebackFile` with no body changes. Internally it
|
||||
//! drives a `WritebackPipeline` that, on Linux, drains dirty pages
|
||||
//! continuously at 32 MB granularity to avoid the kernel's
|
||||
//! accumulate-then-burst flush behaviour. macOS and Windows use a
|
||||
//! no-op pipeline — their default cache policies have not been shown
|
||||
//! to exhibit the same pathology for this access pattern.
|
||||
//!
|
||||
//! `FileSectorSource` is the read-side dual — it implements
|
||||
//! [`crate::sector::SectorSource`] for an ISO file using direct
|
||||
//! `pread`-equivalent calls so the kernel's own readahead policy runs
|
||||
//! (which interleaves naturally with the concurrent writeback). It
|
||||
//! pairs that with periodic `posix_fadvise(DONTNEED)` drops on the
|
||||
//! consumed window so an 85 GB streaming ISO read doesn't fill the
|
||||
//! page cache and starve the concurrent MKV write.
|
||||
//!
|
||||
//! `Pipeline` + `Sink` (0.18) is the generic producer/consumer primitive
|
||||
//! used by sweep, patch, and mux to overlap reads with writes via a
|
||||
//! bounded channel + dedicated consumer thread.
|
||||
//!
|
||||
//! `byte_channel` is a byte-sized producer/consumer channel for the
|
||||
//! mux pipeline, sized to absorb worst-case input read stalls.
|
||||
|
||||
pub(crate) mod bounded;
|
||||
pub mod byte_channel;
|
||||
pub mod byte_prefetcher;
|
||||
pub mod file_sector_source;
|
||||
pub mod sink;
|
||||
mod writeback;
|
||||
mod writeback_file;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(crate) mod platform_macos;
|
||||
|
||||
pub mod pipeline;
|
||||
|
||||
pub(crate) use writeback_file::WritebackFile;
|
||||
|
||||
// Re-exports for the 0.18 redesign. Sweep, patch, and mux are all
|
||||
// wired up (disc/sweep.rs, disc/patch.rs, autorip's ripper/mux.rs).
|
||||
pub use pipeline::{
|
||||
DEFAULT_PIPELINE_DEPTH, Flow, Pipeline, READ_PIPELINE_DEPTH, Sink, WRITE_PIPELINE_DEPTH,
|
||||
WRITE_THROUGH_DEPTH,
|
||||
};
|
||||
@@ -0,0 +1,913 @@
|
||||
//! Generic bounded producer/consumer pipeline.
|
||||
//!
|
||||
//! `Pipeline<I, R>` spawns a single consumer thread, hands it items
|
||||
//! through a bounded `mpsc::sync_channel`, and joins it on `finish()`.
|
||||
//! The consumer's behaviour is supplied by a [`Sink`] implementation:
|
||||
//! `apply` is called once per item, `close` is called once at the end.
|
||||
//!
|
||||
//! Three call sites in libfreemkv want a producer/consumer split —
|
||||
//! sweep (migrated to `disc/sweep.rs::SweepSink`), patch, and mux.
|
||||
//! 0.18 collapses all three onto this primitive; sweep is in,
|
||||
//! patch and mux migrate in later 0.18 slices.
|
||||
//!
|
||||
//! ## Cancellation and error semantics
|
||||
//!
|
||||
//! - Producer dropping the channel (via `Pipeline::finish` dropping
|
||||
//! `tx`) signals end-of-stream; consumer flushes via `close()` and
|
||||
//! returns its `Output`.
|
||||
//! - Consumer returning [`Flow::Stop`] also calls `close()` and
|
||||
//! returns its `Output`. `send()` from the producer will then either
|
||||
//! succeed (if the item already fit in the channel buffer) or fail
|
||||
//! with `Err(item)` once the consumer has dropped its receiver.
|
||||
//! - Consumer returning `Err` from `apply` skips `close()` entirely;
|
||||
//! the consumer keeps draining the channel so the producer never
|
||||
//! blocks on a dead receiver, and the first error is propagated as
|
||||
//! the `JoinHandle` result.
|
||||
//! - Consumer panic is converted into
|
||||
//! `Error::IoError { source: io::Error::other(...) }`.
|
||||
//!
|
||||
//! ## Debug logging
|
||||
//!
|
||||
//! Set `FREEMKV_DEBUG=1` environment variable to enable verbose debug
|
||||
//! logging throughout the pipeline (channel sends/receives, backpressure,
|
||||
//! consumer lag detection). This is critical for diagnosing stalls.
|
||||
|
||||
use std::io;
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crossbeam_channel::{Sender, TrySendError, bounded};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::halt::Halt;
|
||||
|
||||
/// Deadline for [`Pipeline::finish_with_halt`]'s polling join. Chosen
|
||||
/// to be comfortably longer than the autorip hard watchdog
|
||||
/// (`HARD_WATCHDOG_STALL_SECS = 300s`) so the watchdog's `exit(1)`
|
||||
/// fires first when both are racing on the same wedged consumer.
|
||||
///
|
||||
/// 10 minutes is a backstop, not a normal timeout — the consumer is
|
||||
/// expected to drain in seconds. If we hit this, something is wedged
|
||||
/// inside a kernel call the consumer thread can't unwind from, and the
|
||||
/// caller has already lost the rip.
|
||||
pub const JOIN_TIMEOUT_SECS: u64 = 600;
|
||||
|
||||
/// Halt-check cadence for the send loop. Producer blocks on
|
||||
/// [`crossbeam_channel::Sender::send_timeout`] for this slice — the
|
||||
/// kernel wakes it the instant the consumer drains a slot, so on the
|
||||
/// happy path there's no throughput cap from this primitive at all
|
||||
/// (the cap is whatever the underlying medium can sustain). When the
|
||||
/// consumer is genuinely wedged, the timeout fires every
|
||||
/// [`crate::halt::POLL_INTERVAL`] and the producer checks the halt
|
||||
/// token; that's the latency a stop request will observe.
|
||||
///
|
||||
/// Single source of truth lives in [`crate::halt::POLL_INTERVAL`]
|
||||
/// (also used by `bounded_syscall`). Aliased here for readability of
|
||||
/// the send/finish call sites below.
|
||||
///
|
||||
/// 0.21.7 replaced an old `std::sync::mpsc::sync_channel` + 50 ms
|
||||
/// `thread::sleep` polling loop that capped mux throughput at
|
||||
/// ~20 frames/sec ≈ 1 MB/s on saturated channels.
|
||||
use crate::halt::POLL_INTERVAL;
|
||||
const SEND_HALT_CHECK_INTERVAL: Duration = POLL_INTERVAL;
|
||||
|
||||
/// Check if verbose debug logging is enabled via FREEMKV_DEBUG env var.
|
||||
pub fn debug_enabled() -> bool {
|
||||
std::env::var("FREEMKV_DEBUG")
|
||||
.ok()
|
||||
.map(|v| v == "1" || v == "true" || v == "yes")
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Default channel depth for callers without a specific reason to
|
||||
/// pick another value. Kept conservative (4) — most callers should
|
||||
/// use READ_PIPELINE_DEPTH or WRITE_PIPELINE_DEPTH instead.
|
||||
pub const DEFAULT_PIPELINE_DEPTH: usize = 4;
|
||||
|
||||
/// Read pipeline depth. Larger buffer compensates for drive variability
|
||||
/// and NFS sync_file_range stalls; keeps ISO reader thread fed even when
|
||||
/// consumer blocks on write.
|
||||
pub const READ_PIPELINE_DEPTH: usize = 32;
|
||||
|
||||
/// Write pipeline depth. Smaller buffer reduces backpressure risk when
|
||||
/// sync_file_range blocks; prevents producer from accumulating too much
|
||||
/// work while consumer waits for NFS to drain.
|
||||
pub const WRITE_PIPELINE_DEPTH: usize = 16;
|
||||
|
||||
/// Channel depth for write-through pipelines. Each `send` fully
|
||||
/// drains before the next can enqueue. Use this when the producer
|
||||
/// must observe consumer side-effects (e.g. mapfile state) before
|
||||
/// emitting the next item. Currently used by `disc::patch`.
|
||||
pub const WRITE_THROUGH_DEPTH: usize = 1;
|
||||
|
||||
/// Outcome of [`Sink::apply`]: either keep feeding items
|
||||
/// ([`Flow::Continue`]), or stop the pipeline early and run `close()`
|
||||
/// ([`Flow::Stop`]).
|
||||
///
|
||||
/// `Stop` has no in-tree caller in this slice — sweep never returns
|
||||
/// it (it always processes the producer's full work-list before the
|
||||
/// channel is dropped). Patch and mux are the intended consumers and
|
||||
/// migrate in later 0.18 slices. The variant ships now so the contract
|
||||
/// is fixed; the targeted `#[allow]` is removed when patch lands.
|
||||
pub enum Flow {
|
||||
Continue,
|
||||
#[allow(dead_code)]
|
||||
Stop,
|
||||
}
|
||||
|
||||
/// Consumer-side of a [`Pipeline`]. The pipeline owns one of these on
|
||||
/// its consumer thread and calls `apply` once per received item, then
|
||||
/// `close` once at end-of-stream.
|
||||
pub trait Sink<I>: Send + 'static {
|
||||
/// Type returned from `close()` and surfaced via
|
||||
/// [`Pipeline::finish`].
|
||||
type Output: Send + 'static;
|
||||
|
||||
/// Apply one item. Returning [`Flow::Continue`] keeps the
|
||||
/// pipeline running; [`Flow::Stop`] ends it cleanly (still calls
|
||||
/// `close()`). An error short-circuits: `close()` is *not* called
|
||||
/// and the error is what `finish()` will return, but the consumer
|
||||
/// keeps draining the channel so the producer never blocks on a
|
||||
/// dead receiver.
|
||||
fn apply(&mut self, item: I) -> Result<Flow, Error>;
|
||||
|
||||
/// Called once at end-of-stream — either because the producer
|
||||
/// dropped `tx` or because `apply` returned [`Flow::Stop`]. Use
|
||||
/// this to flush, fsync, finalise. Skipped if any prior `apply`
|
||||
/// returned `Err`.
|
||||
fn close(self) -> Result<Self::Output, Error>;
|
||||
}
|
||||
|
||||
/// Bounded producer/consumer pipeline. Holds the producer-side
|
||||
/// channel and the consumer thread's join handle.
|
||||
pub struct Pipeline<I: Send + 'static, R: Send + 'static> {
|
||||
tx: Sender<I>,
|
||||
handle: JoinHandle<Result<R, Error>>,
|
||||
}
|
||||
|
||||
impl<I: Send + 'static, R: Send + 'static> Pipeline<I, R> {
|
||||
/// Spawn the consumer thread with the given channel depth and
|
||||
/// [`Sink`].
|
||||
///
|
||||
/// The thread is named `freemkv-pipeline-consumer` so it shows up
|
||||
/// distinctly in stack traces and `top -H`. Callers that want a
|
||||
/// more specific name (e.g. `freemkv-sweep-consumer`) should use
|
||||
/// [`Pipeline::spawn_named`] instead. Returns an `Error::IoError`
|
||||
/// if the OS refuses the thread spawn (resource exhaustion);
|
||||
/// callers already operate in fallible context, so this is
|
||||
/// propagated rather than panicked.
|
||||
///
|
||||
/// Sweep uses [`Pipeline::spawn_named`] directly so the consumer
|
||||
/// thread shows up as `freemkv-sweep-consumer`; mux uses
|
||||
/// `freemkv-mux-consumer`. `Pipeline::spawn` (this function, with
|
||||
/// the default name) is used by `disc::patch` and by the unit
|
||||
/// tests in this module.
|
||||
pub fn spawn<S: Sink<I, Output = R>>(depth: usize, sink: S) -> Result<Self, Error> {
|
||||
Self::spawn_named("freemkv-pipeline-consumer", depth, sink)
|
||||
}
|
||||
|
||||
/// Like [`Pipeline::spawn`] but lets the caller supply the
|
||||
/// consumer thread's name. Useful when several pipelines run in
|
||||
/// the same process and stack traces / `top -H` need to tell them
|
||||
/// apart (e.g. `freemkv-sweep-consumer`, `freemkv-mux-consumer`).
|
||||
pub fn spawn_named<S: Sink<I, Output = R>>(
|
||||
name: &str,
|
||||
depth: usize,
|
||||
sink: S,
|
||||
) -> Result<Self, Error> {
|
||||
let (tx, rx) = bounded::<I>(depth);
|
||||
let handle = thread::Builder::new()
|
||||
.name(name.into())
|
||||
.spawn(move || -> Result<R, Error> {
|
||||
let mut sink = sink;
|
||||
let mut first_err: Option<Error> = None;
|
||||
let mut stopped = false;
|
||||
|
||||
while let Ok(item) = rx.recv() {
|
||||
if debug_enabled() {
|
||||
tracing::debug!("Pipeline receive: item={}", std::any::type_name::<I>());
|
||||
}
|
||||
|
||||
let apply_start = std::time::Instant::now();
|
||||
|
||||
if first_err.is_some() || stopped {
|
||||
// Drain remaining items so the producer never
|
||||
// blocks on a dead receiver. `apply` is not
|
||||
// called once we've decided to stop.
|
||||
continue;
|
||||
}
|
||||
match sink.apply(item) {
|
||||
Ok(Flow::Continue) => {}
|
||||
Ok(Flow::Stop) => {
|
||||
stopped = true;
|
||||
if debug_enabled() {
|
||||
tracing::debug!("Pipeline: consumer returned Flow::Stop");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
if debug_enabled() {
|
||||
tracing::debug!("Pipeline: apply error, stopping, err={:?}", e);
|
||||
}
|
||||
first_err = Some(e);
|
||||
}
|
||||
}
|
||||
|
||||
let apply_elapsed = apply_start.elapsed();
|
||||
if debug_enabled() && apply_elapsed > std::time::Duration::from_millis(100) {
|
||||
tracing::debug!(
|
||||
"Pipeline apply: took {:.2}s, item={}",
|
||||
apply_elapsed.as_secs_f64(),
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
} else if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline apply: OK in {:.3}ms, item={}",
|
||||
apply_elapsed.as_micros(),
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
match first_err {
|
||||
Some(e) => Err(e),
|
||||
None => sink.close(),
|
||||
}
|
||||
})
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
|
||||
Ok(Pipeline { tx, handle })
|
||||
}
|
||||
|
||||
/// Push one item. Blocks if the channel is full — that's the
|
||||
/// back-pressure the whole primitive exists to provide. Returns
|
||||
/// the item back if the consumer thread is gone (panicked or
|
||||
/// already returned).
|
||||
///
|
||||
/// After the consumer returns [`Flow::Stop`], `send` will silently
|
||||
/// buffer items into the channel until the channel fills, then
|
||||
/// return `Err(item)` once the consumer has dropped its receiver.
|
||||
/// Producers that need to stop pushing on `Stop` should track an
|
||||
/// independent signal (e.g. `Halt`) — `send` alone is not the
|
||||
/// notification edge.
|
||||
pub fn send(&self, item: I) -> Result<(), I> {
|
||||
let start = std::time::Instant::now();
|
||||
match self.tx.send(item) {
|
||||
Ok(()) => {
|
||||
let elapsed = start.elapsed();
|
||||
if debug_enabled() && elapsed > std::time::Duration::from_millis(10) {
|
||||
tracing::debug!(
|
||||
"Pipeline send: blocked {:.2}s, item={}",
|
||||
elapsed.as_secs_f64(),
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
} else if debug_enabled() {
|
||||
tracing::debug!("Pipeline send: OK in {:.3}ms", elapsed.as_micros());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
let elapsed = start.elapsed();
|
||||
if debug_enabled() && elapsed > std::time::Duration::from_millis(10) {
|
||||
tracing::debug!(
|
||||
"Pipeline send: blocked {:.2}s before channel closed, item={}",
|
||||
elapsed.as_secs_f64(),
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
} else if debug_enabled() {
|
||||
tracing::debug!("Pipeline send: failed after {:.3}ms", elapsed.as_micros());
|
||||
}
|
||||
Err(e.0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Non-blocking variant of [`Pipeline::send`]. If the channel is
|
||||
/// full or the consumer has hung up, the item is returned in
|
||||
/// `Err`. Useful for best-effort signalling (e.g. sweep's
|
||||
/// throttled `StatsRequest`) where dropping the message is
|
||||
/// preferable to blocking the producer.
|
||||
pub fn try_send(&self, item: I) -> Result<(), TrySendError<I>> {
|
||||
self.tx.try_send(item)
|
||||
}
|
||||
|
||||
/// Halt-aware bounded variant of [`Pipeline::send`].
|
||||
///
|
||||
/// Uses [`crossbeam_channel::Sender::send_timeout`] so the producer
|
||||
/// thread BLOCKS on consumer drain (kernel-wakeup) rather than
|
||||
/// polling. The timeout slice is just the halt-observation cadence
|
||||
/// ([`SEND_HALT_CHECK_INTERVAL`]) — on the happy path the producer
|
||||
/// wakes the instant the consumer drains a slot, so there is no
|
||||
/// throughput cap from this primitive at any medium speed.
|
||||
///
|
||||
/// Returns:
|
||||
///
|
||||
/// - `Ok(())` once the item lands in the channel.
|
||||
/// - `Err(item)` if the consumer disconnected, the halt fired, or
|
||||
/// the deadline elapsed — the caller gets the item back so it
|
||||
/// can decide whether to drop it, route it elsewhere, or unwind.
|
||||
///
|
||||
/// Use this in producer threads that have a `Halt` token threaded
|
||||
/// through (mux, sweep, patch). Plain [`Pipeline::send`] is
|
||||
/// preserved for callers that don't (yet) plumb halt through.
|
||||
///
|
||||
/// Unlike [`Pipeline::send`], this never blocks the producer
|
||||
/// thread inside an unkillable `mpsc::send` — if the consumer is
|
||||
/// wedged inside an unkillable syscall, the producer can still
|
||||
/// observe `/api/stop` and unwind within
|
||||
/// [`SEND_HALT_CHECK_INTERVAL`].
|
||||
pub fn send_with_halt(&self, item: I, halt: &Halt, deadline: Duration) -> Result<(), I> {
|
||||
use crossbeam_channel::SendTimeoutError;
|
||||
let end = Instant::now() + deadline;
|
||||
let mut pending = item;
|
||||
loop {
|
||||
// Pre-check the cheap exit conditions before parking.
|
||||
if halt.is_cancelled() {
|
||||
if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline send_with_halt: halt observed, returning item={}",
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
return Err(pending);
|
||||
}
|
||||
let now = Instant::now();
|
||||
if now >= end {
|
||||
if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline send_with_halt: deadline elapsed, returning item={}",
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
return Err(pending);
|
||||
}
|
||||
// Wait for space-available or halt-check tick, whichever
|
||||
// is sooner. Crossbeam's send_timeout is kernel-wakeup
|
||||
// based: the consumer's recv on a saturated channel
|
||||
// signals this thread the moment a slot opens up.
|
||||
let slice = SEND_HALT_CHECK_INTERVAL.min(end.saturating_duration_since(now));
|
||||
match self.tx.send_timeout(pending, slice) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(SendTimeoutError::Timeout(returned)) => {
|
||||
pending = returned;
|
||||
// loop: re-check halt + deadline, then park again
|
||||
}
|
||||
Err(SendTimeoutError::Disconnected(returned)) => {
|
||||
if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline send_with_halt: consumer disconnected, item={}",
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
return Err(returned);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the producer-side channel and wait for the consumer
|
||||
/// thread to finish. Returns whatever the consumer's `close()`
|
||||
/// produced, or the first `apply` error, or — on consumer panic —
|
||||
/// an `Error::IoError` whose source is `io::Error::other(...)`
|
||||
/// with a "pipeline consumer panicked: <payload>" message
|
||||
/// (callers can match on the constant prefix).
|
||||
pub fn finish(self) -> Result<R, Error> {
|
||||
let Pipeline { tx, handle } = self;
|
||||
// Explicit drop, although the destructure already drops `tx`
|
||||
// at end-of-scope. Being explicit keeps the intent obvious.
|
||||
drop(tx);
|
||||
match handle.join() {
|
||||
Ok(result) => result,
|
||||
Err(payload) => {
|
||||
// Preserve the original panic message when the
|
||||
// consumer's panic payload was a `&str` or `String`
|
||||
// (the two stdlib formats that `panic!` produces).
|
||||
// Anything else falls back to "(no message)".
|
||||
let msg = payload
|
||||
.downcast_ref::<&'static str>()
|
||||
.copied()
|
||||
.or_else(|| payload.downcast_ref::<String>().map(|s| s.as_str()))
|
||||
.unwrap_or("(no message)");
|
||||
Err(Error::IoError {
|
||||
source: io::Error::other(format!("pipeline consumer panicked: {msg}")),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Halt-aware, deadline-bounded variant of [`Pipeline::finish`].
|
||||
///
|
||||
/// Drops the producer-side channel (same as `finish`) and then
|
||||
/// polls `JoinHandle::is_finished()` on a 250 ms cadence. Between
|
||||
/// slices, checks (1) the optional [`Halt`] token and (2) the
|
||||
/// [`JOIN_TIMEOUT_SECS`] deadline. Returns:
|
||||
///
|
||||
/// - `Ok(R)` on a clean consumer exit.
|
||||
/// - `Err(Error::IoError)` with one of three message prefixes for
|
||||
/// wedge cases:
|
||||
/// - `"pipeline join halted"` — halt fired while waiting.
|
||||
/// - `"pipeline join timed out"` — `JOIN_TIMEOUT_SECS` elapsed.
|
||||
/// - `"pipeline consumer panicked"` — same as `finish()`.
|
||||
///
|
||||
/// In the `halted` and `timed out` branches the consumer thread is
|
||||
/// intentionally leaked — exactly the same trade-off the
|
||||
/// `bounded_syscall` primitive makes. The wedged kernel call
|
||||
/// inside the consumer will unwind whenever it does, or at
|
||||
/// process exit. The caller is free to fall back to a degraded
|
||||
/// path (in autorip's case: `exit(1)` after the hard watchdog
|
||||
/// escalation, letting Docker restart the container).
|
||||
///
|
||||
/// Plain [`Pipeline::finish`] is preserved for callers without a
|
||||
/// halt-token plumbed through; that path still blocks indefinitely
|
||||
/// on `join()`, matching pre-0.20.8 behaviour.
|
||||
pub fn finish_with_halt(self, halt: Option<&Halt>) -> Result<R, Error> {
|
||||
let Pipeline { tx, handle } = self;
|
||||
drop(tx);
|
||||
let deadline = Instant::now() + Duration::from_secs(JOIN_TIMEOUT_SECS);
|
||||
loop {
|
||||
if handle.is_finished() {
|
||||
return match handle.join() {
|
||||
Ok(result) => result,
|
||||
Err(payload) => {
|
||||
let msg = payload
|
||||
.downcast_ref::<&'static str>()
|
||||
.copied()
|
||||
.or_else(|| payload.downcast_ref::<String>().map(|s| s.as_str()))
|
||||
.unwrap_or("(no message)");
|
||||
Err(Error::IoError {
|
||||
source: io::Error::other(format!("pipeline consumer panicked: {msg}")),
|
||||
})
|
||||
}
|
||||
};
|
||||
}
|
||||
if let Some(h) = halt {
|
||||
if h.is_cancelled() {
|
||||
// Consumer thread is intentionally leaked.
|
||||
return Err(Error::IoError {
|
||||
source: io::Error::other("pipeline join halted"),
|
||||
});
|
||||
}
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
// Consumer thread is intentionally leaked.
|
||||
return Err(Error::IoError {
|
||||
source: io::Error::other("pipeline join timed out"),
|
||||
});
|
||||
}
|
||||
thread::sleep(POLL_INTERVAL);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Sums u64s; returns the total from `close`.
|
||||
struct SumSink {
|
||||
total: u64,
|
||||
}
|
||||
|
||||
impl Sink<u64> for SumSink {
|
||||
type Output = u64;
|
||||
|
||||
fn apply(&mut self, item: u64) -> Result<Flow, Error> {
|
||||
self.total += item;
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(self) -> Result<u64, Error> {
|
||||
Ok(self.total)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn happy_path_sums_items() {
|
||||
let pipe = Pipeline::spawn(DEFAULT_PIPELINE_DEPTH, SumSink { total: 0 })
|
||||
.expect("spawn should succeed");
|
||||
let mut expected = 0u64;
|
||||
for i in 0..100u64 {
|
||||
expected += i;
|
||||
pipe.send(i).expect("send should succeed");
|
||||
}
|
||||
let total = pipe.finish().expect("finish should succeed");
|
||||
assert_eq!(total, expected);
|
||||
assert_eq!(total, (0..100u64).sum::<u64>());
|
||||
}
|
||||
|
||||
/// Sleeps `delay` per apply; counts how many it received.
|
||||
struct SlowSink {
|
||||
delay: Duration,
|
||||
count: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Sink<()> for SlowSink {
|
||||
type Output = usize;
|
||||
|
||||
fn apply(&mut self, _item: ()) -> Result<Flow, Error> {
|
||||
std::thread::sleep(self.delay);
|
||||
self.count.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(self) -> Result<usize, Error> {
|
||||
Ok(self.count.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn back_pressure_blocks_sender() {
|
||||
// depth=2 + 5 sends + 50ms/apply: with the consumer pinned at
|
||||
// 50 ms per item, the producer can buffer 2 (channel cap) +
|
||||
// 1 (consumer in flight) = 3 items before sends 4 and 5 must
|
||||
// block on consumer progress. Wall-clock floor across all 5
|
||||
// sends is therefore ~2 * 50ms = 100ms (sends 4 and 5 each
|
||||
// wait roughly one apply-cycle). Use 80 ms as the assertion
|
||||
// floor to stay above the 50ms-per-item progress floor while
|
||||
// tolerating CI jitter — it still proves blocking is real.
|
||||
let count = Arc::new(AtomicUsize::new(0));
|
||||
let sink = SlowSink {
|
||||
delay: Duration::from_millis(50),
|
||||
count: count.clone(),
|
||||
};
|
||||
let pipe = Pipeline::spawn(2, sink).expect("spawn should succeed");
|
||||
|
||||
let start = Instant::now();
|
||||
for _ in 0..5 {
|
||||
pipe.send(()).expect("send should succeed");
|
||||
}
|
||||
let elapsed_send = start.elapsed();
|
||||
|
||||
let total = pipe.finish().expect("finish should succeed");
|
||||
assert_eq!(total, 5);
|
||||
assert!(
|
||||
elapsed_send >= Duration::from_millis(80),
|
||||
"back-pressure not observed: 5 sends with depth=2 and 50ms/apply \
|
||||
took {elapsed_send:?}, expected ≥ ~100ms (one or more sends \
|
||||
should have blocked behind the consumer)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns `Err` on the Nth apply (1-indexed). Tracks all calls.
|
||||
struct FailOnNthSink {
|
||||
n: usize,
|
||||
seen: Arc<AtomicUsize>,
|
||||
close_called: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Sink<u64> for FailOnNthSink {
|
||||
type Output = ();
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
let i = self.seen.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if i == self.n {
|
||||
Err(Error::DecryptFailed)
|
||||
} else {
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
fn close(self) -> Result<(), Error> {
|
||||
self.close_called.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_error_drains_then_propagates() {
|
||||
let seen = Arc::new(AtomicUsize::new(0));
|
||||
let close_called = Arc::new(AtomicUsize::new(0));
|
||||
let pipe = Pipeline::spawn(
|
||||
DEFAULT_PIPELINE_DEPTH,
|
||||
FailOnNthSink {
|
||||
n: 3,
|
||||
seen: seen.clone(),
|
||||
close_called: close_called.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
|
||||
// Send 10 items. Subsequent sends after the 3rd error must
|
||||
// still succeed (the consumer is draining).
|
||||
for i in 0..10u64 {
|
||||
pipe.send(i).expect("send should succeed even after error");
|
||||
}
|
||||
|
||||
let res = pipe.finish();
|
||||
assert!(matches!(res, Err(Error::DecryptFailed)));
|
||||
assert_eq!(
|
||||
close_called.load(Ordering::SeqCst),
|
||||
0,
|
||||
"close() must not be called when apply returned Err"
|
||||
);
|
||||
// The consumer kept calling `recv` to drain after the error;
|
||||
// it just stopped invoking `apply`. So `seen` is exactly 3
|
||||
// (apply was called for items 1, 2, 3).
|
||||
assert_eq!(seen.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
/// Returns `Flow::Stop` on the Nth apply.
|
||||
struct StopOnNthSink {
|
||||
n: usize,
|
||||
seen: Arc<AtomicUsize>,
|
||||
close_called: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Sink<u64> for StopOnNthSink {
|
||||
type Output = usize;
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
let i = self.seen.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if i >= self.n {
|
||||
Ok(Flow::Stop)
|
||||
} else {
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
}
|
||||
|
||||
fn close(self) -> Result<usize, Error> {
|
||||
self.close_called.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(self.seen.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_stop_calls_close_and_returns_output() {
|
||||
let seen = Arc::new(AtomicUsize::new(0));
|
||||
let close_called = Arc::new(AtomicUsize::new(0));
|
||||
let pipe = Pipeline::spawn(
|
||||
DEFAULT_PIPELINE_DEPTH,
|
||||
StopOnNthSink {
|
||||
n: 3,
|
||||
seen: seen.clone(),
|
||||
close_called: close_called.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
|
||||
// Send 10 items. After Stop, subsequent sends may either
|
||||
// succeed (already buffered) or fail with Err(I) (channel
|
||||
// closed). Both are valid — we don't assert on the send
|
||||
// results.
|
||||
for i in 0..10u64 {
|
||||
let _ = pipe.send(i);
|
||||
}
|
||||
|
||||
let out = pipe.finish().expect("finish should succeed after Stop");
|
||||
assert_eq!(close_called.load(Ordering::SeqCst), 1);
|
||||
// At least 3 items processed (the one that returned Stop).
|
||||
assert!(
|
||||
out >= 3,
|
||||
"expected ≥ 3 applies before Stop took effect, got {out}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Panics on the first apply call.
|
||||
struct PanickingSink;
|
||||
|
||||
impl Sink<u64> for PanickingSink {
|
||||
type Output = ();
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
panic!("synthetic test panic");
|
||||
}
|
||||
|
||||
fn close(self) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consumer_panic_becomes_io_error() {
|
||||
// Silence the panic message that would otherwise pollute the
|
||||
// test output — we expect this panic.
|
||||
let prev = std::panic::take_hook();
|
||||
std::panic::set_hook(Box::new(|_| {}));
|
||||
|
||||
let pipe =
|
||||
Pipeline::spawn(DEFAULT_PIPELINE_DEPTH, PanickingSink).expect("spawn should succeed");
|
||||
// First send may succeed (item buffered before panic) or fail
|
||||
// (channel closed after panic) — either is fine.
|
||||
let _ = pipe.send(1);
|
||||
// Drain a few more sends; once the channel is closed they'll
|
||||
// return Err(I), which we just discard.
|
||||
for i in 0..5u64 {
|
||||
let _ = pipe.send(i);
|
||||
}
|
||||
let res = pipe.finish();
|
||||
|
||||
std::panic::set_hook(prev);
|
||||
|
||||
match res {
|
||||
Err(Error::IoError { source }) => {
|
||||
let msg = source.to_string();
|
||||
// Constant prefix lets callers match without parsing
|
||||
// the variable payload tail.
|
||||
assert!(
|
||||
msg.contains("pipeline consumer panicked"),
|
||||
"expected constant panic prefix, got: {msg}"
|
||||
);
|
||||
// The original `panic!` payload (a `&'static str`) must
|
||||
// be preserved — without the downcast the message
|
||||
// would just be the prefix.
|
||||
assert!(
|
||||
msg.contains("synthetic test panic"),
|
||||
"expected original panic payload, got: {msg}"
|
||||
);
|
||||
}
|
||||
other => panic!("expected Err(IoError), got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Never-completing sink — `apply` blocks until cancelled. Signals
|
||||
/// `started` once it has consumed its first item so the test
|
||||
/// driver knows the consumer thread is wedged in `apply` (and
|
||||
/// will no longer drain the channel). Used to drive the
|
||||
/// halt/timeout paths of `send_with_halt` and `finish_with_halt`
|
||||
/// without depending on real I/O.
|
||||
struct NeverDrainsSink {
|
||||
cancel: Arc<std::sync::atomic::AtomicBool>,
|
||||
started: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
impl Sink<u64> for NeverDrainsSink {
|
||||
type Output = ();
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
self.started.store(true, Ordering::SeqCst);
|
||||
while !self.cancel.load(Ordering::SeqCst) {
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(self) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Spin until `started` flips or `bail` elapses. Used by the
|
||||
/// send_with_halt tests to synchronise with the consumer thread
|
||||
/// before exercising the bounded-send timeout path.
|
||||
fn wait_for_started(started: &Arc<std::sync::atomic::AtomicBool>, bail: Duration) {
|
||||
let end = Instant::now() + bail;
|
||||
while !started.load(Ordering::SeqCst) {
|
||||
assert!(Instant::now() < end, "consumer never started apply()");
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_with_halt_returns_item_on_deadline() {
|
||||
// depth=1 + consumer wedged in apply on the first item, AND
|
||||
// the channel buffer already loaded with a second item, means
|
||||
// any further `try_send` sees Full; with a 200 ms deadline and
|
||||
// no halt fired, send_with_halt must return `Err(item)` within
|
||||
// roughly the deadline. Synchronising on `started` ensures the
|
||||
// consumer has actually started its wedged apply BEFORE we
|
||||
// load the channel-buffer slot — without that, the consumer
|
||||
// could still drain in a race window.
|
||||
let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipe = Pipeline::spawn(
|
||||
1,
|
||||
NeverDrainsSink {
|
||||
cancel: cancel.clone(),
|
||||
started: started.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
// First send: consumer recv()s it and wedges in apply.
|
||||
pipe.send(0u64).expect("first send hands off to consumer");
|
||||
wait_for_started(&started, Duration::from_secs(2));
|
||||
// Second send: lands in the depth=1 buffer slot, consumer
|
||||
// can't pick it up because it's wedged in apply. Channel now
|
||||
// full from the producer's perspective.
|
||||
pipe.send(1u64).expect("second send fills the buffer");
|
||||
|
||||
let halt = crate::halt::Halt::new();
|
||||
let start = Instant::now();
|
||||
let res = pipe.send_with_halt(99u64, &halt, Duration::from_millis(200));
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Release the leaked consumer so the test process winds down.
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
let _ = pipe.finish();
|
||||
|
||||
assert!(matches!(res, Err(99)), "expected item returned on deadline");
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(150),
|
||||
"deadline returned too early: {elapsed:?}"
|
||||
);
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"deadline blew past tolerance: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_with_halt_returns_item_on_halt() {
|
||||
// Same setup, but the halt fires before the deadline elapses.
|
||||
// The send loop must observe the halt within ~50 ms (the
|
||||
// SEND_POLL_INTERVAL) and return the item.
|
||||
let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipe = Pipeline::spawn(
|
||||
1,
|
||||
NeverDrainsSink {
|
||||
cancel: cancel.clone(),
|
||||
started: started.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
pipe.send(0u64).expect("first send hands off to consumer");
|
||||
wait_for_started(&started, Duration::from_secs(2));
|
||||
pipe.send(1u64).expect("second send fills the buffer");
|
||||
|
||||
let halt = crate::halt::Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
halt2.cancel();
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
let res = pipe.send_with_halt(7u64, &halt, Duration::from_secs(10));
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
let _ = pipe.finish();
|
||||
|
||||
assert!(matches!(res, Err(7)), "expected item returned on halt");
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"halt observation took too long: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_with_halt_returns_halted_when_consumer_wedged() {
|
||||
// Consumer wedges on the first apply; halt fires; finish
|
||||
// returns the documented "pipeline join halted" error rather
|
||||
// than blocking forever.
|
||||
let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipe = Pipeline::spawn(
|
||||
DEFAULT_PIPELINE_DEPTH,
|
||||
NeverDrainsSink {
|
||||
cancel: cancel.clone(),
|
||||
started: started.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
pipe.send(0u64).expect("seed item the consumer wedges on");
|
||||
wait_for_started(&started, Duration::from_secs(2));
|
||||
|
||||
let halt = crate::halt::Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(400));
|
||||
halt2.cancel();
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
let res = pipe.finish_with_halt(Some(&halt));
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Release the leaked consumer so the test process exits cleanly.
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
|
||||
match res {
|
||||
Err(Error::IoError { source }) => {
|
||||
assert!(
|
||||
source.to_string().contains("pipeline join halted"),
|
||||
"expected halt-prefix error, got: {source}"
|
||||
);
|
||||
}
|
||||
other => panic!("expected Err(IoError) halted, got {other:?}"),
|
||||
}
|
||||
// Bailed out within ~1 second of the halt firing (worst case
|
||||
// one POLL_INTERVAL = 250 ms of slack).
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"halt observation took too long: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_with_halt_happy_path_returns_output() {
|
||||
// No halt token, sink completes normally — finish_with_halt
|
||||
// must return the same Output that `finish` would.
|
||||
let pipe = Pipeline::spawn(DEFAULT_PIPELINE_DEPTH, SumSink { total: 0 })
|
||||
.expect("spawn should succeed");
|
||||
for i in 0..10u64 {
|
||||
pipe.send(i).expect("send should succeed");
|
||||
}
|
||||
let total = pipe
|
||||
.finish_with_halt(None)
|
||||
.expect("happy-path finish_with_halt should succeed");
|
||||
assert_eq!(total, (0..10u64).sum::<u64>());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
//! Shared macOS `fcntl(F_PREALLOCATE)` definitions.
|
||||
//!
|
||||
//! The `libc` crate doesn't expose these symbols across all macOS SDK
|
||||
//! versions, so we define them locally with values from
|
||||
//! `/usr/include/sys/fcntl.h`. Two call sites (
|
||||
//! [`crate::io::writeback_file`] and [`crate::io::sink::preallocate`])
|
||||
//! need the same constants and `fstore_t` layout — keeping a single
|
||||
//! source of truth here prevents the two copies from drifting.
|
||||
//!
|
||||
//! Module-level cfg gate lives in the parent (`io/mod.rs`); this file
|
||||
//! is only compiled on macOS, so no inner `#![cfg]` is needed.
|
||||
|
||||
/// `fcntl(F_PREALLOCATE)` command number from `sys/fcntl.h`.
|
||||
pub(crate) const F_PREALLOCATE: libc::c_int = 42;
|
||||
|
||||
/// Anchor preallocation at the current physical EOF.
|
||||
pub(crate) const F_PEOFPOSMODE: libc::c_int = 3;
|
||||
|
||||
/// Prefer a contiguous allocation. Try this first; on `EINVAL` (no
|
||||
/// contiguous run of that size), fall back to `F_ALLOCATEALL`.
|
||||
pub(crate) const F_ALLOCATECONTIG: libc::c_uint = 0x0000_0002;
|
||||
|
||||
/// Allow non-contiguous allocation. Stronger guarantee than just
|
||||
/// asking for `F_ALLOCATECONTIG` because the kernel will piece
|
||||
/// together fragments rather than failing.
|
||||
pub(crate) const F_ALLOCATEALL: libc::c_uint = 0x0000_0004;
|
||||
|
||||
/// `fstore_t` from `sys/fcntl.h`. `repr(C)` because we hand it to
|
||||
/// `fcntl(F_PREALLOCATE)` which writes through the pointer.
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct Fstore {
|
||||
pub fst_flags: libc::c_uint,
|
||||
pub fst_posmode: libc::c_int,
|
||||
pub fst_offset: libc::off_t,
|
||||
pub fst_length: libc::off_t,
|
||||
pub fst_bytesalloc: libc::off_t,
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
//! `LocalFileSink` — `BufWriter<File>` for the common local-disk case.
|
||||
//!
|
||||
//! Buffering: 4 MiB internal `BufWriter`. Sized to coalesce the small
|
||||
//! per-PES writes that come out of the muxer into kernel-page-aligned
|
||||
//! flushes without making the buffer big enough to matter for memory
|
||||
//! pressure on a single concurrent rip.
|
||||
//!
|
||||
//! `Seek` flushes the underlying `BufWriter` first; otherwise a seek
|
||||
//! could leapfrog buffered data and silently corrupt the file. This is
|
||||
//! the same shape `BufWriter` itself uses when it impls `Seek` in
|
||||
//! stdlib, and is necessary for MKV's seek-back operations (cluster
|
||||
//! size patch, Cues index, segment header backpatch) to land on the
|
||||
//! right offset.
|
||||
//!
|
||||
//! `RandomAccessSink` is satisfied via the blanket impl in
|
||||
//! [`super::mod`]; no explicit impl needed here.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, BufWriter, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use super::preallocate;
|
||||
|
||||
const BUFFER_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Random-access write sink for local disks.
|
||||
///
|
||||
/// Wraps a `BufWriter<File>` with a 4 MiB internal buffer and forwards
|
||||
/// `Write`/`Seek` so any call site that previously held a `File` or
|
||||
/// `WritebackFile` can drop this in. `finish()` flushes the buffer and
|
||||
/// runs `sync_all` on the underlying file so the caller can drop it
|
||||
/// without losing data.
|
||||
///
|
||||
/// Construction always opens the file `create + truncate + read +
|
||||
/// write`. `read` is enabled so the same handle can be reused for a
|
||||
/// verification re-read after the mux (the existing
|
||||
/// `FileSectorSink::create` pattern). On Linux, [`with_size_hint`]
|
||||
/// additionally calls `fallocate(FALLOC_FL_KEEP_SIZE)` to pre-reserve
|
||||
/// extents.
|
||||
///
|
||||
/// [`with_size_hint`]: Self::with_size_hint
|
||||
pub struct LocalFileSink {
|
||||
inner: BufWriter<File>,
|
||||
}
|
||||
|
||||
impl LocalFileSink {
|
||||
/// Open `path` for writing, truncating any existing contents.
|
||||
pub fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
Ok(Self {
|
||||
inner: BufWriter::with_capacity(BUFFER_BYTES, file),
|
||||
})
|
||||
}
|
||||
|
||||
/// Like [`Self::create`] but additionally calls the per-OS
|
||||
/// preallocate path with `size_bytes`. On Linux this is
|
||||
/// `fallocate(FALLOC_FL_KEEP_SIZE)` so the on-disk extents are
|
||||
/// reserved up front (reducing fragmentation for big sequential
|
||||
/// muxer output); on other OSes it is a no-op today. Failures
|
||||
/// from the preallocate call are non-fatal — the file is still
|
||||
/// returned, just without the size reservation.
|
||||
pub fn with_size_hint(path: &Path, size_bytes: u64) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
preallocate::preallocate(&file, size_bytes);
|
||||
Ok(Self {
|
||||
inner: BufWriter::with_capacity(BUFFER_BYTES, file),
|
||||
})
|
||||
}
|
||||
|
||||
/// Drain the internal buffer and `fsync` the underlying file.
|
||||
/// Idempotent with `Drop` (the `BufWriter` also flushes on drop;
|
||||
/// this call additionally surfaces fsync errors to the caller).
|
||||
#[allow(dead_code)] // exposed for parity with WritebackFile::sync_all
|
||||
pub fn sync_all(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()?;
|
||||
self.inner.get_ref().sync_all()
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for LocalFileSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.inner.write(buf)
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.inner.write_all(buf)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for LocalFileSink {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
// Flush before seeking so buffered bytes land at the offset
|
||||
// they were written for, not the new one.
|
||||
self.inner.flush()?;
|
||||
self.inner.get_mut().seek(from)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
|
||||
#[test]
|
||||
fn write_seek_roundtrip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("rt.bin");
|
||||
let mut s = LocalFileSink::create(&p).unwrap();
|
||||
s.write_all(b"AAAA").unwrap();
|
||||
s.write_all(b"BBBB").unwrap();
|
||||
// Seek back over the second word and overwrite.
|
||||
s.seek(SeekFrom::Start(4)).unwrap();
|
||||
s.write_all(b"CCCC").unwrap();
|
||||
s.sync_all().unwrap();
|
||||
drop(s);
|
||||
|
||||
let mut f = File::open(&p).unwrap();
|
||||
let mut got = Vec::new();
|
||||
f.read_to_end(&mut got).unwrap();
|
||||
assert_eq!(&got[..], b"AAAACCCC");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_flushes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("drop.bin");
|
||||
{
|
||||
let mut s = LocalFileSink::create(&p).unwrap();
|
||||
s.write_all(b"buffered").unwrap();
|
||||
// No explicit flush / sync_all — BufWriter drop runs the
|
||||
// flush and the file should land on disk.
|
||||
}
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..], b"buffered");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_size_hint_creates_writable_file() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("sz.bin");
|
||||
let mut s = LocalFileSink::with_size_hint(&p, 64 * 1024).unwrap();
|
||||
s.write_all(b"hint-ok").unwrap();
|
||||
s.sync_all().unwrap();
|
||||
drop(s);
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..], b"hint-ok");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
//! Output-sink trait split for the buffering architecture.
|
||||
//!
|
||||
//! Two traits, one for each capability axis of an output destination:
|
||||
//!
|
||||
//! - [`SequentialSink`] — anything you can `Write` to in order. Sockets,
|
||||
//! pipes, append-only stores, plain files. Containers that don't need
|
||||
//! seek (M2TS, fMP4, HEVC elementary) target this.
|
||||
//! - [`RandomAccessSink`] — everything `SequentialSink` plus a working
|
||||
//! `Seek`. Local files, NFS files, anything with random-write
|
||||
//! semantics. Containers that need backpatch (MKV cluster sizes, Cues
|
||||
//! index, MP4 moov-at-end) target this.
|
||||
//!
|
||||
//! `RandomAccessSink: SequentialSink` — every random-access sink is
|
||||
//! also a valid sequential sink. The muxer is generic over which it
|
||||
//! requires (`MkvMux<S: RandomAccessSink>`, `M2tsMux<S: SequentialSink>`)
|
||||
//! so an attempt to mux MKV to a network socket is a compile error.
|
||||
//!
|
||||
//! Buffering policy belongs to the concrete sink, not to a wrapper at
|
||||
//! the call site. `LocalFileSink` wraps a `BufWriter<File>` with a
|
||||
//! 4 MiB buffer for the common local-disk case; `WritebackFile`
|
||||
//! (separate module) wraps a `File` with the adaptive-chunk
|
||||
//! `sync_file_range` machinery for the Linux+NFS case.
|
||||
|
||||
use std::io::{Seek, Write};
|
||||
|
||||
mod local_file;
|
||||
mod preallocate;
|
||||
mod socket;
|
||||
|
||||
pub use local_file::LocalFileSink;
|
||||
pub use socket::{SocketSink, UdpSocketSink};
|
||||
|
||||
/// Sequential-only write destination. Sockets, pipes, append-only
|
||||
/// stores. No seek. Implementations own their write buffering — the
|
||||
/// trait does not impose or hide any buffering of its own.
|
||||
///
|
||||
/// `finish` drains any internal buffering and signals end-of-stream to
|
||||
/// the underlying transport (close-write on a socket, flush on a
|
||||
/// buffered writer, etc.). The default impl is a no-op; concrete
|
||||
/// implementations that need explicit shutdown can override it but the
|
||||
/// blanket impl below keeps it optional for adapter types like
|
||||
/// `&mut File`.
|
||||
pub trait SequentialSink: Write + Send {
|
||||
fn finish(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Random-access write destination. Local files, NFS files, anything
|
||||
/// with a working `Seek`. Inherits the `SequentialSink` contract — a
|
||||
/// random-access sink is always usable as a sequential sink.
|
||||
pub trait RandomAccessSink: SequentialSink + Seek {}
|
||||
|
||||
// Blanket impls so any `Write + Send` type acts as a `SequentialSink`
|
||||
// (with default `finish`), and any sink that also impls `Seek` is
|
||||
// automatically a `RandomAccessSink`. Keeps call-site ergonomics simple
|
||||
// — `&mut File`, `LocalFileSink`, `WritebackFile`, `BufWriter<File>`,
|
||||
// and `Cursor<Vec<u8>>` all satisfy the right trait without per-type
|
||||
// boilerplate.
|
||||
impl<T: Write + Send + ?Sized> SequentialSink for T {}
|
||||
impl<T: SequentialSink + Seek + ?Sized> RandomAccessSink for T {}
|
||||
|
||||
/// Pick the right `RandomAccessSink` impl for `dest` based on its
|
||||
/// filesystem type.
|
||||
///
|
||||
/// - Linux + NFS path → `WritebackFile` with its adaptive-chunk
|
||||
/// sync_file_range machinery and (when supported) `fallocate` size
|
||||
/// hint.
|
||||
/// - everything else → [`LocalFileSink`] over `BufWriter<File>`. On
|
||||
/// non-Linux there is no `WritebackFile` machinery to opt into, and
|
||||
/// on local Linux the kernel's default writeback policy is already
|
||||
/// fine.
|
||||
///
|
||||
/// `size_hint`, when present, is forwarded to the per-OS preallocate
|
||||
/// path (`fallocate(KEEP_SIZE)` on Linux, `F_PREALLOCATE` on macOS when
|
||||
/// implemented, no-op elsewhere).
|
||||
///
|
||||
/// Returns a boxed trait object so the call site (mux construction)
|
||||
/// stays agnostic of which concrete sink got picked.
|
||||
#[allow(dead_code)] // wiring to mux::resolve is a follow-up commit
|
||||
pub fn open_for_mkv(
|
||||
dest: &std::path::Path,
|
||||
size_hint: Option<u64>,
|
||||
) -> std::io::Result<Box<dyn RandomAccessSink>> {
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
use crate::platform::fs_type::detect;
|
||||
#[cfg(target_os = "linux")]
|
||||
use crate::platform::fs_type::{FsType, detect};
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
if detect(dest) == FsType::Nfs {
|
||||
let wf = match size_hint {
|
||||
Some(n) => crate::io::WritebackFile::create_with_size_hint(dest, n)?,
|
||||
None => crate::io::WritebackFile::create(dest)?,
|
||||
};
|
||||
return Ok(Box::new(wf));
|
||||
}
|
||||
}
|
||||
// Silence the unused-binding warning on non-Linux where the only
|
||||
// branch above is cfg-gated out.
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
let _ = detect(dest);
|
||||
}
|
||||
|
||||
let sink = match size_hint {
|
||||
Some(n) => LocalFileSink::with_size_hint(dest, n)?,
|
||||
None => LocalFileSink::create(dest)?,
|
||||
};
|
||||
Ok(Box::new(sink))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs::File;
|
||||
|
||||
// Type-level assertion: the blanket impls cover the shapes we care
|
||||
// about. These functions never run; they just have to type-check.
|
||||
fn _assert_file_is_sequential(_: &mut dyn SequentialSink) {}
|
||||
fn _assert_file_is_random_access(_: &mut dyn RandomAccessSink) {}
|
||||
|
||||
#[test]
|
||||
fn blanket_impls_cover_file_and_localfilesink() {
|
||||
// `File` directly via blanket impls.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut f = File::create(dir.path().join("a.bin")).unwrap();
|
||||
_assert_file_is_sequential(&mut f);
|
||||
_assert_file_is_random_access(&mut f);
|
||||
|
||||
// `LocalFileSink` ditto.
|
||||
let mut s = LocalFileSink::create(&dir.path().join("b.bin")).unwrap();
|
||||
_assert_file_is_sequential(&mut s);
|
||||
_assert_file_is_random_access(&mut s);
|
||||
|
||||
// `WritebackFile` — confirms the Phase 1 type still satisfies
|
||||
// the trait via the blanket impl without needing an explicit
|
||||
// `impl RandomAccessSink for WritebackFile {}`.
|
||||
let mut wf = crate::io::WritebackFile::create(&dir.path().join("c.bin")).unwrap();
|
||||
_assert_file_is_sequential(&mut wf);
|
||||
_assert_file_is_random_access(&mut wf);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_for_mkv_returns_a_random_access_sink() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("c.bin");
|
||||
let mut sink = open_for_mkv(&p, Some(64 * 1024)).unwrap();
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
sink.write_all(b"hello").unwrap();
|
||||
sink.seek(SeekFrom::Start(0)).unwrap();
|
||||
sink.finish().unwrap();
|
||||
drop(sink);
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..5], b"hello");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
//! Linux `fallocate(FALLOC_FL_KEEP_SIZE)` preallocation.
|
||||
//!
|
||||
//! `KEEP_SIZE` reserves extents without changing the apparent file
|
||||
//! length, which matches the muxer's expectation that writes still grow
|
||||
//! the file naturally.
|
||||
|
||||
use std::fs::File;
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
pub(super) fn preallocate_impl(file: &File, size_bytes: u64) {
|
||||
let fd = file.as_raw_fd();
|
||||
// FALLOC_FL_KEEP_SIZE = 0x01.
|
||||
let rc = unsafe { libc::fallocate(fd, libc::FALLOC_FL_KEEP_SIZE, 0, size_bytes as i64) };
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink fallocate size_hint={size_bytes} rc={rc} ok={}",
|
||||
rc == 0
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
//! macOS `F_PREALLOCATE` extent reservation.
|
||||
//!
|
||||
//! `fcntl(F_PREALLOCATE)` with `F_ALLOCATECONTIG` first (try for a
|
||||
//! contiguous run) and fall back to `F_ALLOCATEALL` (non-contig OK).
|
||||
//! Reported file size is unchanged — the muxer's writes still grow it.
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
use crate::io::platform_macos::{
|
||||
F_ALLOCATEALL, F_ALLOCATECONTIG, F_PEOFPOSMODE, F_PREALLOCATE, Fstore,
|
||||
};
|
||||
|
||||
pub(super) fn preallocate_impl(file: &File, size_bytes: u64) {
|
||||
let fd = file.as_raw_fd();
|
||||
let mut store = Fstore {
|
||||
fst_flags: F_ALLOCATECONTIG,
|
||||
fst_posmode: F_PEOFPOSMODE,
|
||||
fst_offset: 0,
|
||||
fst_length: size_bytes as libc::off_t,
|
||||
fst_bytesalloc: 0,
|
||||
};
|
||||
let mut rc = unsafe { libc::fcntl(fd, F_PREALLOCATE, &mut store as *mut Fstore) };
|
||||
if rc == -1 {
|
||||
// Fall back to non-contiguous.
|
||||
store.fst_flags = F_ALLOCATEALL;
|
||||
rc = unsafe { libc::fcntl(fd, F_PREALLOCATE, &mut store as *mut Fstore) };
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink F_PREALLOCATE size_hint={size_bytes} rc={rc} bytesalloc={}",
|
||||
store.fst_bytesalloc
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! Per-OS extent preallocation. Best-effort; failures are logged at
|
||||
//! debug and otherwise swallowed because the file is still usable
|
||||
//! without the size reservation — only large-file fragmentation gets
|
||||
//! marginally worse.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
mod other;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux::preallocate_impl;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos::preallocate_impl;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
use other::preallocate_impl;
|
||||
|
||||
/// Reserve `size_bytes` of disk space for `file`'s on-disk extents.
|
||||
/// Reported file size is unchanged — writes still grow the file
|
||||
/// naturally; only the allocator's extent map is primed.
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
preallocate_impl(file, size_bytes);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
//! Fallback preallocate impl. No-op.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn preallocate_impl(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink preallocate size_hint={size_bytes} skipped (no platform impl)"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
//! TCP / UDP socket sinks (sequential-only).
|
||||
//!
|
||||
//! [`SocketSink`] wraps a `TcpStream` in a 1 MiB `BufWriter`. Constructor
|
||||
//! tunes `SO_SNDBUF` to a caller hint when provided. `finish()` flushes
|
||||
//! the buffer then `shutdown(Write)`s the socket so the peer sees clean
|
||||
//! end-of-stream.
|
||||
//!
|
||||
//! [`UdpSocketSink`] wraps a connected `UdpSocket`. Each `write` call
|
||||
//! emits exactly one datagram — the caller is responsible for packetizing
|
||||
//! to a reasonable MTU (188 × 7 = 1316 bytes for MPEG-TS-over-UDP is the
|
||||
//! conventional choice). `finish()` is a no-op; UDP has no end-of-stream
|
||||
//! marker.
|
||||
//!
|
||||
//! Both types satisfy [`SequentialSink`] via the blanket impl in
|
||||
//! `super::mod`. Neither implements `Seek`, so neither satisfies
|
||||
//! [`RandomAccessSink`] — using one with `MkvMux` is a compile error,
|
||||
//! which is the design intent.
|
||||
//!
|
||||
//! [`SequentialSink`]: super::SequentialSink
|
||||
//! [`RandomAccessSink`]: super::RandomAccessSink
|
||||
|
||||
use std::io::{self, BufWriter, Write};
|
||||
use std::net::{Shutdown, TcpStream, ToSocketAddrs, UdpSocket};
|
||||
|
||||
/// `BufWriter` capacity for [`SocketSink`]. 1 MiB matches the typical
|
||||
/// kernel send-buffer ceiling and keeps small-write amplification from
|
||||
/// containers (TS = 188-byte packets, fMP4 fragment headers = ~100 bytes)
|
||||
/// from translating into syscall storms.
|
||||
const TCP_BUF_CAPACITY: usize = 1024 * 1024;
|
||||
|
||||
/// Sequential-only sink over a TCP connection.
|
||||
///
|
||||
/// Wraps a `BufWriter<TcpStream>`; the inner `TcpStream` is kept as a
|
||||
/// clone so [`finish`](Self::finish) can call `shutdown(Write)` after
|
||||
/// flushing the buffer (the buffered writer doesn't expose the socket
|
||||
/// directly).
|
||||
pub struct SocketSink {
|
||||
/// Buffered write half. All payload bytes go through this.
|
||||
buf: BufWriter<TcpStream>,
|
||||
/// Shutdown handle — clone of the socket inside `buf`. Used only by
|
||||
/// `finish()` for `shutdown(Write)`; never read or written through.
|
||||
shutdown_handle: TcpStream,
|
||||
}
|
||||
|
||||
impl SocketSink {
|
||||
/// Open a TCP connection to `addr` and wrap it for sequential
|
||||
/// writing. `sndbuf_bytes`, when present, is forwarded to
|
||||
/// `setsockopt(SO_SNDBUF)` as a kernel hint — the OS may clamp it.
|
||||
///
|
||||
/// `addr` accepts anything `ToSocketAddrs` does: `"192.0.2.1:1234"`,
|
||||
/// `("host", 1234)`, a `SocketAddr`, etc.
|
||||
pub fn connect<A: ToSocketAddrs>(addr: A, sndbuf_bytes: Option<usize>) -> io::Result<Self> {
|
||||
let stream = TcpStream::connect(addr)?;
|
||||
// `set_nodelay(true)` keeps small writes (TS packet trains, fMP4
|
||||
// moof headers) from sitting in Nagle's algorithm until the buffer
|
||||
// fills. The BufWriter already absorbs syscall overhead; Nagle
|
||||
// would just add latency without coalescing more.
|
||||
stream.set_nodelay(true)?;
|
||||
if let Some(n) = sndbuf_bytes {
|
||||
set_send_buffer(&stream, n)?;
|
||||
}
|
||||
let shutdown_handle = stream.try_clone()?;
|
||||
Ok(Self {
|
||||
buf: BufWriter::with_capacity(TCP_BUF_CAPACITY, stream),
|
||||
shutdown_handle,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for SocketSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.buf.write(buf)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.buf.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl SocketSink {
|
||||
/// Drain the BufWriter and `shutdown(Write)` the underlying socket
|
||||
/// so the peer sees a clean EOF.
|
||||
///
|
||||
/// Note: [`SequentialSink::finish`](super::SequentialSink::finish)'s
|
||||
/// blanket-impl default is a no-op. Trait-object call sites that
|
||||
/// need socket shutdown should call this inherent method directly
|
||||
/// before dropping the sink, or hold the concrete `SocketSink` type
|
||||
/// (typical pattern: each muxer's `finish()` calls the appropriate
|
||||
/// inherent close method on its captured concrete sink).
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
self.buf.flush()?;
|
||||
// `shutdown(Write)` signals clean EOF to the peer. Errors here
|
||||
// are non-fatal — the connection may have already been torn down
|
||||
// by the peer — but we surface them so callers can log.
|
||||
self.shutdown_handle.shutdown(Shutdown::Write)
|
||||
}
|
||||
}
|
||||
|
||||
/// Sequential-only sink over a connected UDP socket.
|
||||
///
|
||||
/// Each [`write`](Write::write) call sends exactly one datagram. The
|
||||
/// caller is responsible for splitting payload at packet boundaries —
|
||||
/// for MPEG-TS this means 7 × 188 = 1316 bytes per datagram, the
|
||||
/// industry standard for MPEG-TS-over-UDP. No buffering happens here;
|
||||
/// adding it would silently merge datagrams.
|
||||
///
|
||||
/// `finish()` is a no-op: UDP has no end-of-stream marker. Closing the
|
||||
/// socket happens on drop.
|
||||
pub struct UdpSocketSink {
|
||||
socket: UdpSocket,
|
||||
}
|
||||
|
||||
impl UdpSocketSink {
|
||||
/// Bind a local UDP socket to an ephemeral port and `connect` it to
|
||||
/// `peer`. `connect` doesn't open a connection — it just fixes the
|
||||
/// peer address so subsequent `send` calls don't need to repeat it,
|
||||
/// and so receive-side filtering rejects packets from other sources.
|
||||
///
|
||||
/// `sndbuf_bytes`, when present, is a hint to `SO_SNDBUF`.
|
||||
pub fn connect<A: ToSocketAddrs>(peer: A, sndbuf_bytes: Option<usize>) -> io::Result<Self> {
|
||||
// Bind to all-zeros / any port. The kernel picks an ephemeral
|
||||
// source port and the source IP at first send.
|
||||
let socket = UdpSocket::bind("0.0.0.0:0")?;
|
||||
socket.connect(peer)?;
|
||||
if let Some(n) = sndbuf_bytes {
|
||||
set_udp_send_buffer(&socket, n)?;
|
||||
}
|
||||
Ok(Self { socket })
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for UdpSocketSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
// `send` writes the entire datagram or fails — no partial sends
|
||||
// for UDP. Match `Write::write`'s contract by reporting bytes
|
||||
// accepted.
|
||||
self.socket.send(buf)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl UdpSocketSink {
|
||||
/// No-op — UDP has no end-of-stream marker. Provided for parity
|
||||
/// with [`SocketSink::finish`] so call sites can treat them uniformly.
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ── Platform `SO_SNDBUF` tuning ────────────────────────────────────────────
|
||||
//
|
||||
// std's `TcpStream` / `UdpSocket` don't expose `SO_SNDBUF`. We drop to
|
||||
// libc on Linux + macOS (the libc-dep targets in Cargo.toml). On other
|
||||
// targets the hint is silently ignored — the socket still works, the
|
||||
// kernel just picks its own send-buffer size.
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn set_send_buffer(stream: &TcpStream, bytes: usize) -> io::Result<()> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
setsockopt_sndbuf(stream.as_raw_fd(), bytes)
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn set_udp_send_buffer(socket: &UdpSocket, bytes: usize) -> io::Result<()> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
setsockopt_sndbuf(socket.as_raw_fd(), bytes)
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
fn set_send_buffer(_stream: &TcpStream, _bytes: usize) -> io::Result<()> {
|
||||
// Non-Linux-non-macOS targets aren't in Cargo.toml's libc dep list;
|
||||
// silently ignore the hint rather than failing the connect. Callers
|
||||
// can detect via the lack of an explicit "sndbuf applied" signal
|
||||
// (not provided, intentionally — this is a hint, not a guarantee).
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
fn set_udp_send_buffer(_socket: &UdpSocket, _bytes: usize) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn setsockopt_sndbuf(fd: std::os::unix::io::RawFd, bytes: usize) -> io::Result<()> {
|
||||
// Clamp into c_int range; SO_SNDBUF takes an `int` argument.
|
||||
let want: libc::c_int = bytes.try_into().unwrap_or(libc::c_int::MAX);
|
||||
let ret = unsafe {
|
||||
libc::setsockopt(
|
||||
fd,
|
||||
libc::SOL_SOCKET,
|
||||
libc::SO_SNDBUF,
|
||||
&want as *const _ as *const libc::c_void,
|
||||
std::mem::size_of::<libc::c_int>() as libc::socklen_t,
|
||||
)
|
||||
};
|
||||
if ret != 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
use std::net::{TcpListener, UdpSocket};
|
||||
use std::thread;
|
||||
|
||||
/// Bind a listener, accept on a thread, return (listener_addr,
|
||||
/// accepted-bytes future via JoinHandle).
|
||||
#[test]
|
||||
fn socket_sink_round_trips_bytes() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let accept = thread::spawn(move || {
|
||||
let (mut sock, _) = listener.accept().unwrap();
|
||||
let mut buf = Vec::new();
|
||||
sock.read_to_end(&mut buf).unwrap();
|
||||
buf
|
||||
});
|
||||
|
||||
let mut sink = SocketSink::connect(addr, Some(256 * 1024)).unwrap();
|
||||
// Write enough to overflow the BufWriter at least once, then a
|
||||
// tail that lives in the buffer until `finish` flushes.
|
||||
let big: Vec<u8> = (0..(2 * TCP_BUF_CAPACITY))
|
||||
.map(|i| (i & 0xff) as u8)
|
||||
.collect();
|
||||
sink.write_all(&big).unwrap();
|
||||
sink.write_all(b"tail\n").unwrap();
|
||||
sink.finish().unwrap();
|
||||
drop(sink);
|
||||
|
||||
let received = accept.join().unwrap();
|
||||
assert_eq!(received.len(), big.len() + 5);
|
||||
assert_eq!(&received[..big.len()], &big[..]);
|
||||
assert_eq!(&received[big.len()..], b"tail\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn socket_sink_is_sequential_only() {
|
||||
// Compile-time assertion via dyn — if this ever started
|
||||
// satisfying `RandomAccessSink`, the trait split would be broken.
|
||||
fn _assert_seq(_: &mut dyn super::super::SequentialSink) {}
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let _accept = thread::spawn(move || {
|
||||
let _ = listener.accept();
|
||||
});
|
||||
let mut sink = SocketSink::connect(addr, None).unwrap();
|
||||
_assert_seq(&mut sink);
|
||||
// The negative is harder to assert directly (no `is_not<T>`),
|
||||
// but `SocketSink` does not impl `Seek`, so it can't unify with
|
||||
// `RandomAccessSink`'s super-bound. The Phase 2 blanket impl
|
||||
// `impl<T: SequentialSink + Seek> RandomAccessSink for T {}` thus
|
||||
// excludes it by construction.
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn udp_socket_sink_delivers_datagrams() {
|
||||
let receiver = UdpSocket::bind("127.0.0.1:0").unwrap();
|
||||
receiver
|
||||
.set_read_timeout(Some(std::time::Duration::from_secs(2)))
|
||||
.unwrap();
|
||||
let addr = receiver.local_addr().unwrap();
|
||||
let mut sink = UdpSocketSink::connect(addr, Some(128 * 1024)).unwrap();
|
||||
|
||||
sink.write_all(&[1, 2, 3, 4, 5]).unwrap();
|
||||
sink.write_all(&[9, 9, 9]).unwrap();
|
||||
sink.finish().unwrap();
|
||||
|
||||
let mut buf = [0u8; 64];
|
||||
let n1 = receiver.recv(&mut buf).unwrap();
|
||||
assert_eq!(&buf[..n1], &[1, 2, 3, 4, 5]);
|
||||
let n2 = receiver.recv(&mut buf).unwrap();
|
||||
assert_eq!(&buf[..n2], &[9, 9, 9]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
//! Per-platform writeback pipeline. On Linux, drains dirty pages
|
||||
//! continuously at chunk granularity to keep the kernel's writeback
|
||||
//! queue bounded. On macOS and Windows, a no-op stub.
|
||||
//!
|
||||
//! The platform decision lives entirely in this file (the cfg-gated
|
||||
//! `pub use` below). Callers — and `DiskWriter` itself — are
|
||||
//! platform-independent.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
mod noop;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) use linux::WritebackPipeline;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub(super) use noop::WritebackPipeline;
|
||||
@@ -0,0 +1,459 @@
|
||||
//! Linux writeback pipeline using `sync_file_range` + `posix_fadvise`.
|
||||
//!
|
||||
//! Pathology this fixes: the kernel's default `vm.dirty_ratio` (~20 %
|
||||
//! of RAM) lets dirty pages accumulate to hundreds of MB during a
|
||||
//! big sequential write, then bursts a flush at 99 % disk utilisation.
|
||||
//! While the burst runs, app writes block on the writeback queue —
|
||||
//! observed empirically as instantaneous speed dropping from ~15 MB/s
|
||||
//! to ~1 MB/s every ~30 s during a Pass 1 sweep.
|
||||
//!
|
||||
//! Strategy: every `chunk_bytes` of new sequential output, kick async
|
||||
//! writeback (`SYNC_FILE_RANGE_WRITE`) on the just-completed chunk and
|
||||
//! finalise the *previous* chunk via `WAIT_AFTER` + `posix_fadvise
|
||||
//! (DONTNEED)`. By the time we finalise, that previous chunk has had
|
||||
//! a full chunk's worth of work to flush — the wait is near-instant.
|
||||
//! Dirty cache stays bounded at ~2 × `chunk_bytes` and writes drain
|
||||
//! continuously instead of in bursts.
|
||||
//!
|
||||
//! The chunk size is adaptive: we measure the elapsed time of the
|
||||
//! `WAIT_AFTER` call over a rolling window of the last 16 chunks and
|
||||
//! resize the chunk based on the p95. Slow storage (NFS, network
|
||||
//! shares, HDD) sees larger chunks to amortise per-chunk overhead;
|
||||
//! fast storage (NVMe) sees smaller chunks to keep cache pressure
|
||||
//! tight. Bounds: [4 MiB, 256 MiB].
|
||||
//!
|
||||
//! ## NFS escape hatch
|
||||
//!
|
||||
//! `sync_file_range(WAIT_AFTER)` on an NFS-mounted file can block
|
||||
//! indefinitely waiting for the server's commit ack. If the server
|
||||
//! never acks (network partition, server-side hang, slow commit), the
|
||||
//! syscall never returns and the consumer thread is stuck inside the
|
||||
//! kernel — `/api/stop` can't reach it because halt is cooperative.
|
||||
//!
|
||||
//! When `fstatfs` reports the file lives on an NFS mount
|
||||
//! (`f_type == NFS_SUPER_MAGIC`), the pipeline skips the WAIT_AFTER +
|
||||
//! `posix_fadvise(DONTNEED)` dance entirely. NFS clients have their
|
||||
//! own buffering and commit semantics that handle dirty-page bounds
|
||||
//! without us forcing the issue. The async `SYNC_FILE_RANGE_WRITE`
|
||||
//! kickoff still runs (non-blocking by spec) so writeback still gets
|
||||
//! a nudge.
|
||||
//!
|
||||
//! ## Defence in depth: WAIT_AFTER timeout
|
||||
//!
|
||||
//! Even on local storage, a degraded disk or odd filesystem driver
|
||||
//! could in principle wedge inside WAIT_AFTER. Each WAIT_AFTER call
|
||||
//! runs on a worker thread with a 30s recv_timeout on its result
|
||||
//! channel. On timeout we log a loud error, set a `degraded` flag,
|
||||
//! and from then on skip WAIT_AFTER + DONTNEED for the rest of the
|
||||
//! pipeline's life (same shape as the NFS path). The worker thread
|
||||
//! is intentionally leaked — it unwinds whenever the syscall
|
||||
//! eventually returns or the process exits. The mux continues; the
|
||||
//! original dirty-burst pathology re-emerges but the rip can still
|
||||
//! finish instead of freezing.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::{AsRawFd, RawFd};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const ADAPTIVE_WINDOW: usize = 16;
|
||||
const CHUNK_BYTES_MIN: u64 = 4 * 1024 * 1024;
|
||||
const CHUNK_BYTES_MAX: u64 = 256 * 1024 * 1024;
|
||||
const ADAPTIVE_GROW_MS: u64 = 200;
|
||||
const ADAPTIVE_SHRINK_MS: u64 = 20;
|
||||
/// Every N chunks, emit a `debug!` snapshot of the current chunk
|
||||
/// size so operators tailing the log can see where the autoscaler
|
||||
/// settled.
|
||||
const SIZE_LOG_INTERVAL: u64 = 32;
|
||||
/// Hard upper bound on a single `sync_file_range(WAIT_AFTER)` call.
|
||||
/// Beyond this we declare the pipeline degraded and stop calling
|
||||
/// WAIT_AFTER for the rest of its life.
|
||||
const WAIT_AFTER_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
pub(crate) struct WritebackPipeline {
|
||||
/// Aliases the wrapping `WritebackFile::file`. Only valid for the
|
||||
/// lifetime of that struct — moving the `File` independently
|
||||
/// would silently UAF this fd. The pipeline is a private field of
|
||||
/// `WritebackFile` and never exposed outside that wrapper, which
|
||||
/// is what keeps the alias sound.
|
||||
fd: RawFd,
|
||||
chunk_bytes: u64,
|
||||
last_flush_pos: u64,
|
||||
pending: Option<(u64, u64)>,
|
||||
/// Rolling window of recent `WAIT_AFTER` elapsed_ms measurements.
|
||||
wait_after_window: VecDeque<u64>,
|
||||
/// Count of chunks emitted (used to space out periodic
|
||||
/// `debug!` size snapshots).
|
||||
chunk_count: u64,
|
||||
/// True when the underlying file is on an NFS mount. NFS makes
|
||||
/// WAIT_AFTER unsafe (can block forever on missing server ack), so
|
||||
/// we skip it entirely and let the NFS client handle commit on
|
||||
/// close.
|
||||
is_nfs: bool,
|
||||
/// Set the first time WAIT_AFTER exceeds [`WAIT_AFTER_TIMEOUT`].
|
||||
/// Once set, behaviour matches the NFS path for the rest of the
|
||||
/// pipeline's life. Wrapped in `Arc` only because both this
|
||||
/// struct and the spawned worker thread (which itself doesn't
|
||||
/// touch the flag) share-via-fd patterns might one day need it;
|
||||
/// today it's effectively a single-owner cell — the `Arc` shape
|
||||
/// keeps the door open for moving the read side into a worker
|
||||
/// without re-plumbing types.
|
||||
degraded: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl WritebackPipeline {
|
||||
/// Construct a pipeline aliasing `file`'s file descriptor. The
|
||||
/// returned `WritebackPipeline` MUST be dropped before `file`
|
||||
/// itself, or kept inside the same struct that owns `file` — the
|
||||
/// alias is unchecked.
|
||||
pub(crate) fn new(file: &File, start_pos: u64, chunk_bytes: u64) -> Self {
|
||||
let fd = file.as_raw_fd();
|
||||
let is_nfs = detect_nfs(fd);
|
||||
tracing::info!(
|
||||
target: "mux",
|
||||
"WritebackPipeline fd={fd} is_nfs={is_nfs} chunk_bytes={chunk_bytes} strategy={}",
|
||||
if is_nfs { "nfs-skip-wait" } else { "wait+dontneed" }
|
||||
);
|
||||
Self {
|
||||
fd,
|
||||
chunk_bytes,
|
||||
last_flush_pos: start_pos,
|
||||
pending: None,
|
||||
wait_after_window: VecDeque::with_capacity(ADAPTIVE_WINDOW),
|
||||
chunk_count: 0,
|
||||
is_nfs,
|
||||
degraded: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
/// True if we should bypass the WAIT_AFTER + DONTNEED finalisation
|
||||
/// step. NFS always bypasses; local storage bypasses once the
|
||||
/// pipeline has flipped to degraded after a WAIT_AFTER timeout.
|
||||
#[inline]
|
||||
fn skip_wait(&self) -> bool {
|
||||
self.is_nfs || self.degraded.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Caller advanced the file position to `pos`. If a chunk boundary
|
||||
/// was crossed, kick async writeback for the just-completed chunk
|
||||
/// and finalise the previous one.
|
||||
pub(crate) fn note_progress(&mut self, pos: u64) {
|
||||
if pos < self.last_flush_pos.saturating_add(self.chunk_bytes) {
|
||||
return;
|
||||
}
|
||||
let chunk_off = self.last_flush_pos as i64;
|
||||
let chunk_len = (pos - self.last_flush_pos) as i64;
|
||||
let mut wait_ms: u64 = 0;
|
||||
let mut fadvise_ms: u64 = 0;
|
||||
// Async kickoff for the just-completed chunk runs on every
|
||||
// path (NFS, degraded, normal) — it's nominally non-blocking
|
||||
// by spec and gives the kernel an early hint that this range
|
||||
// is ready to flush.
|
||||
unsafe {
|
||||
libc::sync_file_range(self.fd, chunk_off, chunk_len, libc::SYNC_FILE_RANGE_WRITE);
|
||||
}
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
if self.skip_wait() {
|
||||
// NFS branch (or degraded fallback after a prior
|
||||
// timeout): the WAIT_AFTER + DONTNEED dance is what
|
||||
// hangs on NFS — skip it. We still advance `pending`
|
||||
// so the next call has a stable cycle.
|
||||
} else {
|
||||
// Normal local-storage branch with belt-and-braces
|
||||
// timeout. If WAIT_AFTER hangs > WAIT_AFTER_TIMEOUT
|
||||
// we mark the pipeline degraded, log a loud error,
|
||||
// and fall through to the skip path on subsequent
|
||||
// calls.
|
||||
match wait_after_with_timeout(self.fd, prev_off, prev_len) {
|
||||
Some(ms) => {
|
||||
wait_ms = ms;
|
||||
let t_fadv = Instant::now();
|
||||
unsafe {
|
||||
libc::posix_fadvise(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
}
|
||||
fadvise_ms = t_fadv.elapsed().as_millis() as u64;
|
||||
self.record_wait(wait_ms);
|
||||
}
|
||||
None => {
|
||||
// Timeout branch: switch to NFS-style skip
|
||||
// for the rest of the pipeline's life. Do
|
||||
// NOT call DONTNEED — if WAIT_AFTER hasn't
|
||||
// returned, the pages aren't safely flushed.
|
||||
self.degraded.store(true, Ordering::Relaxed);
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackPipeline WAIT_AFTER timed out after {}s on chunk off={} len={}, marking writeback degraded (subsequent chunks will skip WAIT_AFTER + DONTNEED)",
|
||||
WAIT_AFTER_TIMEOUT.as_secs(),
|
||||
prev_off,
|
||||
prev_len
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.pending = Some((chunk_off as u64, chunk_len as u64));
|
||||
self.last_flush_pos = pos;
|
||||
self.chunk_count += 1;
|
||||
tracing::trace!(
|
||||
target: "mux",
|
||||
"WritebackPipeline chunk off={} len={} sync_file_range_ms={wait_ms} fadvise_ms={fadvise_ms} chunk_bytes={} skip_wait={}",
|
||||
chunk_off,
|
||||
chunk_len,
|
||||
self.chunk_bytes,
|
||||
self.skip_wait(),
|
||||
);
|
||||
if self.chunk_count % SIZE_LOG_INTERVAL == 0 {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackPipeline chunk_bytes={} after {} chunks is_nfs={} degraded={}",
|
||||
self.chunk_bytes,
|
||||
self.chunk_count,
|
||||
self.is_nfs,
|
||||
self.degraded.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Push a new `WAIT_AFTER` measurement into the rolling window
|
||||
/// and, if the window is full, adapt `chunk_bytes` based on p95.
|
||||
fn record_wait(&mut self, wait_ms: u64) {
|
||||
if self.wait_after_window.len() == ADAPTIVE_WINDOW {
|
||||
self.wait_after_window.pop_front();
|
||||
}
|
||||
self.wait_after_window.push_back(wait_ms);
|
||||
if self.wait_after_window.len() < ADAPTIVE_WINDOW {
|
||||
return;
|
||||
}
|
||||
// p95 of 16 samples ≈ sorted[14] (5 % of 16 = 0.8 ≈ 1 above).
|
||||
let mut sorted: Vec<u64> = self.wait_after_window.iter().copied().collect();
|
||||
sorted.sort_unstable();
|
||||
let p95 = sorted[14];
|
||||
let old = self.chunk_bytes;
|
||||
let new = if p95 > ADAPTIVE_GROW_MS && self.chunk_bytes < CHUNK_BYTES_MAX {
|
||||
(self.chunk_bytes * 2).min(CHUNK_BYTES_MAX)
|
||||
} else if p95 < ADAPTIVE_SHRINK_MS && self.chunk_bytes > CHUNK_BYTES_MIN {
|
||||
(self.chunk_bytes / 2).max(CHUNK_BYTES_MIN)
|
||||
} else {
|
||||
self.chunk_bytes
|
||||
};
|
||||
if new != old {
|
||||
self.chunk_bytes = new;
|
||||
tracing::info!(
|
||||
target: "mux",
|
||||
"WritebackPipeline adaptive chunk_bytes {} -> {} p95_ms={p95}",
|
||||
old,
|
||||
new
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Caller is about to seek away from the current write region.
|
||||
/// Drain any in-flight chunk and reset tracking.
|
||||
pub(crate) fn handle_seek(&mut self, new_pos: u64) {
|
||||
self.finalize();
|
||||
self.last_flush_pos = new_pos;
|
||||
}
|
||||
|
||||
/// Drain any in-flight chunk. Idempotent. Call before `sync_all()`
|
||||
/// or when discarding the pipeline.
|
||||
pub(crate) fn finalize(&mut self) {
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackPipeline finalize chunk off={prev_off} len={prev_len} skip_wait={} is_nfs={} degraded={}",
|
||||
self.skip_wait(),
|
||||
self.is_nfs,
|
||||
self.degraded.load(Ordering::Relaxed),
|
||||
);
|
||||
if self.skip_wait() {
|
||||
// NFS / degraded: skip WAIT_AFTER + DONTNEED. close()
|
||||
// / sync_all() handle commit through their normal
|
||||
// paths.
|
||||
return;
|
||||
}
|
||||
match wait_after_with_timeout(self.fd, prev_off, prev_len) {
|
||||
Some(_ms) => unsafe {
|
||||
libc::posix_fadvise(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
},
|
||||
None => {
|
||||
self.degraded.store(true, Ordering::Relaxed);
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackPipeline finalize WAIT_AFTER timed out after {}s on chunk off={prev_off} len={prev_len}, marking writeback degraded",
|
||||
WAIT_AFTER_TIMEOUT.as_secs(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe whether `fd` lives on an NFS mount. Thin wrapper around
|
||||
/// [`crate::platform::fs_type::detect_fd`] so writeback policy and
|
||||
/// general-purpose fs-type classification stay in sync (same magic
|
||||
/// numbers, same musl-vs-glibc cast handling).
|
||||
///
|
||||
/// Fails open: any classification other than NFS counts as "not NFS"
|
||||
/// (including `Unknown` on `fstatfs` error) — better to run the
|
||||
/// normal local-storage path on a misdetected NFS mount and surface
|
||||
/// the freeze loudly via [`WAIT_AFTER_TIMEOUT`] than to needlessly
|
||||
/// disable writeback bounding on every local file because of a
|
||||
/// transient stat error.
|
||||
fn detect_nfs(fd: RawFd) -> bool {
|
||||
matches!(
|
||||
crate::platform::fs_type::detect_fd(fd),
|
||||
crate::platform::fs_type::FsType::Nfs
|
||||
)
|
||||
}
|
||||
|
||||
/// Run `sync_file_range(WAIT_AFTER)` on a worker thread and wait up
|
||||
/// to [`WAIT_AFTER_TIMEOUT`] for it to return. `Some(elapsed_ms)` on
|
||||
/// success; `None` on timeout. On timeout the worker thread is
|
||||
/// 0.20.6 generalizes the worker-thread + recv_timeout pattern into
|
||||
/// [`crate::io::bounded::bounded_syscall`]; this helper now just adapts
|
||||
/// the generic primitive to the WAIT_AFTER call shape (returns elapsed_ms
|
||||
/// instead of the syscall's `()` return, treats `WorkerLost` as a benign
|
||||
/// no-op to match the original semantics).
|
||||
fn wait_after_with_timeout(fd: RawFd, off: u64, len: u64) -> Option<u64> {
|
||||
let started = Instant::now();
|
||||
match crate::io::bounded::bounded_syscall(None, WAIT_AFTER_TIMEOUT, move || unsafe {
|
||||
libc::sync_file_range(fd, off as i64, len as i64, libc::SYNC_FILE_RANGE_WAIT_AFTER);
|
||||
}) {
|
||||
Ok(()) => Some(started.elapsed().as_millis() as u64),
|
||||
Err(crate::io::bounded::BoundedError::Timeout)
|
||||
| Err(crate::io::bounded::BoundedError::Halted) => None,
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => {
|
||||
// Worker thread spawn failed or panicked before sending.
|
||||
// Treat as a benign success (no syscall ran) rather than
|
||||
// a degrade trigger — falling through with elapsed_ms=0
|
||||
// matches the no-op behaviour.
|
||||
Some(0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
/// Helper: build a `WritebackPipeline` over a local tempfile. On
|
||||
/// every test rig (linux dev box, CI) the tempfile lives on a
|
||||
/// local FS, so `is_nfs=false` and `skip_wait` returns false until
|
||||
/// we explicitly mark the pipeline degraded.
|
||||
fn local_pipeline(chunk_bytes: u64) -> (NamedTempFile, WritebackPipeline) {
|
||||
let f = NamedTempFile::new().expect("tempfile create");
|
||||
let pipeline = WritebackPipeline::new(f.as_file(), 0, chunk_bytes);
|
||||
(f, pipeline)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_pipeline_starts_active() {
|
||||
let (_f, p) = local_pipeline(32 * 1024 * 1024);
|
||||
assert!(!p.is_nfs, "local tempfile must not classify as NFS");
|
||||
assert!(!p.degraded.load(Ordering::Relaxed));
|
||||
assert!(!p.skip_wait(), "fresh local pipeline must not skip wait");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn degraded_flag_short_circuits_wait() {
|
||||
let (_f, p) = local_pipeline(32 * 1024 * 1024);
|
||||
assert!(!p.skip_wait());
|
||||
p.degraded.store(true, Ordering::Relaxed);
|
||||
assert!(
|
||||
p.skip_wait(),
|
||||
"degraded flag must force the wait+dontneed bypass"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_wait_grows_chunk_on_high_p95() {
|
||||
let (_f, mut p) = local_pipeline(16 * 1024 * 1024);
|
||||
// Fill the window with samples above the grow threshold.
|
||||
for _ in 0..ADAPTIVE_WINDOW {
|
||||
p.record_wait(ADAPTIVE_GROW_MS + 50);
|
||||
}
|
||||
assert!(
|
||||
p.chunk_bytes > 16 * 1024 * 1024,
|
||||
"chunk should have grown; got {}",
|
||||
p.chunk_bytes
|
||||
);
|
||||
assert!(p.chunk_bytes <= CHUNK_BYTES_MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_wait_shrinks_chunk_on_low_p95() {
|
||||
let (_f, mut p) = local_pipeline(64 * 1024 * 1024);
|
||||
for _ in 0..ADAPTIVE_WINDOW {
|
||||
p.record_wait(1); // well under ADAPTIVE_SHRINK_MS
|
||||
}
|
||||
assert!(
|
||||
p.chunk_bytes < 64 * 1024 * 1024,
|
||||
"chunk should have shrunk; got {}",
|
||||
p.chunk_bytes
|
||||
);
|
||||
assert!(p.chunk_bytes >= CHUNK_BYTES_MIN);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_wait_no_op_below_window_fill() {
|
||||
let (_f, mut p) = local_pipeline(16 * 1024 * 1024);
|
||||
let initial = p.chunk_bytes;
|
||||
// Only push a few samples; window not full → no adaptation.
|
||||
for _ in 0..(ADAPTIVE_WINDOW - 1) {
|
||||
p.record_wait(ADAPTIVE_GROW_MS + 100);
|
||||
}
|
||||
assert_eq!(
|
||||
p.chunk_bytes, initial,
|
||||
"chunk must not change before window is full"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_wait_clamps_to_chunk_bounds() {
|
||||
// Grow past the max.
|
||||
let (_f, mut p) = local_pipeline(CHUNK_BYTES_MAX);
|
||||
for _ in 0..ADAPTIVE_WINDOW {
|
||||
p.record_wait(ADAPTIVE_GROW_MS + 1000);
|
||||
}
|
||||
assert_eq!(p.chunk_bytes, CHUNK_BYTES_MAX, "must clamp to MAX");
|
||||
|
||||
// Shrink past the min.
|
||||
let (_f, mut p) = local_pipeline(CHUNK_BYTES_MIN);
|
||||
for _ in 0..ADAPTIVE_WINDOW {
|
||||
p.record_wait(0);
|
||||
}
|
||||
assert_eq!(p.chunk_bytes, CHUNK_BYTES_MIN, "must clamp to MIN");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_nfs_local_file_is_false() {
|
||||
// Local tempfile must not classify as NFS. This locks in the
|
||||
// consolidation through `crate::platform::fs_type::detect_fd`.
|
||||
let f = NamedTempFile::new().expect("tempfile create");
|
||||
use std::os::unix::io::AsRawFd;
|
||||
assert!(!detect_nfs(f.as_file().as_raw_fd()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn note_progress_below_chunk_is_noop() {
|
||||
let (_f, mut p) = local_pipeline(32 * 1024 * 1024);
|
||||
// No-op return before crossing the first chunk boundary.
|
||||
let before = p.chunk_count;
|
||||
p.note_progress(1024); // < 32 MiB
|
||||
assert_eq!(p.chunk_count, before);
|
||||
assert!(p.pending.is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//! No-op writeback pipeline for non-Linux targets. macOS and Windows
|
||||
//! page cache policies have not been shown to exhibit the Linux
|
||||
//! accumulate-then-burst flush pathology for our access pattern.
|
||||
//! If that changes, replace this stub with a real implementation
|
||||
//! (e.g. `F_NOCACHE` on macOS, `FILE_FLAG_WRITE_THROUGH` on Windows).
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(crate) struct WritebackPipeline;
|
||||
|
||||
impl WritebackPipeline {
|
||||
pub(crate) fn new(_file: &File, _start_pos: u64, _chunk_bytes: u64) -> Self {
|
||||
Self
|
||||
}
|
||||
pub(crate) fn note_progress(&mut self, _pos: u64) {}
|
||||
pub(crate) fn handle_seek(&mut self, _new_pos: u64) {}
|
||||
pub(crate) fn finalize(&mut self) {}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//! Linux platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! - `preallocate`: `fallocate(FALLOC_FL_KEEP_SIZE)` — reserve extents
|
||||
//! without growing the reported file size. Reduces extent
|
||||
//! fragmentation on large sequential writes (mux output on NFS in
|
||||
//! particular).
|
||||
//! - `durable_sync`: `fsync` wrapped in
|
||||
//! [`crate::io::bounded::bounded_syscall`] with a 60 s deadline so a
|
||||
//! wedged NFS server can't trap the calling thread indefinitely.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Pre-reserve extents for `size_bytes` of upcoming sequential writes.
|
||||
/// Best-effort: a non-zero rc is logged but not propagated, since the
|
||||
/// caller would just continue with the unreserved file anyway.
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
// FALLOC_FL_KEEP_SIZE = 0x01 — keep the reported file size at 0
|
||||
// (writes grow it normally) while still pre-reserving the extents.
|
||||
let rc = unsafe {
|
||||
libc::fallocate(
|
||||
file.as_raw_fd(),
|
||||
libc::FALLOC_FL_KEEP_SIZE,
|
||||
0,
|
||||
size_bytes as i64,
|
||||
)
|
||||
};
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile fallocate size_hint={size_bytes} rc={rc} ok={}",
|
||||
rc == 0
|
||||
);
|
||||
}
|
||||
|
||||
/// Run `fsync` on `file` with a 60 s deadline. On timeout we log loudly
|
||||
/// and return `Ok(())` — the kernel will still flush on close, so the
|
||||
/// data is best-effort durable; the alternative (trap the thread for
|
||||
/// the rest of the rip) defeats `/api/stop`.
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
let fd = file.as_raw_fd();
|
||||
match crate::io::bounded::bounded_syscall(
|
||||
None,
|
||||
Duration::from_secs(60),
|
||||
move || -> io::Result<()> {
|
||||
let rc = unsafe { libc::fsync(fd) };
|
||||
if rc == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::last_os_error())
|
||||
}
|
||||
},
|
||||
) {
|
||||
Ok(inner) => inner,
|
||||
Err(crate::io::bounded::BoundedError::Timeout) => {
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all fsync timed out after 60s; kernel will flush on close (best-effort)"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(crate::io::bounded::BoundedError::Halted) => Ok(()),
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => Ok(()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
//! macOS platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! - `preallocate`: `fcntl(F_PREALLOCATE)` — macOS's fallocate-equiv.
|
||||
//! Reserves a contiguous extent when possible, falling back to a
|
||||
//! non-contiguous reservation if the FS can't satisfy it. Reported
|
||||
//! file size is unchanged (`F_ALLOCATEALL` is not set, so allocation
|
||||
//! is "best effort up to length"; growth happens via writes).
|
||||
//! - `durable_sync`: `fcntl(F_FULLFSYNC)` wrapped in
|
||||
//! [`crate::io::bounded::bounded_syscall`] with a 60 s deadline.
|
||||
//! F_FULLFSYNC is HFS+/APFS's true-fsync (flushes the disk's own
|
||||
//! write cache) — what `fsync` should have been on macOS. Falls back
|
||||
//! to plain `fsync` if F_FULLFSYNC returns ENOTSUP.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::io::platform_macos::{
|
||||
F_ALLOCATEALL, F_ALLOCATECONTIG, F_PEOFPOSMODE, F_PREALLOCATE, Fstore,
|
||||
};
|
||||
|
||||
/// `fcntl(F_FULLFSYNC)` opcode. Documented in `man 2 fcntl` on macOS;
|
||||
/// not in the `libc` crate as a named constant.
|
||||
const F_FULLFSYNC: libc::c_int = 51;
|
||||
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
let mut fst = Fstore {
|
||||
fst_flags: F_ALLOCATECONTIG | F_ALLOCATEALL,
|
||||
fst_posmode: F_PEOFPOSMODE,
|
||||
fst_offset: 0,
|
||||
fst_length: size_bytes as libc::off_t,
|
||||
fst_bytesalloc: 0,
|
||||
};
|
||||
// First attempt: contiguous.
|
||||
let mut rc = unsafe { libc::fcntl(file.as_raw_fd(), F_PREALLOCATE, &mut fst) };
|
||||
if rc == -1 {
|
||||
// Fall back: drop the contiguous hint, allow scattered extents.
|
||||
fst.fst_flags = F_ALLOCATEALL;
|
||||
rc = unsafe { libc::fcntl(file.as_raw_fd(), F_PREALLOCATE, &mut fst) };
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile F_PREALLOCATE size_hint={size_bytes} rc={rc} bytes_allocated={} ok={}",
|
||||
fst.fst_bytesalloc,
|
||||
rc != -1
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
let fd = file.as_raw_fd();
|
||||
match crate::io::bounded::bounded_syscall(
|
||||
None,
|
||||
Duration::from_secs(60),
|
||||
move || -> io::Result<()> {
|
||||
// Try F_FULLFSYNC first. If it isn't supported on this
|
||||
// filesystem (older HFS, some network mounts) fall back to
|
||||
// plain fsync — better than nothing.
|
||||
let rc = unsafe { libc::fcntl(fd, F_FULLFSYNC, 0) };
|
||||
if rc == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let err = io::Error::last_os_error();
|
||||
if err.raw_os_error() == Some(libc::ENOTSUP) {
|
||||
let rc = unsafe { libc::fsync(fd) };
|
||||
if rc == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::last_os_error())
|
||||
}
|
||||
} else {
|
||||
Err(err)
|
||||
}
|
||||
},
|
||||
) {
|
||||
Ok(inner) => inner,
|
||||
Err(crate::io::bounded::BoundedError::Timeout) => {
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all F_FULLFSYNC timed out after 60s; kernel will flush on close (best-effort)"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(crate::io::bounded::BoundedError::Halted) => Ok(()),
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => Ok(()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
//! `WritebackFile` — a `File` wrapper whose reason for existing is the
|
||||
//! bounded-cache writeback pipeline.
|
||||
//!
|
||||
//! Why: large sequential writes (sweep, patch, mux on UHD-scale output)
|
||||
//! left to the kernel's default writeback policy accumulate hundreds of
|
||||
//! megabytes of dirty pages and then burst-flush, stalling subsequent
|
||||
//! writes for seconds at a time. `WritebackFile` drives a continuous
|
||||
//! [`super::writeback::WritebackPipeline`] that on Linux issues
|
||||
//! incremental `sync_file_range` + `posix_fadvise(DONTNEED)` calls at
|
||||
//! 32 MB granularity so dirty pages drain at the same rate they're
|
||||
//! produced. macOS and Windows fall through to a no-op pipeline — their
|
||||
//! default cache policies have not been shown to exhibit the same
|
||||
//! pathology for this access pattern.
|
||||
//!
|
||||
//! It implements `Write` and `Seek` so any call site that wrote to a
|
||||
//! plain `File` through those traits (sweep, patch, mux) can swap in
|
||||
//! `WritebackFile` without touching the body of the loop. The wrapper
|
||||
//! also tracks the current file position to feed the pipeline with
|
||||
//! progress + seek boundaries.
|
||||
//!
|
||||
//! See `super::writeback::linux` for the underlying pathology and the
|
||||
//! strategy.
|
||||
//!
|
||||
//! ## Platform split
|
||||
//!
|
||||
//! The platform-specific pieces of this wrapper — extent preallocation
|
||||
//! (Linux `fallocate(KEEP_SIZE)`, macOS `F_PREALLOCATE`, Windows
|
||||
//! `SetFileValidData`) and the durable-flush primitive (Linux/macOS
|
||||
//! `fsync`/`F_FULLFSYNC` wrapped in a bounded syscall, Windows
|
||||
//! `FlushFileBuffers`) — live in per-OS sibling modules. The dispatch
|
||||
//! happens once at the bottom of this file via cfg-gated `mod` decls.
|
||||
//! No inline `#[cfg(target_os = "...")]` in the business-logic above.
|
||||
//!
|
||||
//! ## Write path
|
||||
//!
|
||||
//! Writes are direct passthrough to the underlying `File` (no writer
|
||||
//! thread, no ring, no batching). Empirically the Phase-2.5
|
||||
//! writer-thread architecture introduced a ~60% mux throughput
|
||||
//! regression on NFS bidirectional workloads; reverting the write path
|
||||
//! to direct passthrough restores the 0.20.7 baseline. The writeback
|
||||
//! pipeline still runs (it's called inline from `write` / `write_all` /
|
||||
//! `seek`) so the bounded-cache invariant on Linux is preserved.
|
||||
//!
|
||||
//! ## Halt-safety
|
||||
//!
|
||||
//! `sync_all` runs the per-OS durable-flush primitive, which on
|
||||
//! Linux/macOS is wrapped in [`crate::io::bounded::bounded_syscall`]
|
||||
//! with a 60 s deadline. A wedged NFS server cannot trap the muxer
|
||||
//! indefinitely on the final fsync.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
mod other;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux as platform;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos as platform;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
use other as platform;
|
||||
#[cfg(target_os = "windows")]
|
||||
use windows as platform;
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use super::writeback::WritebackPipeline;
|
||||
|
||||
/// Granularity at which the Linux writeback pipeline issues
|
||||
/// `sync_file_range` pairs. 32 MiB is the empirically best value on
|
||||
/// the rip1 test bed (NFS to unraid-1 over 1 GbE, single-disk SAS):
|
||||
/// 8 MiB / 64 MiB / 128 MiB all measured worse in the 0.21.x mux
|
||||
/// iteration runs. Override via `FREEMKV_WRITEBACK_CHUNK_MIB` —
|
||||
/// faster backends (NVMe, RAID) may tolerate larger windows.
|
||||
const WRITEBACK_CHUNK_BYTES_DEFAULT: u64 = 32 * 1024 * 1024;
|
||||
|
||||
fn writeback_chunk_bytes() -> u64 {
|
||||
std::env::var("FREEMKV_WRITEBACK_CHUNK_MIB")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.filter(|&n| n > 0)
|
||||
.map(|n| n * 1024 * 1024)
|
||||
.unwrap_or(WRITEBACK_CHUNK_BYTES_DEFAULT)
|
||||
}
|
||||
|
||||
pub(crate) struct WritebackFile {
|
||||
file: File,
|
||||
pipeline: WritebackPipeline,
|
||||
pos: u64,
|
||||
}
|
||||
|
||||
impl WritebackFile {
|
||||
/// Wrap an open `File`. The current OS file position is queried
|
||||
/// once so the pipeline starts tracking from wherever the file
|
||||
/// already is (typically 0 for fresh files; non-zero for resumed
|
||||
/// or appended files).
|
||||
pub(crate) fn new(mut file: File) -> io::Result<Self> {
|
||||
let pos = file.stream_position()?;
|
||||
let pipeline = WritebackPipeline::new(&file, pos, writeback_chunk_bytes());
|
||||
Ok(Self {
|
||||
file,
|
||||
pipeline,
|
||||
pos,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new file at `path` (truncating any existing contents)
|
||||
/// and wrap it. Convenience for the common
|
||||
/// `File::create(path)` + `WritebackFile::new(file)` pair so callers
|
||||
/// don't have to assemble a `File` first.
|
||||
///
|
||||
/// Callers that know the target output size should prefer
|
||||
/// [`Self::create_with_size_hint`] so the kernel can pre-reserve
|
||||
/// extents.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = File::create(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Like [`Self::create`] but pre-reserves `size_bytes` of disk
|
||||
/// space via the platform's extent-preallocation primitive (Linux
|
||||
/// `fallocate(KEEP_SIZE)`, macOS `F_PREALLOCATE`, Windows
|
||||
/// `SetFileValidData` stub). The reported file size is unchanged
|
||||
/// (writes still grow the file naturally) — only the on-disk extent
|
||||
/// allocation is preallocated, which reduces extent fragmentation
|
||||
/// on large sequential writes (mux output, especially on slow
|
||||
/// storage / NFS).
|
||||
///
|
||||
/// On platforms without an extent-preallocation primitive this is
|
||||
/// equivalent to `create` — the size hint is dropped after a debug
|
||||
/// log.
|
||||
pub(crate) fn create_with_size_hint(path: &Path, size_bytes: u64) -> io::Result<Self> {
|
||||
let file = File::create(path)?;
|
||||
platform::preallocate(&file, size_bytes);
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Open an existing file at `path` for writing (no truncation) and
|
||||
/// wrap it. Mirrors `File::open` semantics for the writable case
|
||||
/// — used by patch / resume paths that mutate an existing ISO in
|
||||
/// place.
|
||||
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new().write(true).open(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Drain in-flight writeback then issue a full fsync. Use this in
|
||||
/// place of `File::sync_all`.
|
||||
///
|
||||
/// The final durable flush is wrapped in
|
||||
/// [`crate::io::bounded::bounded_syscall`] (per the per-OS module)
|
||||
/// with a 60 s deadline on Linux/macOS — a wedged NFS server cannot
|
||||
/// trap the calling thread indefinitely. On timeout the page cache
|
||||
/// is left to the kernel's normal flush-on-close path — best
|
||||
/// effort, but bounded.
|
||||
pub(crate) fn sync_all(&mut self) -> io::Result<()> {
|
||||
self.pipeline.finalize();
|
||||
platform::durable_sync(&self.file)
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for WritebackFile {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let n = self.file.write(buf)?;
|
||||
self.pos += n as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.file.write_all(buf)?;
|
||||
self.pos += buf.len() as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for WritebackFile {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
let p = self.file.seek(from)?;
|
||||
// Only treat seeks that actually move the position as
|
||||
// boundaries — sweep does a redundant `seek(Current(pos))`
|
||||
// before every write, and we don't want that to drain the
|
||||
// pipeline on every iteration.
|
||||
if p != self.pos {
|
||||
// Diagnostic for the NFS mux hang: the MKV format requires
|
||||
// the muxer to seek back occasionally (cluster size
|
||||
// patching, Cues index write, Segment header backpatch).
|
||||
// Each such seek invalidates the writeback chunk tracking
|
||||
// and forces a finalize → WAIT_AFTER on the in-flight
|
||||
// chunk. Logging the seek delta lets us correlate hang
|
||||
// offsets with specific muxer operations.
|
||||
let from_pos = self.pos;
|
||||
let to_pos = p;
|
||||
let delta: i64 = (to_pos as i64).wrapping_sub(from_pos as i64);
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile seek from={from_pos} to={to_pos} delta={delta}"
|
||||
);
|
||||
self.pipeline.handle_seek(p);
|
||||
self.pos = p;
|
||||
}
|
||||
Ok(p)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WritebackFile {
|
||||
fn drop(&mut self) {
|
||||
// Run the pipeline's tail finalize so the last in-flight chunk
|
||||
// gets its `WAIT_AFTER` + `posix_fadvise(DONTNEED)`. Without
|
||||
// this, callers that drop a `WritebackFile` without calling
|
||||
// `sync_all` (panic, early-return, idiomatic `let _ = w;`)
|
||||
// leave the trailing chunk in cache; the kernel still flushes
|
||||
// on close, but the bounded-cache invariant fails at the tail.
|
||||
// We deliberately do *not* call `self.file.sync_all()` here —
|
||||
// close already triggers a flush, and an `fsync` from `Drop`
|
||||
// would silently swallow its `io::Error` anyway. `finalize` is
|
||||
// idempotent so an explicit `sync_all` followed by drop is
|
||||
// still safe.
|
||||
self.pipeline.finalize();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
|
||||
fn read_back(path: &Path) -> Vec<u8> {
|
||||
let mut f = File::open(path).unwrap();
|
||||
let mut v = Vec::new();
|
||||
f.read_to_end(&mut v).unwrap();
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_then_drop_persists_bytes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("a.bin");
|
||||
{
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"hello world").unwrap();
|
||||
// Drop drains the pipeline tail.
|
||||
}
|
||||
assert_eq!(read_back(&p), b"hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_all_drains_and_flushes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("b.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
for _ in 0..32 {
|
||||
w.write_all(&[0x5au8; 1024]).unwrap();
|
||||
}
|
||||
// After sync_all, the bytes MUST be visible to a separate
|
||||
// reader. The pipeline has been finalised and durable-sync has
|
||||
// run.
|
||||
w.sync_all().unwrap();
|
||||
let bytes = read_back(&p);
|
||||
assert_eq!(bytes.len(), 32 * 1024);
|
||||
assert!(bytes.iter().all(|&b| b == 0x5a));
|
||||
drop(w);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seek_then_patch_roundtrip() {
|
||||
// Write A; seek back; patch with B; read back; the patch lands
|
||||
// at the right offset.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("c.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
let big = vec![b'A'; 4096];
|
||||
w.write_all(&big).unwrap();
|
||||
// Seek back to offset 1000 and overwrite 8 bytes.
|
||||
w.seek(SeekFrom::Start(1000)).unwrap();
|
||||
w.write_all(b"PATCHED!").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
let bytes = read_back(&p);
|
||||
assert_eq!(bytes.len(), 4096);
|
||||
assert_eq!(&bytes[1000..1008], b"PATCHED!");
|
||||
// Bytes outside the patch are still 'A'.
|
||||
assert_eq!(bytes[999], b'A');
|
||||
assert_eq!(bytes[1008], b'A');
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flush_is_observed_in_order() {
|
||||
// `Write::flush` should not panic or reorder; verify the bytes
|
||||
// land in order through interleaved flushes.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("f.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"one").unwrap();
|
||||
w.flush().unwrap();
|
||||
w.write_all(b"two").unwrap();
|
||||
w.flush().unwrap();
|
||||
w.write_all(b"three").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
assert_eq!(read_back(&p), b"onetwothree");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//! Fallback platform impl for [`super::WritebackFile`] on targets
|
||||
//! without a dedicated implementation (BSDs, illumos, etc.).
|
||||
//!
|
||||
//! - `preallocate` is a logged no-op.
|
||||
//! - `durable_sync` calls `File::sync_all` directly (no bounded-syscall
|
||||
//! wrapper — the wrapper depends on Linux/macOS unix idioms that
|
||||
//! aren't universally portable). If a future BSD impl needs the
|
||||
//! 60-s deadline, it should land in its own per-OS file rather than
|
||||
//! bloat this fallback.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
|
||||
pub(super) fn preallocate(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile preallocate size_hint={size_bytes} skipped (no impl on this target)"
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
file.sync_all()
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//! Windows platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! TODO: this stub matches the design's "validate without a Windows
|
||||
//! build env, leave a stub" carve-out. The real impl should use:
|
||||
//!
|
||||
//! - `SetEndOfFile` + `SetFileValidData` for extent preallocation
|
||||
//! (caller needs `SE_MANAGE_VOLUME_NAME` privilege; if unavailable
|
||||
//! fall back to a write-zero path or just skip).
|
||||
//! - `FlushFileBuffers` for fsync-equivalent durable flush.
|
||||
//!
|
||||
//! Until then: preallocate is a debug-logged no-op; durable_sync calls
|
||||
//! the std `File::sync_all` (which on Windows maps to
|
||||
//! `FlushFileBuffers` internally).
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
|
||||
pub(super) fn preallocate(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile preallocate size_hint={size_bytes} skipped (windows stub; TODO: SetFileValidData)"
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
// `File::sync_all` on Windows is `FlushFileBuffers`. Acceptable
|
||||
// for now; the bounded-syscall wrapper is not used here because
|
||||
// the stub also skips the worker-thread + leak machinery (the
|
||||
// wrapper would need an `unsafe impl Send` for `RawHandle`, and
|
||||
// designing that without a Windows test env is asking for it).
|
||||
file.sync_all()
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
//! Key sources — the lookup layer that hands libfreemkv a [`Key`].
|
||||
//!
|
||||
//! libfreemkv performs NO key lookup. An application resolves a key for a disc
|
||||
//! through one or more [`KeySource`]s, each a dumb adapter over a backing store
|
||||
//! (a keydb file, a key server, the mapfile cache): given the disc's
|
||||
//! [`DiscInputs`] it returns the raw [`Key`] at whatever level it holds. The
|
||||
//! library then derives down and decrypts via `Disc::decrypt_with`.
|
||||
//!
|
||||
//! Source implementations are published in the companion `freemkv-keysources`
|
||||
//! crate — keeping all key *policy* (which store, which order, online vs local)
|
||||
//! out of the library while all key *mechanism* (the AACS derivation chain)
|
||||
//! stays in it.
|
||||
|
||||
use crate::disc::Key;
|
||||
|
||||
/// The public AACS inputs a key source needs to look a disc up. Captured at
|
||||
/// scan; contains no secrets — only the disc identity and the on-disc AACS
|
||||
/// structures a source or key server may key on.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DiscInputs {
|
||||
/// SHA-1 of `Unit_Key_RO.inf`, `0x`-prefixed hex. The value a keydb keys
|
||||
/// its per-disc entries by, and a key server identifies the disc with.
|
||||
pub disc_hash: String,
|
||||
/// Volume ID (16 bytes). `[0u8; 16]` when no authenticated handshake ran
|
||||
/// (e.g. an ISO/mapfile flow), which disables VID-keyed lookups.
|
||||
pub volume_id: [u8; 16],
|
||||
/// Raw MKB bytes. Empty when not captured.
|
||||
pub mkb: Vec<u8>,
|
||||
/// Raw `Unit_Key_RO.inf` bytes. Empty when not captured.
|
||||
pub unit_key_ro: Vec<u8>,
|
||||
/// Encrypted on-disc content sample units (each a 6144-byte aligned unit),
|
||||
/// for sources that validate a key server-side against real ciphertext
|
||||
/// (e.g. an online key service). Empty for sources that don't need them
|
||||
/// (a local keydb). Populated by the application — reading content requires
|
||||
/// the disc reader, which the library's scan does not retain — so
|
||||
/// [`crate::Disc::inputs`] leaves it empty for the caller to fill.
|
||||
pub samples: Vec<Vec<u8>>,
|
||||
/// The disc's human title — the UDF/ISO volume identifier (e.g.
|
||||
/// `WICKED_FOR_GOOD`), falling back to the BDMV `<di:name>` when present.
|
||||
/// `None` when not captured. Identity only, no secret; a key service may
|
||||
/// record it (keyed by `disc_hash`) to build a hash→title catalog. Not used
|
||||
/// in any AACS derivation.
|
||||
pub volume_label: Option<String>,
|
||||
}
|
||||
|
||||
/// A key source: a stateful provider that hands a disc's candidate [`Key`]s out
|
||||
/// **one at a time**, in whatever order it judges best for its backing store.
|
||||
///
|
||||
/// Dumb by contract — a source queries its store and yields the raw material it
|
||||
/// holds at whatever level it has (device / processing / media / volume / unit).
|
||||
/// It performs NO AACS derivation and NO validation: `Disc::decrypt_with`
|
||||
/// derives down AND validates against real ciphertext, returning `Err` for a key
|
||||
/// that does not decrypt this disc. That keeps every derivation step and the one
|
||||
/// validation gate in the library, across AACS 1.0 / 2.0 / 2.1 / 2.x.
|
||||
///
|
||||
/// The source is the one that knows how many candidates it has and in what order
|
||||
/// to try them — a keydb holds a per-disc UK *and* VUK *and* a device-key pool,
|
||||
/// so it hands them out cheapest/most-specific first (UK ▸ VK ▸ MK ▸ DK) and
|
||||
/// reports exhaustion when its list runs out; an online key service or a mapfile
|
||||
/// cache hold exactly one. The caller drives the loop: `next_key` →
|
||||
/// `Disc::decrypt_with` → on `Err`, ask again → until a key decrypts or the
|
||||
/// source returns `None` (a genuine "no key for this disc"). Compose several
|
||||
/// sources, in the caller's chosen order, with [`crate`]'s `MultiSource`.
|
||||
pub trait KeySource {
|
||||
/// Hand the NEXT candidate key for this disc, or `None` once this source is
|
||||
/// exhausted. Stateful: the source tracks what it already handed out this
|
||||
/// session, so asking again after a rejected key yields the next candidate
|
||||
/// (or `None`) — it never re-offers a key or re-hits a one-shot backend (an
|
||||
/// online service is asked at most once). A source failure (I/O, network,
|
||||
/// parse) surfaces as `None` — there is simply nothing more to try.
|
||||
fn next_key(&mut self, inputs: &DiscInputs) -> Option<Key>;
|
||||
|
||||
/// Whether this source needs [`DiscInputs::samples`] populated (encrypted
|
||||
/// content samples) — true for a source that validates against ciphertext
|
||||
/// server-side, false for one that keys purely on disc identity. The caller
|
||||
/// reads samples (an extra disc read) only when some source needs them.
|
||||
fn needs_samples(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Whether this source FAILED (I/O, network, parse) rather than simply
|
||||
/// having no key. Checked after exhaustion so the caller can tell a genuine
|
||||
/// "no key for this disc" apart from "the key service was unreachable". A
|
||||
/// store that treats absence as not-an-error (a missing keydb / mapfile)
|
||||
/// leaves this `false`.
|
||||
fn errored(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
//! BDMV disc-library metadata (`/BDMV/META/DL/bdmt_<lang>.xml`).
|
||||
//!
|
||||
//! Every commercial Blu-ray carries a disc-library metadata directory
|
||||
//! with one XML file per shipped language. The schema is the Blu-ray
|
||||
//! "disc library metadata" namespace (`urn:BDA:bdmv;disclibmeta`),
|
||||
//! conventionally prefixed `di:`. Fields commonly present:
|
||||
//!
|
||||
//! - `<di:title>` or `<di:name>` — the title string. Vendor practice
|
||||
//! varies (Paramount discs tend to use `<di:name>`).
|
||||
//! - `<di:description>` — optional synopsis (often absent on retail
|
||||
//! discs; common on box sets and special editions).
|
||||
//! - `<di:discNumber>` / `<di:numSets>` (or `<di:numberOfSets>`) —
|
||||
//! set position for multi-disc releases.
|
||||
//!
|
||||
//! This module is intentionally separate from the BD-J `StreamLabel`
|
||||
//! parsers under `labels/*.rs`. The XML here is disc-level (title,
|
||||
//! description, set position), not per-stream — wiring into the main
|
||||
//! parser registry happens elsewhere.
|
||||
//!
|
||||
//! Real-world XML is irregular: missing description elements, multiple
|
||||
//! title elements (first one wins), and occasional malformed content.
|
||||
//! Extraction is best-effort — a malformed file is treated as "no
|
||||
//! metadata" (returns `None` from the helper), and the caller can
|
||||
//! still get metadata from sibling-language XML files.
|
||||
|
||||
// The module wiring (registry hook + public re-export) is added
|
||||
// separately. Until then the parse/detect entry points have no
|
||||
use super::xml;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Disc-level metadata extracted from `/BDMV/META/DL/bdmt_*.xml`.
|
||||
///
|
||||
/// All maps are keyed by 3-char ISO 639-2 language code (e.g.
|
||||
/// `"eng"`, `"fra"`, `"jpn"`) — the same key segment used in the
|
||||
/// `bdmt_<lang>.xml` filename.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)]
|
||||
pub struct DiscMetadata {
|
||||
/// Localized titles, keyed by 3-char ISO 639-2 lang code
|
||||
/// (e.g. "eng" → "Aurora Drift")
|
||||
pub titles: BTreeMap<String, String>,
|
||||
/// First-line / short description, per lang
|
||||
pub descriptions: BTreeMap<String, String>,
|
||||
/// Disc N of M for box sets (None if not a box set)
|
||||
pub disc_number: Option<(u32, u32)>,
|
||||
}
|
||||
|
||||
/// True if `/BDMV/META/DL/` exists and contains at least one
|
||||
/// `bdmt_*.xml` file.
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
let Some(dir) = udf.find_dir("/BDMV/META/DL") else {
|
||||
return false;
|
||||
};
|
||||
dir.entries
|
||||
.iter()
|
||||
.any(|e| !e.is_dir && is_bdmt_filename(&e.name))
|
||||
}
|
||||
|
||||
/// Read every `bdmt_<lang>.xml` under `/BDMV/META/DL/` and return the
|
||||
/// aggregated [`DiscMetadata`]. Returns `None` if no titles could be
|
||||
/// extracted from any file.
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<DiscMetadata> {
|
||||
let dir = udf.find_dir("/BDMV/META/DL")?;
|
||||
let mut out = DiscMetadata::default();
|
||||
|
||||
for entry in &dir.entries {
|
||||
if entry.is_dir {
|
||||
continue;
|
||||
}
|
||||
let Some(lang) = lang_code_from_filename(&entry.name) else {
|
||||
continue;
|
||||
};
|
||||
let path = format!("/BDMV/META/DL/{}", entry.name);
|
||||
let Ok(bytes) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(text) = std::str::from_utf8(&bytes) else {
|
||||
continue;
|
||||
};
|
||||
let Some((title, description, disc_set)) = parse_bdmt_xml(&lang, text) else {
|
||||
continue;
|
||||
};
|
||||
out.titles.insert(lang.clone(), title);
|
||||
if let Some(desc) = description {
|
||||
out.descriptions.insert(lang.clone(), desc);
|
||||
}
|
||||
// Disc-set position is disc-global; first one we successfully
|
||||
// read wins. (All bdmt_*.xml on a given disc carry the same
|
||||
// value in practice.)
|
||||
if out.disc_number.is_none() {
|
||||
if let Some(ds) = disc_set {
|
||||
out.disc_number = Some(ds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if out.titles.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// True if `name` matches the `bdmt_<lang>.xml` convention with a
|
||||
/// 3-character ISO 639-2 lang code segment. Case-insensitive.
|
||||
fn is_bdmt_filename(name: &str) -> bool {
|
||||
lang_code_from_filename(name).is_some()
|
||||
}
|
||||
|
||||
/// Extract the 3-char language code from a `bdmt_<lang>.xml` filename.
|
||||
/// Returns `None` if the filename doesn't match. Lang code is
|
||||
/// lowercased so callers always see e.g. `"eng"` not `"ENG"`.
|
||||
fn lang_code_from_filename(name: &str) -> Option<String> {
|
||||
let lower = name.to_ascii_lowercase();
|
||||
let stem = lower.strip_suffix(".xml")?;
|
||||
let lang = stem.strip_prefix("bdmt_")?;
|
||||
// ISO 639-2 codes are exactly 3 ASCII letters. Be strict — keeps
|
||||
// us from picking up unrelated `bdmt_foo.xml` siblings.
|
||||
if lang.len() != 3 || !lang.chars().all(|c| c.is_ascii_alphabetic()) {
|
||||
return None;
|
||||
}
|
||||
Some(lang.to_string())
|
||||
}
|
||||
|
||||
/// Tuple returned by [`parse_bdmt_xml`]: `(title, description?, disc_set?)`.
|
||||
/// Aliased so the function signature isn't a clippy::type-complexity offender.
|
||||
pub(crate) type BdmtFields = (String, Option<String>, Option<(u32, u32)>);
|
||||
|
||||
/// Parse one `bdmt_<lang>.xml` document and return
|
||||
/// `(title, description?, disc_set?)`. Returns `None` if no title
|
||||
/// could be located — the caller treats this as "skip this file".
|
||||
///
|
||||
/// Title-element preference: `<di:name>` → `<di:title>` →
|
||||
/// `<di:tableOfContents>/<di:titleName>` (first match wins, per the
|
||||
/// authoring-tool conventions documented at the module level).
|
||||
pub(crate) fn parse_bdmt_xml(_lang_code: &str, xml_text: &str) -> Option<BdmtFields> {
|
||||
let title = extract_title(xml_text)?;
|
||||
let description = xml::text(xml_text, "description")
|
||||
.filter(|s| !s.is_empty())
|
||||
.filter(|s| !looks_like_xml(s))
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty());
|
||||
let disc_set = extract_disc_set(xml_text);
|
||||
Some((title, description, disc_set))
|
||||
}
|
||||
|
||||
/// Reject candidate description strings that are themselves XML
|
||||
/// fragments — observed on a captured disc, where
|
||||
/// `<di:description>` contained `<di:thumbnail href="…"/>` child
|
||||
/// elements and no actual prose. Surfacing that raw to the JSON
|
||||
/// output is worse than dropping the field entirely.
|
||||
fn looks_like_xml(s: &str) -> bool {
|
||||
let t = s.trim_start();
|
||||
t.starts_with('<')
|
||||
}
|
||||
|
||||
/// Try title-bearing element variants in priority order. The `xml`
|
||||
/// helpers are case- and namespace-insensitive, so callers pass the
|
||||
/// bare local name (no `di:` prefix).
|
||||
fn extract_title(xml_text: &str) -> Option<String> {
|
||||
// Order matches the module-level convention: <di:name> first
|
||||
// (Paramount-style), then <di:title>, then the nested
|
||||
// tableOfContents/titleName form.
|
||||
for tag in ["name", "title"] {
|
||||
if let Some(s) = xml::text(xml_text, tag) {
|
||||
let trimmed = s.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
// tableOfContents/titleName: search inside the toc block so we
|
||||
// don't accidentally pick a stray <titleName> from elsewhere.
|
||||
if let Some((s, e)) = xml::find_element(xml_text, "tableOfContents", 0) {
|
||||
let block = &xml_text[s..e];
|
||||
if let Some(t) = xml::text(block, "titleName") {
|
||||
let trimmed = t.trim();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Extract `(discNumber, numSets)` if both are present and parse as
|
||||
/// `u32`. Accepts either `<di:numSets>` or `<di:numberOfSets>` for
|
||||
/// the denominator (both forms appear in the wild).
|
||||
fn extract_disc_set(xml_text: &str) -> Option<(u32, u32)> {
|
||||
let n = xml::text(xml_text, "discNumber")?
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.ok()?;
|
||||
let total = xml::text(xml_text, "numSets")
|
||||
.or_else(|| xml::text(xml_text, "numberOfSets"))?
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.ok()?;
|
||||
Some((n, total))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn extract_simple_title() {
|
||||
// Minimal Paramount-style document: <di:name> as the title
|
||||
// carrier inside a <discInfo> root.
|
||||
let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Aurora Drift</di:name>
|
||||
</discInfo>"#;
|
||||
let (title, desc, set) = parse_bdmt_xml("eng", xml).expect("title should parse");
|
||||
assert_eq!(title, "Aurora Drift");
|
||||
assert_eq!(desc, None);
|
||||
assert_eq!(set, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_title_element_variant() {
|
||||
// <di:title> is the alternate carrier; should be picked up
|
||||
// when <di:name> is absent.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:title>Echo Chamber</di:title>
|
||||
<di:description>A film about machines.</di:description>
|
||||
</discInfo>"#;
|
||||
let (title, desc, _) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(title, "Echo Chamber");
|
||||
assert_eq!(desc.as_deref(), Some("A film about machines."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_title_from_table_of_contents_fallback() {
|
||||
// Some authoring tools nest the title under tableOfContents.
|
||||
// No <di:name> or <di:title> at top level → fall back to
|
||||
// titleName inside tableOfContents.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:tableOfContents>
|
||||
<di:titleName>Feelings Two</di:titleName>
|
||||
</di:tableOfContents>
|
||||
</discInfo>"#;
|
||||
let (title, _, _) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(title, "Feelings Two");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_box_set_position() {
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Box Set Disc 2</di:name>
|
||||
<di:discNumber>2</di:discNumber>
|
||||
<di:numSets>5</di:numSets>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(set, Some((2, 5)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_box_set_position_alternate_total_tag() {
|
||||
// <di:numberOfSets> is an alternate spelling we've seen.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>X</di:name>
|
||||
<di:discNumber>3</di:discNumber>
|
||||
<di:numberOfSets>6</di:numberOfSets>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(set, Some((3, 6)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_box_set_requires_both_fields() {
|
||||
// discNumber alone (no total) yields None — we don't fabricate
|
||||
// a denominator.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>X</di:name>
|
||||
<di:discNumber>1</di:discNumber>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(set, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_languages_keyed_correctly() {
|
||||
// Simulate driving parse_bdmt_xml from two synthetic XML
|
||||
// blobs and aggregating into DiscMetadata the same way parse()
|
||||
// would. This exercises the BTreeMap key handling without
|
||||
// needing a UdfFs.
|
||||
let eng_xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Aurora Drift</di:name>
|
||||
</discInfo>"#;
|
||||
let fra_xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Aurora Drift (Partie Deux)</di:name>
|
||||
<di:description>Suite du film fictif.</di:description>
|
||||
</discInfo>"#;
|
||||
|
||||
let mut meta = DiscMetadata::default();
|
||||
for (lang, blob) in [("eng", eng_xml), ("fra", fra_xml)] {
|
||||
let (title, desc, ds) = parse_bdmt_xml(lang, blob).unwrap();
|
||||
meta.titles.insert(lang.to_string(), title);
|
||||
if let Some(d) = desc {
|
||||
meta.descriptions.insert(lang.to_string(), d);
|
||||
}
|
||||
if meta.disc_number.is_none() {
|
||||
if let Some(d) = ds {
|
||||
meta.disc_number = Some(d);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
meta.titles.get("eng").map(String::as_str),
|
||||
Some("Aurora Drift")
|
||||
);
|
||||
assert_eq!(
|
||||
meta.titles.get("fra").map(String::as_str),
|
||||
Some("Aurora Drift (Partie Deux)")
|
||||
);
|
||||
assert!(meta.descriptions.get("eng").is_none());
|
||||
assert_eq!(
|
||||
meta.descriptions.get("fra").map(String::as_str),
|
||||
Some("Suite du film fictif.")
|
||||
);
|
||||
assert_eq!(meta.disc_number, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_xml_returns_none() {
|
||||
// Random gibberish has no recognizable title element. We
|
||||
// document the contract: parse_bdmt_xml returns None, and
|
||||
// parse() (the caller) skips the file. Aggregating across
|
||||
// zero files leaves DiscMetadata::default() — which parse()
|
||||
// surfaces as None to its caller. Either is documented as
|
||||
// acceptable per the module spec.
|
||||
let bad = "this is not xml &&& <<< nope";
|
||||
assert!(parse_bdmt_xml("eng", bad).is_none());
|
||||
|
||||
// Half-open tag, no body, no close: also yields no title.
|
||||
let truncated = "<discInfo><di:name>";
|
||||
assert!(parse_bdmt_xml("eng", truncated).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn description_with_only_child_xml_is_dropped() {
|
||||
// Real-world bug from a captured disc (2026-05-11
|
||||
// capture): <di:description> contained only <di:thumbnail/>
|
||||
// child elements with no actual prose. The previous parser
|
||||
// surfaced the raw XML fragment as the description string.
|
||||
// Now we reject candidates that begin with `<`.
|
||||
let xml = r#"<discInfo>
|
||||
<di:name>Skyline Run</di:name>
|
||||
<di:description>
|
||||
<di:thumbnail href="sample_meta_sm.jpg" />
|
||||
<di:thumbnail href="sample_meta_lg.jpg" />
|
||||
</di:description>
|
||||
</discInfo>"#;
|
||||
let (title, description, _) =
|
||||
parse_bdmt_xml("eng", xml).expect("title is present so parse must succeed");
|
||||
assert_eq!(title, "Skyline Run");
|
||||
assert!(
|
||||
description.is_none(),
|
||||
"description containing only XML children must be dropped, got {description:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn description_with_plain_text_passes_through() {
|
||||
// The legitimate case still works: a description with actual
|
||||
// prose survives the looks_like_xml filter.
|
||||
let xml = r#"<discInfo>
|
||||
<di:name>Some Movie</di:name>
|
||||
<di:description>An epic tale of one man's quest for tea.</di:description>
|
||||
</discInfo>"#;
|
||||
let (_, description, _) = parse_bdmt_xml("eng", xml).expect("must parse");
|
||||
assert_eq!(
|
||||
description.as_deref(),
|
||||
Some("An epic tale of one man's quest for tea.")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whitespace_in_title_is_trimmed() {
|
||||
let xml = r#"<discInfo><di:name>
|
||||
Aurora Drift
|
||||
</di:name></discInfo>"#;
|
||||
let (title, _, _) = parse_bdmt_xml("eng", xml).unwrap();
|
||||
assert_eq!(title, "Aurora Drift");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_code_extraction() {
|
||||
assert_eq!(lang_code_from_filename("bdmt_eng.xml"), Some("eng".into()));
|
||||
assert_eq!(lang_code_from_filename("BDMT_FRA.XML"), Some("fra".into()));
|
||||
assert_eq!(lang_code_from_filename("bdmt_jpn.xml"), Some("jpn".into()));
|
||||
// Non-matching cases:
|
||||
assert_eq!(lang_code_from_filename("bdmt_.xml"), None);
|
||||
assert_eq!(lang_code_from_filename("bdmt_engl.xml"), None);
|
||||
assert_eq!(lang_code_from_filename("bdmt_e1g.xml"), None);
|
||||
assert_eq!(lang_code_from_filename("bdmt_eng.txt"), None);
|
||||
assert_eq!(lang_code_from_filename("foo.xml"), None);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,291 @@
|
||||
//! CLPI vs MPLS cross-validation diagnostic.
|
||||
//!
|
||||
//! Empirical question (raised 2026-05-11): is CLPI's per-stream
|
||||
//! language and codec data truly redundant with MPLS's STN-table data
|
||||
//! on real-world Blu-rays?
|
||||
//!
|
||||
//! Build a quick audit that walks both sources, normalizes their stream
|
||||
//! lists by (PID, language, coding_type), and flags any disagreement.
|
||||
//!
|
||||
//! Three classes of mismatch we want to detect:
|
||||
//!
|
||||
//! 1. **CLPI has streams MPLS doesn't reference.** Orphan streams in
|
||||
//! the .m2ts that no playlist's STN table includes. Means the user
|
||||
//! can't reach them through the menu but they're physically on the
|
||||
//! disc.
|
||||
//! 2. **MPLS has streams CLPI doesn't list.** Should never happen if
|
||||
//! both parsers are correct — playlists reference clips which
|
||||
//! reference streams. If it happens, one of our parsers has a bug.
|
||||
//! 3. **Same PID, different language / coding_type.** The playlist re-
|
||||
//! tagged a stream's metadata. Rare but spec-permitted. Means CLPI
|
||||
//! and MPLS disagree about the same physical stream's properties.
|
||||
//!
|
||||
//! If audits across the corpus show zero mismatches of any class, CLPI
|
||||
//! program_info extraction is **empirically redundant** for labels and
|
||||
//! we can leave it out of the registry. If even one mismatch surfaces,
|
||||
//! we add a CLPI label parser to the registry as belt-and-suspenders.
|
||||
//!
|
||||
//! This module exposes `audit(reader, udf)` returning a structured
|
||||
//! report. Surfaced via the labels-analyze tool — not part of the
|
||||
//! `analyze()` pipeline (no impact on the label output).
|
||||
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// One row in the audit: a stream PID that's known to one source or
|
||||
/// both, with the fields each source reported.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ClpiVsMplsRow {
|
||||
pub pid: u16,
|
||||
pub clpi_coding_type: Option<u8>,
|
||||
pub clpi_language: Option<String>,
|
||||
pub mpls_coding_type: Option<u8>,
|
||||
pub mpls_language: Option<String>,
|
||||
}
|
||||
|
||||
impl ClpiVsMplsRow {
|
||||
/// Three rules for classification:
|
||||
/// - both sources missing (impossible — caller wouldn't insert)
|
||||
/// - one source missing → class A or B (orphan-on-disc / playlist-only)
|
||||
/// - both present but fields differ → class C (metadata divergence)
|
||||
/// - both present and identical → no mismatch
|
||||
pub fn class(&self) -> ClpiVsMplsClass {
|
||||
match (
|
||||
self.clpi_coding_type.is_some(),
|
||||
self.mpls_coding_type.is_some(),
|
||||
) {
|
||||
(true, false) => ClpiVsMplsClass::ClpiOnly,
|
||||
(false, true) => ClpiVsMplsClass::MplsOnly,
|
||||
(true, true) => {
|
||||
let coding_match = self.clpi_coding_type == self.mpls_coding_type;
|
||||
let lang_match = self.clpi_language == self.mpls_language;
|
||||
if coding_match && lang_match {
|
||||
ClpiVsMplsClass::Match
|
||||
} else {
|
||||
ClpiVsMplsClass::Divergent
|
||||
}
|
||||
}
|
||||
(false, false) => ClpiVsMplsClass::Match,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Classification of one (PID) row.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ClpiVsMplsClass {
|
||||
/// PID seen in CLPI ProgramInfo but no MPLS STN table references
|
||||
/// it. Orphan on disc.
|
||||
ClpiOnly,
|
||||
/// PID seen in MPLS STN table but no CLPI ProgramInfo includes it.
|
||||
/// One of our parsers probably has a bug.
|
||||
MplsOnly,
|
||||
/// Both sources see this PID with the same coding_type + language.
|
||||
Match,
|
||||
/// Both sources see this PID but disagree on coding_type or language.
|
||||
/// MPLS wins for label rendering (playlist-authoritative view); CLPI
|
||||
/// is the per-clip ground truth.
|
||||
Divergent,
|
||||
}
|
||||
|
||||
/// Full audit report.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ClpiVsMplsAudit {
|
||||
pub rows: Vec<ClpiVsMplsRow>,
|
||||
}
|
||||
|
||||
impl ClpiVsMplsAudit {
|
||||
pub fn class_counts(&self) -> (usize, usize, usize, usize) {
|
||||
let mut clpi_only = 0;
|
||||
let mut mpls_only = 0;
|
||||
let mut matches = 0;
|
||||
let mut divergent = 0;
|
||||
for r in &self.rows {
|
||||
match r.class() {
|
||||
ClpiVsMplsClass::ClpiOnly => clpi_only += 1,
|
||||
ClpiVsMplsClass::MplsOnly => mpls_only += 1,
|
||||
ClpiVsMplsClass::Match => matches += 1,
|
||||
ClpiVsMplsClass::Divergent => divergent += 1,
|
||||
}
|
||||
}
|
||||
(clpi_only, mpls_only, matches, divergent)
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk `/BDMV/CLIPINF/*.clpi` and `/BDMV/PLAYLIST/*.mpls`, build a
|
||||
/// dedup-by-PID table of (CLPI fields, MPLS fields), return the
|
||||
/// merged view. Missing files (read errors, parse failures) are
|
||||
/// silently skipped — this is diagnostic, not correctness-critical.
|
||||
pub fn audit(reader: &mut dyn SectorSource, udf: &UdfFs) -> ClpiVsMplsAudit {
|
||||
// Aggregate by PID across all CLPI files. If a PID appears in
|
||||
// multiple clips (typical — main movie clip + trailers reference
|
||||
// the same audio stream PIDs), first encountered wins (they should
|
||||
// all agree per BD spec).
|
||||
let mut clpi_by_pid: BTreeMap<u16, (u8, String)> = BTreeMap::new();
|
||||
if let Some(dir) = udf.find_dir("/BDMV/CLIPINF") {
|
||||
let names: Vec<String> = dir
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && e.name.to_ascii_lowercase().ends_with(".clpi"))
|
||||
.map(|e| e.name.clone())
|
||||
.collect();
|
||||
for name in names {
|
||||
let path = format!("/BDMV/CLIPINF/{}", name);
|
||||
let Ok(data) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(clip) = crate::clpi::parse(&data) else {
|
||||
continue;
|
||||
};
|
||||
for s in clip.streams {
|
||||
clpi_by_pid
|
||||
.entry(s.pid)
|
||||
.or_insert((s.coding_type, s.language));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Same for MPLS streams.
|
||||
let mut mpls_by_pid: BTreeMap<u16, (u8, String)> = BTreeMap::new();
|
||||
if let Some(dir) = udf.find_dir("/BDMV/PLAYLIST") {
|
||||
let names: Vec<String> = dir
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && e.name.to_ascii_lowercase().ends_with(".mpls"))
|
||||
.map(|e| e.name.clone())
|
||||
.collect();
|
||||
for name in names {
|
||||
let path = format!("/BDMV/PLAYLIST/{}", name);
|
||||
let Ok(data) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(pl) = crate::mpls::parse(&data) else {
|
||||
continue;
|
||||
};
|
||||
for s in pl.streams {
|
||||
if s.pid == 0 {
|
||||
// PID 0 means "no PID in stream entry" — skip rather
|
||||
// than collide with other entries.
|
||||
continue;
|
||||
}
|
||||
mpls_by_pid
|
||||
.entry(s.pid)
|
||||
.or_insert((s.coding_type, s.language));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Merge views: every PID seen anywhere gets a row.
|
||||
let mut all_pids: std::collections::BTreeSet<u16> = std::collections::BTreeSet::new();
|
||||
all_pids.extend(clpi_by_pid.keys().copied());
|
||||
all_pids.extend(mpls_by_pid.keys().copied());
|
||||
|
||||
let mut rows = Vec::with_capacity(all_pids.len());
|
||||
for pid in all_pids {
|
||||
let clpi = clpi_by_pid.get(&pid);
|
||||
let mpls = mpls_by_pid.get(&pid);
|
||||
rows.push(ClpiVsMplsRow {
|
||||
pid,
|
||||
clpi_coding_type: clpi.map(|(c, _)| *c),
|
||||
clpi_language: clpi.map(|(_, l)| l.clone()),
|
||||
mpls_coding_type: mpls.map(|(c, _)| *c),
|
||||
mpls_language: mpls.map(|(_, l)| l.clone()),
|
||||
});
|
||||
}
|
||||
|
||||
ClpiVsMplsAudit { rows }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn class_match_when_identical() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("eng".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::Match);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_clpi_only_when_mpls_missing() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: None,
|
||||
mpls_language: None,
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::ClpiOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_mpls_only_when_clpi_missing() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: None,
|
||||
clpi_language: None,
|
||||
mpls_coding_type: Some(0x90),
|
||||
mpls_language: Some("fra".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::MplsOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_divergent_on_lang_disagreement() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("und".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::Divergent);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_counts_sum_rows() {
|
||||
let audit = ClpiVsMplsAudit {
|
||||
rows: vec![
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("eng".into()),
|
||||
}, // Match
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1101,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("fra".into()),
|
||||
mpls_coding_type: None,
|
||||
mpls_language: None,
|
||||
}, // ClpiOnly
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1102,
|
||||
clpi_coding_type: None,
|
||||
clpi_language: None,
|
||||
mpls_coding_type: Some(0x90),
|
||||
mpls_language: Some("eng".into()),
|
||||
}, // MplsOnly
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1103,
|
||||
clpi_coding_type: Some(0x86),
|
||||
clpi_language: Some("spa".into()),
|
||||
mpls_coding_type: Some(0x86),
|
||||
mpls_language: Some("ita".into()),
|
||||
}, // Divergent
|
||||
],
|
||||
};
|
||||
let (co, mo, m, d) = audit.class_counts();
|
||||
assert_eq!(co, 1);
|
||||
assert_eq!(mo, 1);
|
||||
assert_eq!(m, 1);
|
||||
assert_eq!(d, 1);
|
||||
}
|
||||
}
|
||||
+56
-92
@@ -3,8 +3,8 @@
|
||||
//! Clean structured XML with Content/Qualifier per stream and
|
||||
//! stream number mapping via playbackconfig.
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, xml};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -12,7 +12,7 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "streamproperties.xml")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let sp_data = super::read_jar_file(reader, udf, "streamproperties.xml")?;
|
||||
let sp_text = std::str::from_utf8(&sp_data).ok()?;
|
||||
|
||||
@@ -66,7 +66,8 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(labels)
|
||||
// High confidence: streamproperties.xml is fully structured.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
|
||||
struct StreamInfo {
|
||||
@@ -78,105 +79,68 @@ struct StreamInfo {
|
||||
qualifier: LabelQualifier,
|
||||
}
|
||||
|
||||
fn parse_stream_infos(xml: &str) -> Vec<StreamInfo> {
|
||||
fn parse_stream_infos(text: &str) -> Vec<StreamInfo> {
|
||||
let mut infos = Vec::new();
|
||||
let mut pos = 0;
|
||||
|
||||
while pos < xml.len() {
|
||||
let (tag, stream_type) = if let Some(p) = xml[pos..].find("<AudioStreamInfos>") {
|
||||
(p + pos, StreamLabelType::Audio)
|
||||
} else if let Some(p) = xml[pos..].find("<SubtitleStreamInfos>") {
|
||||
(p + pos, StreamLabelType::Subtitle)
|
||||
} else {
|
||||
break;
|
||||
};
|
||||
for (tag_name, stream_type) in [
|
||||
("AudioStreamInfos", StreamLabelType::Audio),
|
||||
("SubtitleStreamInfos", StreamLabelType::Subtitle),
|
||||
] {
|
||||
let mut from = 0;
|
||||
while let Some((start, end)) = xml::find_element(text, tag_name, from) {
|
||||
let block = &text[start..end];
|
||||
let id = xml::text(block, "ID").unwrap_or_default();
|
||||
let lang_id = xml::text(block, "LangInfoID").unwrap_or_default();
|
||||
let content = xml::text(block, "Content").unwrap_or_default();
|
||||
let qualifier_str = xml::text(block, "Qualifier").unwrap_or_default();
|
||||
|
||||
let end_tag = match stream_type {
|
||||
StreamLabelType::Audio => "</AudioStreamInfos>",
|
||||
StreamLabelType::Subtitle => "</SubtitleStreamInfos>",
|
||||
};
|
||||
let (language, variant) = if lang_id.contains('_') {
|
||||
let parts: Vec<&str> = lang_id.splitn(2, '_').collect();
|
||||
(parts[0].to_lowercase(), parts[1].to_string())
|
||||
} else {
|
||||
(lang_id.to_lowercase(), String::new())
|
||||
};
|
||||
|
||||
let block_end = match xml[tag..].find(end_tag) {
|
||||
Some(p) => tag + p + end_tag.len(),
|
||||
None => break,
|
||||
};
|
||||
let purpose = if content.eq_ignore_ascii_case("COMMENTARY") {
|
||||
LabelPurpose::Commentary
|
||||
} else {
|
||||
LabelPurpose::Normal
|
||||
};
|
||||
|
||||
let block = &xml[tag..block_end];
|
||||
let id = extract_tag(block, "ID").unwrap_or_default();
|
||||
let lang_id = extract_tag(block, "LangInfoID").unwrap_or_default();
|
||||
let content = extract_tag(block, "Content").unwrap_or_default();
|
||||
let qualifier_str = extract_tag(block, "Qualifier").unwrap_or_default();
|
||||
let qualifier = match qualifier_str.to_ascii_uppercase().as_str() {
|
||||
"SDH" => LabelQualifier::Sdh,
|
||||
"DS" => LabelQualifier::DescriptiveService,
|
||||
_ => LabelQualifier::None,
|
||||
};
|
||||
|
||||
let (language, variant) = if lang_id.contains('_') {
|
||||
let parts: Vec<&str> = lang_id.splitn(2, '_').collect();
|
||||
(parts[0].to_lowercase(), parts[1].to_string())
|
||||
} else {
|
||||
(lang_id.to_lowercase(), String::new())
|
||||
};
|
||||
|
||||
let purpose = match content.as_str() {
|
||||
"COMMENTARY" => LabelPurpose::Commentary,
|
||||
_ => LabelPurpose::Normal,
|
||||
};
|
||||
|
||||
let qualifier = match qualifier_str.as_str() {
|
||||
"SDH" => LabelQualifier::Sdh,
|
||||
"DS" => LabelQualifier::DescriptiveService,
|
||||
_ => LabelQualifier::None,
|
||||
};
|
||||
|
||||
infos.push(StreamInfo {
|
||||
id,
|
||||
stream_type,
|
||||
language,
|
||||
variant,
|
||||
purpose,
|
||||
qualifier,
|
||||
});
|
||||
pos = block_end;
|
||||
infos.push(StreamInfo {
|
||||
id,
|
||||
stream_type,
|
||||
language,
|
||||
variant,
|
||||
purpose,
|
||||
qualifier,
|
||||
});
|
||||
from = end;
|
||||
}
|
||||
}
|
||||
infos
|
||||
}
|
||||
|
||||
fn parse_playback_config(xml: &str, map: &mut HashMap<String, u16>) {
|
||||
let mut pos = 0;
|
||||
while pos < xml.len() {
|
||||
let tag_start = if let Some(p) = xml[pos..].find("<AudioStreams>") {
|
||||
Some(p + pos)
|
||||
} else {
|
||||
xml[pos..].find("<SubtitlesStreams>").map(|p| p + pos)
|
||||
};
|
||||
|
||||
let tag_start = match tag_start {
|
||||
Some(p) => p,
|
||||
None => break,
|
||||
};
|
||||
|
||||
let block_end = xml[tag_start..]
|
||||
.find("</AudioStreams>")
|
||||
.or_else(|| xml[tag_start..].find("</SubtitlesStreams>"))
|
||||
.map(|p| tag_start + p + 20)
|
||||
.unwrap_or(xml.len());
|
||||
|
||||
let block = &xml[tag_start..block_end];
|
||||
|
||||
if let (Some(stream_id_str), Some(info_id)) = (
|
||||
extract_tag(block, "StreamID"),
|
||||
extract_tag(block, "StreamInfo_ID"),
|
||||
) {
|
||||
if let Ok(stream_num) = stream_id_str.parse::<u16>() {
|
||||
map.insert(info_id, stream_num);
|
||||
fn parse_playback_config(text: &str, map: &mut HashMap<String, u16>) {
|
||||
for tag_name in ["AudioStreams", "SubtitlesStreams"] {
|
||||
let mut from = 0;
|
||||
while let Some((start, end)) = xml::find_element(text, tag_name, from) {
|
||||
let block = &text[start..end];
|
||||
if let (Some(stream_id_str), Some(info_id)) = (
|
||||
xml::text(block, "StreamID"),
|
||||
xml::text(block, "StreamInfo_ID"),
|
||||
) {
|
||||
if let Ok(stream_num) = stream_id_str.parse::<u16>() {
|
||||
map.insert(info_id, stream_num);
|
||||
}
|
||||
}
|
||||
from = end;
|
||||
}
|
||||
|
||||
pos = block_end;
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_tag(xml: &str, tag: &str) -> Option<String> {
|
||||
let open = format!("<{tag}>");
|
||||
let close = format!("</{tag}>");
|
||||
let start = xml.find(&open)? + open.len();
|
||||
let end = xml[start..].find(&close)? + start;
|
||||
Some(xml[start..end].trim().to_string())
|
||||
}
|
||||
|
||||
+185
-17
@@ -4,8 +4,8 @@
|
||||
//! When both exist, language_streams.txt provides structured types while
|
||||
//! menu_base.prop provides stream number → button name mapping.
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType, vocab};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, vocab};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -14,7 +14,7 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
|| super::jar_file_exists(udf, "language_streams.txt")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
// Try language_streams.txt first (richer structured data)
|
||||
let ls_labels = parse_language_streams(reader, udf);
|
||||
|
||||
@@ -22,12 +22,18 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
let mb_labels = parse_menu_base(reader, udf);
|
||||
|
||||
// If we have both, merge: language_streams for structure, menu_base for names
|
||||
match (ls_labels, mb_labels) {
|
||||
(Some(ls), Some(mb)) => Some(merge(ls, mb)),
|
||||
(Some(ls), None) => Some(ls),
|
||||
(None, Some(mb)) => Some(mb),
|
||||
(None, None) => None,
|
||||
let labels = match (ls_labels, mb_labels) {
|
||||
(Some(ls), Some(mb)) => merge(ls, mb),
|
||||
(Some(ls), None) => ls,
|
||||
(None, Some(mb)) => mb,
|
||||
(None, None) => return None,
|
||||
};
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// High confidence: both language_streams.txt and menu_base.prop
|
||||
// are structured key-value formats with documented types.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
|
||||
fn merge(ls: Vec<StreamLabel>, mb: Vec<StreamLabel>) -> Vec<StreamLabel> {
|
||||
@@ -49,7 +55,7 @@ fn merge(ls: Vec<StreamLabel>, mb: Vec<StreamLabel>) -> Vec<StreamLabel> {
|
||||
|
||||
// ── language_streams.txt parser ────────────────────────────────────────────
|
||||
|
||||
fn parse_language_streams(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
fn parse_language_streams(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
let data = super::read_jar_file(reader, udf, "language_streams.txt")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
@@ -172,9 +178,167 @@ fn parse_language_streams(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<
|
||||
Some(labels)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Build a minimal menu_base.prop text and run `parse_menu_base`'s
|
||||
/// inner logic via a temporary closure. This isolates the prop
|
||||
/// parsing without needing a SectorSource.
|
||||
fn parse_props(text: &str) -> Vec<StreamLabel> {
|
||||
// Mirror the inner loop of parse_menu_base exactly. Kept
|
||||
// separate so the test doesn't need disc fixtures.
|
||||
use std::collections::HashMap;
|
||||
let mut entries: HashMap<String, HashMap<String, String>> = HashMap::new();
|
||||
for line in text.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let Some(eq_pos) = line.find('=') else {
|
||||
continue;
|
||||
};
|
||||
let full_key = &line[..eq_pos];
|
||||
let value = &line[eq_pos + 1..];
|
||||
if let Some(dot_pos) = full_key.rfind('.') {
|
||||
entries
|
||||
.entry(full_key[..dot_pos].to_string())
|
||||
.or_default()
|
||||
.insert(full_key[dot_pos + 1..].to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
let mut labels = Vec::new();
|
||||
for (prefix, props) in &entries {
|
||||
let is_audio = props
|
||||
.get("class")
|
||||
.is_some_and(|c| c.contains("AudioButton"))
|
||||
|| prefix.starts_with("audio_");
|
||||
let is_subtitle = props
|
||||
.get("class")
|
||||
.is_some_and(|c| c.contains("SubtitleButton"))
|
||||
|| prefix.starts_with("subtitle_");
|
||||
let stream_num_str = props
|
||||
.get("streamNumber")
|
||||
.or_else(|| props.get("audioStream"))
|
||||
.or_else(|| props.get("subtitleStream"));
|
||||
let stream_num: u16 = match stream_num_str.and_then(|s| s.parse().ok()) {
|
||||
Some(n) if n > 0 => n,
|
||||
_ => continue,
|
||||
};
|
||||
if !is_audio && !is_subtitle {
|
||||
continue;
|
||||
}
|
||||
let name = props.get("name").cloned().unwrap_or_default();
|
||||
let purpose = match vocab::purpose(&name) {
|
||||
LabelPurpose::Normal if prefix.contains("comm") => LabelPurpose::Commentary,
|
||||
p => p,
|
||||
};
|
||||
let qualifier = if is_subtitle {
|
||||
vocab::qualifier(&name)
|
||||
} else {
|
||||
LabelQualifier::None
|
||||
};
|
||||
let stream_type = if is_audio {
|
||||
StreamLabelType::Audio
|
||||
} else {
|
||||
StreamLabelType::Subtitle
|
||||
};
|
||||
let language = props
|
||||
.get("audioLanguage")
|
||||
.or_else(|| props.get("subtitleLanguage"))
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
labels.push(StreamLabel {
|
||||
stream_number: stream_num,
|
||||
stream_type,
|
||||
language,
|
||||
name,
|
||||
purpose,
|
||||
qualifier,
|
||||
codec_hint: String::new(),
|
||||
variant: String::new(),
|
||||
});
|
||||
}
|
||||
labels.sort_by_key(|l| (l.stream_type as u8, l.stream_number));
|
||||
labels
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commentary_via_name() {
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=2\n\
|
||||
audio_1.name=Director's Commentary\n\
|
||||
audio_1.audioLanguage=eng\n",
|
||||
);
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Commentary);
|
||||
assert_eq!(labels[0].language, "eng");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commentary_via_prefix_when_name_silent() {
|
||||
let labels = parse_props(
|
||||
"audio_commentary_1.class=AudioButton\n\
|
||||
audio_commentary_1.streamNumber=2\n\
|
||||
audio_commentary_1.name=Track 2\n\
|
||||
audio_commentary_1.audioLanguage=eng\n",
|
||||
);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commenter_does_not_false_match_commentary() {
|
||||
// Regression for the pre-refactor `name.contains("comment")`
|
||||
// bug: this would wrongly classify a "Commenter Pro" track as
|
||||
// Commentary. vocab::purpose enforces a word boundary.
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=2\n\
|
||||
audio_1.name=Commenter Pro Track\n\
|
||||
audio_1.audioLanguage=eng\n",
|
||||
);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptive_via_name() {
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=3\n\
|
||||
audio_1.name=English Descriptive Audio\n",
|
||||
);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sdh_only_on_subtitles() {
|
||||
// SDH applied to a subtitle stream.
|
||||
let labels = parse_props(
|
||||
"subtitle_1.class=SubtitleButton\n\
|
||||
subtitle_1.streamNumber=4\n\
|
||||
subtitle_1.name=English SDH\n",
|
||||
);
|
||||
assert_eq!(labels[0].qualifier, LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sdh_not_applied_to_audio_stream_even_if_name_contains_sdh() {
|
||||
// Audio streams should not pick up SDH (it's a subtitle
|
||||
// concept). Edge case: badly-authored name happens to include
|
||||
// "SDH" — we don't propagate it to audio metadata.
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=5\n\
|
||||
audio_1.name=English SDH (track?)\n",
|
||||
);
|
||||
assert_eq!(labels[0].qualifier, LabelQualifier::None);
|
||||
}
|
||||
}
|
||||
|
||||
// ── menu_base.prop parser ──────────────────────────────────────────────────
|
||||
|
||||
fn parse_menu_base(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
fn parse_menu_base(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
let data = super::read_jar_file(reader, udf, "menu_base.prop")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
@@ -231,16 +395,20 @@ fn parse_menu_base(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<Str
|
||||
}
|
||||
|
||||
let name = props.get("name").cloned().unwrap_or_default();
|
||||
let name_lower = name.to_lowercase();
|
||||
|
||||
let purpose = if name_lower.contains("comment") || prefix.contains("comm") {
|
||||
LabelPurpose::Commentary
|
||||
} else {
|
||||
LabelPurpose::Normal
|
||||
// Purpose: ask vocab first (word-boundary matched — avoids the
|
||||
// "Commenter" false positive the prior `name.contains("comment")`
|
||||
// had). Then fall back to the structural prefix check
|
||||
// (`audio_commentary.foo`-style keys group commentary streams
|
||||
// regardless of display name).
|
||||
let purpose = match vocab::purpose(&name) {
|
||||
LabelPurpose::Normal if prefix.contains("comm") => LabelPurpose::Commentary,
|
||||
p => p,
|
||||
};
|
||||
|
||||
let qualifier = if is_subtitle && name_lower.contains("sdh") {
|
||||
LabelQualifier::Sdh
|
||||
// Qualifier: only apply to subtitles (SDH is a subtitle concept).
|
||||
let qualifier = if is_subtitle {
|
||||
vocab::qualifier(&name)
|
||||
} else {
|
||||
LabelQualifier::None
|
||||
};
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
//! "dbp" framework — Magnolia Pictures BD-J authoring shop (per
|
||||
//! `bd-live.magpictures.com` referenced in the disc's
|
||||
//! `com/dbp/bluray.MenuXlet.perm`). Detected on UHD discs whose
|
||||
//! `/BDMV/JAR/<x>.jar` (top-level, not in a subdir) contains
|
||||
//! `com/dbp/` package paths.
|
||||
//!
|
||||
//! Stream labels live as plain ASCII strings inside compiled `.class`
|
||||
//! files in the jar — a quirk of the menu-rendering layer encoding
|
||||
//! its TextField positions and content as constant strings the
|
||||
//! Java compiler retained in the class string pool. Observed format:
|
||||
//!
|
||||
//! ```text
|
||||
//! LTextField,Audio1,English Dolby Atmos,Fontstrip_Composite,...
|
||||
//! RTextField,Audio2,English Descriptive Audio,Fontstrip_Composite,...
|
||||
//! HTextField,Subtitle1,English SDH,Fontstrip_Composite,...
|
||||
//! ATextField,Subtitle0,None,Fontstrip_Composite,...
|
||||
//! ```
|
||||
//!
|
||||
//! The single uppercase letter before `TextField` is string-pool
|
||||
//! prefix noise — the parser anchors on `TextField,` regardless of
|
||||
//! what precedes it. `Subtitle0` is the disable-subtitles menu
|
||||
//! button and is skipped (not a real subtitle stream).
|
||||
//!
|
||||
//! ## Implementation
|
||||
//!
|
||||
//! v2 (2026-05-10): rewritten on top of [`super::class_reader`] —
|
||||
//! iterates `CpInfo::Utf8` constant-pool entries instead of raw byte
|
||||
//! scanning each class file. Equivalent label coverage (the literal
|
||||
//! `TextField,...` strings live in the CP as Utf8 entries), but
|
||||
//! structurally cleaner: no false-positive risk from method bytecode
|
||||
//! or attribute names happening to contain `TextField,`. Language /
|
||||
//! purpose / qualifier classification moved to [`super::vocab`] so all
|
||||
//! Java-parser families share one source of truth.
|
||||
|
||||
use super::class_reader::CpInfo;
|
||||
use super::{ParseResult, StreamLabel, StreamLabelType, jar, vocab};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// dbp detect can't peek inside a jar without a SectorSource (the
|
||||
/// trait function only takes `&UdfFs`), so we trigger on the cheap
|
||||
/// signal "any top-level .jar in /BDMV/JAR/." That fires on every
|
||||
/// BD-J disc, but parse() does the real `com/dbp/` check and
|
||||
/// returns None on a mismatch — so this parser only ever consumes
|
||||
/// time on discs that fell through every earlier parser.
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
jar::has_any_top_level_jar(udf)
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
jar::for_each_jar(reader, udf, |_entry_name, archive| {
|
||||
if !jar::has_path_prefix(archive, "com/dbp/") {
|
||||
return None;
|
||||
}
|
||||
let labels = scan_jar(archive);
|
||||
if labels.is_empty() {
|
||||
None
|
||||
} else {
|
||||
// High confidence: TextField,Audio1,... is a stable anchor
|
||||
// pattern + vocab routes language/purpose/qualifier.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn scan_jar(archive: &mut jar::Jar) -> Vec<StreamLabel> {
|
||||
// BTreeMap so we keep the highest-numbered (last-written) label
|
||||
// for each stream slot deterministic across runs. The same
|
||||
// TextField,Audio1,... string can appear in multiple classes
|
||||
// (button-state variants, localization fallbacks). Last write
|
||||
// wins — they should all agree, but the structure is defensive.
|
||||
let mut audios: BTreeMap<u16, String> = BTreeMap::new();
|
||||
let mut subs: BTreeMap<u16, String> = BTreeMap::new();
|
||||
|
||||
jar::for_each_class(archive, |_class_name, class| {
|
||||
for (_idx, cp) in class.constant_pool.iter() {
|
||||
if let CpInfo::Utf8(s) = cp {
|
||||
collect_textfield(s, &mut audios, &mut subs);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut out = Vec::new();
|
||||
for (num, label) in audios {
|
||||
out.push(make_label(num, label, StreamLabelType::Audio));
|
||||
}
|
||||
for (num, label) in subs {
|
||||
out.push(make_label(num, label, StreamLabelType::Subtitle));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn collect_textfield(
|
||||
s: &str,
|
||||
audios: &mut BTreeMap<u16, String>,
|
||||
subs: &mut BTreeMap<u16, String>,
|
||||
) {
|
||||
// Anchor on "TextField," — the prefix character before it varies
|
||||
// (string-pool ordering inside compiled Java) and is irrelevant.
|
||||
let Some(idx) = s.find("TextField,") else {
|
||||
return;
|
||||
};
|
||||
let after = &s[idx + "TextField,".len()..];
|
||||
let mut parts = after.splitn(3, ',');
|
||||
let kind_n = parts.next().unwrap_or("").trim();
|
||||
let label = parts.next().unwrap_or("").trim();
|
||||
if label.is_empty() {
|
||||
return;
|
||||
}
|
||||
if let Some(rest) = kind_n.strip_prefix("Audio") {
|
||||
if let Ok(n) = rest.parse::<u16>() {
|
||||
audios.insert(n, label.to_string());
|
||||
}
|
||||
} else if let Some(rest) = kind_n.strip_prefix("Subtitle") {
|
||||
if let Ok(n) = rest.parse::<u16>() {
|
||||
// Subtitle0 is conventionally the "None / Off" disable
|
||||
// button, not an actual subtitle stream.
|
||||
if n > 0 {
|
||||
subs.insert(n, label.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn make_label(num: u16, label: String, stream_type: StreamLabelType) -> StreamLabel {
|
||||
let lang_info = vocab::lang(&label);
|
||||
let language = lang_info.map(|l| l.code).unwrap_or("").to_string();
|
||||
let variant = lang_info.map(|l| l.variant).unwrap_or("").to_string();
|
||||
let qualifier = vocab::qualifier(&label);
|
||||
let purpose = vocab::purpose(&label);
|
||||
StreamLabel {
|
||||
stream_number: num,
|
||||
stream_type,
|
||||
language,
|
||||
name: label,
|
||||
purpose,
|
||||
qualifier,
|
||||
codec_hint: String::new(),
|
||||
variant,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::{LabelPurpose, LabelQualifier};
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn collect_extracts_audio_and_subtitle_indices() {
|
||||
let mut audios = BTreeMap::new();
|
||||
let mut subs = BTreeMap::new();
|
||||
let lines = [
|
||||
"LTextField,Audio1,English Dolby Atmos,Fontstrip_Composite,296,763,275,25,left",
|
||||
"RTextField,Audio2,English Descriptive Audio,Fontstrip_Composite,296,803,275,25,left",
|
||||
"RTextField,Audio3,Spanish 5.1 Dolby Digital,Fontstrip_Composite,296,843,275,25,left",
|
||||
"ATextField,Subtitle0,None,Fontstrip_Composite,1312,843,275,25,left",
|
||||
"HTextField,Subtitle1,English SDH,Fontstrip_Composite,1312,763,275,25,left",
|
||||
"DTextField,Subtitle2,Spanish,Fontstrip_Composite,1312,803,275,25,left",
|
||||
];
|
||||
for s in &lines {
|
||||
collect_textfield(s, &mut audios, &mut subs);
|
||||
}
|
||||
assert_eq!(audios.len(), 3);
|
||||
assert_eq!(audios[&1], "English Dolby Atmos");
|
||||
assert_eq!(audios[&2], "English Descriptive Audio");
|
||||
assert_eq!(audios[&3], "Spanish 5.1 Dolby Digital");
|
||||
// Subtitle0 ("None") is skipped — disable button, not a stream.
|
||||
assert_eq!(subs.len(), 2);
|
||||
assert_eq!(subs[&1], "English SDH");
|
||||
assert_eq!(subs[&2], "Spanish");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_ignores_non_textfield_strings() {
|
||||
let mut audios = BTreeMap::new();
|
||||
let mut subs = BTreeMap::new();
|
||||
for s in [
|
||||
"GraphicButton,SU_Audio",
|
||||
"AudioMenu",
|
||||
"CommentaryMenuAlternateScenes",
|
||||
"PrimaryAudioControl",
|
||||
] {
|
||||
collect_textfield(s, &mut audios, &mut subs);
|
||||
}
|
||||
assert!(audios.is_empty());
|
||||
assert!(subs.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_routes_via_vocab() {
|
||||
let l = make_label(1, "English SDH".to_string(), StreamLabelType::Subtitle);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.qualifier, LabelQualifier::Sdh);
|
||||
assert_eq!(l.purpose, LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_descriptive_audio() {
|
||||
let l = make_label(
|
||||
2,
|
||||
"English Descriptive Audio".to_string(),
|
||||
StreamLabelType::Audio,
|
||||
);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.purpose, LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_commentary() {
|
||||
let l = make_label(
|
||||
3,
|
||||
"English Director's Commentary".to_string(),
|
||||
StreamLabelType::Audio,
|
||||
);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_compound_languages_populate_variant() {
|
||||
let brazilian = make_label(1, "Brazilian Portuguese 5.1".into(), StreamLabelType::Audio);
|
||||
assert_eq!(brazilian.language, "por");
|
||||
assert_eq!(brazilian.variant, "Brazilian");
|
||||
|
||||
let castilian = make_label(1, "Castilian Spanish".into(), StreamLabelType::Audio);
|
||||
assert_eq!(castilian.language, "spa");
|
||||
assert_eq!(castilian.variant, "Castilian");
|
||||
|
||||
let canadian = make_label(
|
||||
1,
|
||||
"Canadian French Dolby Digital".into(),
|
||||
StreamLabelType::Audio,
|
||||
);
|
||||
assert_eq!(canadian.language, "fra");
|
||||
assert_eq!(canadian.variant, "Canadian");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_bare_language_has_empty_variant() {
|
||||
let l = make_label(1, "English Dolby Atmos".into(), StreamLabelType::Audio);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.variant, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_unknown_language_is_empty() {
|
||||
// vocab::lang returns None — make_label converts both fields to "".
|
||||
let l = make_label(1, "Klingon Dolby Atmos".into(), StreamLabelType::Audio);
|
||||
assert_eq!(l.language, "");
|
||||
assert_eq!(l.variant, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn make_label_rnib_descriptive_service() {
|
||||
let l = make_label(1, "English RNIB".into(), StreamLabelType::Subtitle);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.qualifier, LabelQualifier::DescriptiveService);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,151 @@
|
||||
//! BD-J jar utilities — common scaffolding for parsers that read
|
||||
//! `/BDMV/JAR/*.jar`.
|
||||
//!
|
||||
//! Composes with [`class_reader`](super::class_reader) for structured
|
||||
//! `.class` access. Used by `dbp` (string-pool scan via constant pool)
|
||||
//! and `deluxe` (bytecode pattern matching) — those parsers express
|
||||
//! "open every top-level jar, look at every .class inside" without
|
||||
//! repeating the zip-archive boilerplate.
|
||||
|
||||
// `try_each_class` is staged for `labels::deluxe`, which needs the
|
||||
// early-return form to short-circuit class iteration on a match.
|
||||
// dead-code allow comes off when deluxe lands.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use super::class_reader::ClassFile;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::io::Cursor;
|
||||
use zip::ZipArchive;
|
||||
|
||||
/// In-memory zip archive: backed by a `Vec<u8>` read from UDF. Owns
|
||||
/// the buffer; callers pass it to [`has_path_prefix`], [`for_each_class`],
|
||||
/// etc.
|
||||
pub type Jar = ZipArchive<Cursor<Vec<u8>>>;
|
||||
|
||||
/// True if `/BDMV/JAR/` contains at least one top-level `.jar` file
|
||||
/// (not under a subdir). Used by `detect()` in parsers whose real
|
||||
/// signal lives inside a jar — they can't open the jar without a
|
||||
/// `SectorSource`, so they use this cheap pre-check and do the real
|
||||
/// `com/<vendor>/` discriminator in `parse()`.
|
||||
pub fn has_any_top_level_jar(udf: &UdfFs) -> bool {
|
||||
let Some(jar_dir) = udf.find_dir("/BDMV/JAR") else {
|
||||
return false;
|
||||
};
|
||||
jar_dir
|
||||
.entries
|
||||
.iter()
|
||||
.any(|e| !e.is_dir && e.name.to_lowercase().ends_with(".jar"))
|
||||
}
|
||||
|
||||
/// Open every top-level `*.jar` entry in `/BDMV/JAR/` and yield each
|
||||
/// `(entry_name, Jar)` to `f`. Returns the first `Some(R)` the callback
|
||||
/// produces, or `None` if every jar was visited without a hit.
|
||||
///
|
||||
/// "Top-level" means entries directly under `/BDMV/JAR/`, not nested
|
||||
/// under a subdir. (Pixelogic, Criterion, Paramount, etc. put their
|
||||
/// data files inside `/BDMV/JAR/<x>/`; dbp and Deluxe put their jar
|
||||
/// directly at `/BDMV/JAR/<name>.jar`.)
|
||||
///
|
||||
/// Entries that fail to read from UDF or that aren't valid zips are
|
||||
/// silently skipped — same defensive shape as the existing dbp parser.
|
||||
pub fn for_each_jar<R, F>(reader: &mut dyn SectorSource, udf: &UdfFs, mut f: F) -> Option<R>
|
||||
where
|
||||
F: FnMut(&str, &mut Jar) -> Option<R>,
|
||||
{
|
||||
let jar_dir = udf.find_dir("/BDMV/JAR")?;
|
||||
for entry in &jar_dir.entries {
|
||||
if entry.is_dir {
|
||||
continue;
|
||||
}
|
||||
if !entry.name.to_lowercase().ends_with(".jar") {
|
||||
continue;
|
||||
}
|
||||
let path = format!("/BDMV/JAR/{}", entry.name);
|
||||
let Ok(bytes) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(mut archive) = ZipArchive::new(Cursor::new(bytes)) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(r) = f(&entry.name, &mut archive) {
|
||||
return Some(r);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// True if any entry in this jar's central directory starts with
|
||||
/// `prefix`. Fast — only reads filenames, never extracts bytes.
|
||||
///
|
||||
/// Used by parsers as a cheap "is this MY framework's jar?" check
|
||||
/// (e.g. `has_path_prefix(archive, "com/dbp/")` for dbp,
|
||||
/// `has_path_prefix(archive, "com/bydeluxe/")` for Deluxe).
|
||||
pub fn has_path_prefix(archive: &mut Jar, prefix: &str) -> bool {
|
||||
for i in 0..archive.len() {
|
||||
if let Ok(f) = archive.by_index(i) {
|
||||
if f.name().starts_with(prefix) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Iterate every `.class` entry in the jar, parse it with
|
||||
/// [`class_reader`], and call `f` with `(entry_name, &ClassFile)`.
|
||||
///
|
||||
/// Entries that fail to read or parse are silently skipped — this is
|
||||
/// label-extraction code, robustness matters more than completeness.
|
||||
/// Callers that need to know which classes failed should use the
|
||||
/// lower-level [`class_reader`] API directly.
|
||||
pub fn for_each_class<F>(archive: &mut Jar, mut f: F)
|
||||
where
|
||||
F: FnMut(&str, &ClassFile),
|
||||
{
|
||||
for i in 0..archive.len() {
|
||||
let Ok(mut entry) = archive.by_index(i) else {
|
||||
continue;
|
||||
};
|
||||
if !entry.name().ends_with(".class") {
|
||||
continue;
|
||||
}
|
||||
let name = entry.name().to_string();
|
||||
let mut bytes = Vec::with_capacity(entry.size() as usize);
|
||||
if std::io::Read::read_to_end(&mut entry, &mut bytes).is_err() {
|
||||
continue;
|
||||
}
|
||||
let Ok(class) = ClassFile::parse(&bytes) else {
|
||||
continue;
|
||||
};
|
||||
f(&name, &class);
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`for_each_class`] but allows the callback to short-circuit
|
||||
/// iteration. Returns the first `Some(R)` the callback produces.
|
||||
pub fn try_each_class<R, F>(archive: &mut Jar, mut f: F) -> Option<R>
|
||||
where
|
||||
F: FnMut(&str, &ClassFile) -> Option<R>,
|
||||
{
|
||||
for i in 0..archive.len() {
|
||||
let Ok(mut entry) = archive.by_index(i) else {
|
||||
continue;
|
||||
};
|
||||
if !entry.name().ends_with(".class") {
|
||||
continue;
|
||||
}
|
||||
let name = entry.name().to_string();
|
||||
let mut bytes = Vec::with_capacity(entry.size() as usize);
|
||||
if std::io::Read::read_to_end(&mut entry, &mut bytes).is_err() {
|
||||
continue;
|
||||
}
|
||||
let Ok(class) = ClassFile::parse(&bytes) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(r) = f(&name, &class) {
|
||||
return Some(r);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
+1313
-18
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,597 @@
|
||||
//! Universal MPLS-based stream labels.
|
||||
//!
|
||||
//! Unlike the framework-specific parsers in this directory (dbp,
|
||||
//! pixelogic, ctrm, criterion, ...), this module is the *floor*:
|
||||
//! every Blu-ray ships with MPLS playlists under `/BDMV/PLAYLIST/`,
|
||||
//! and every MPLS file has an STN table with per-stream ISO 639-2
|
||||
//! language codes plus coding-type / channel-layout / sample-rate
|
||||
//! bytes from the BD spec.
|
||||
//!
|
||||
//! The framework parsers extract richer editorial labels ("English
|
||||
//! Dolby Atmos", "Director's Commentary") when the disc was authored
|
||||
//! with a recognized tool. When none of them match (e.g. a "no BD-J"
|
||||
//! disc, or an authoring framework we haven't catalogued), MPLS still
|
||||
//! gives us language + codec on every stream — enough to render
|
||||
//! something more useful than the bare PID.
|
||||
//!
|
||||
//! Output confidence is Low: MPLS carries language + codec but
|
||||
//! never purpose/qualifier info (no way to tell "Commentary" from
|
||||
//! "Normal" from the STN table alone). Higher-confidence framework
|
||||
//! parsers, when present, always win on the registry's max-by-confidence
|
||||
//! tiebreaker — MPLS is only chosen when nothing else matched.
|
||||
|
||||
use super::{
|
||||
LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType,
|
||||
vocab::{self, LangInfo},
|
||||
};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
/// True iff `/BDMV/PLAYLIST/` exists and contains at least one
|
||||
/// `.mpls` file. Cheap directory walk only — no sector reads.
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
let Some(dir) = udf.find_dir("/BDMV/PLAYLIST") else {
|
||||
return false;
|
||||
};
|
||||
dir.entries
|
||||
.iter()
|
||||
.any(|e| !e.is_dir && has_mpls_extension(&e.name))
|
||||
}
|
||||
|
||||
/// Walk every `*.mpls` in `/BDMV/PLAYLIST/`, parse it, and convert
|
||||
/// each StreamEntry to a [`StreamLabel`]. Streams shared across
|
||||
/// playlists (same PID) are deduped.
|
||||
///
|
||||
/// Returns `None` if no labels could be produced (e.g. no .mpls files
|
||||
/// parsed successfully, or every parsed stream was a type we skip
|
||||
/// like IG / DV EL).
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let playlist_dir = udf.find_dir("/BDMV/PLAYLIST")?;
|
||||
|
||||
// Collect mpls filenames first so we don't hold a borrow on udf
|
||||
// while we call udf.read_file (which takes &self).
|
||||
let mpls_names: Vec<String> = playlist_dir
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && has_mpls_extension(&e.name))
|
||||
.map(|e| e.name.clone())
|
||||
.collect();
|
||||
|
||||
if mpls_names.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut labels: Vec<StreamLabel> = Vec::new();
|
||||
// (stream_type_tag, language, codec_hint, pid) — PID is the
|
||||
// canonical "same physical stream" key; type+lang+codec round
|
||||
// out the rare case where two distinct logical streams happen
|
||||
// to share a PID across playlists with different metadata.
|
||||
let mut seen: Vec<(u8, String, String, u16)> = Vec::new();
|
||||
|
||||
// Global 1-based counters keyed by StreamLabelType. Incremented
|
||||
// only when an entry survives dedup, so stream_numbers are dense
|
||||
// (1, 2, 3, ...) per type across the whole disc — not reset per
|
||||
// playlist. A disc with 2 MPLS files that each list the same
|
||||
// 8 audio streams ends up with audio_1..audio_8, not audio_1..
|
||||
// audio_16 or audio_1..audio_8 with audio_1 duplicated.
|
||||
let mut audio_idx: u16 = 0;
|
||||
let mut sub_idx: u16 = 0;
|
||||
|
||||
for name in &mpls_names {
|
||||
let path = format!("/BDMV/PLAYLIST/{}", name);
|
||||
let Ok(data) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(playlist) = crate::mpls::parse(&data) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
for entry in &playlist.streams {
|
||||
let label_type = match entry.stream_type {
|
||||
2 | 5 => StreamLabelType::Audio, // primary + secondary audio
|
||||
3 => StreamLabelType::Subtitle, // PG subtitle
|
||||
// 1 = primary video, 6 = secondary video, 7 = DV EL
|
||||
// → no StreamLabelType variant for video, skip.
|
||||
// 4 = IG (interactive graphics) — not a user-facing
|
||||
// stream, skip.
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
let language = normalize_language(&entry.language);
|
||||
let name = language_display_name(&language);
|
||||
let codec_hint = build_codec_hint(label_type, entry);
|
||||
|
||||
let type_tag = type_tag(label_type);
|
||||
let key = (type_tag, language.clone(), codec_hint.clone(), entry.pid);
|
||||
if seen.contains(&key) {
|
||||
continue;
|
||||
}
|
||||
seen.push(key);
|
||||
|
||||
let stream_number = match label_type {
|
||||
StreamLabelType::Audio => {
|
||||
audio_idx += 1;
|
||||
audio_idx
|
||||
}
|
||||
StreamLabelType::Subtitle => {
|
||||
sub_idx += 1;
|
||||
sub_idx
|
||||
}
|
||||
};
|
||||
|
||||
labels.push(StreamLabel {
|
||||
stream_number,
|
||||
stream_type: label_type,
|
||||
language,
|
||||
name,
|
||||
purpose: LabelPurpose::Normal,
|
||||
qualifier: LabelQualifier::None,
|
||||
codec_hint,
|
||||
variant: String::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// MPLS gives language + codec but never editorial info (no
|
||||
// commentary/SDH/director's cut). Low confidence means framework
|
||||
// parsers (paramount, criterion, pixelogic, ctrm, dbp, deluxe) always
|
||||
// win when they match. MPLS only gets chosen as the parser when
|
||||
// nothing else fired — exactly the universal-fallback role we want.
|
||||
Some(ParseResult::low(labels))
|
||||
}
|
||||
|
||||
fn has_mpls_extension(name: &str) -> bool {
|
||||
// Case-insensitive ".mpls" suffix. Some discs use uppercase,
|
||||
// some lowercase; UDF filenames preserve case but we don't.
|
||||
let n = name.len();
|
||||
if n < 5 {
|
||||
return false;
|
||||
}
|
||||
name[n - 5..].eq_ignore_ascii_case(".mpls")
|
||||
}
|
||||
|
||||
/// Lowercase + trim the raw 3-char ISO 639-2 code. If the lowered
|
||||
/// string maps via [`vocab::lang`] (it won't for plain "eng" — that
|
||||
/// matcher is for English-name fragments, not codes) use its
|
||||
/// canonical code; otherwise return the trimmed lowercase string.
|
||||
fn normalize_language(raw: &str) -> String {
|
||||
let trimmed = raw.trim().to_ascii_lowercase();
|
||||
if trimmed.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
// vocab::lang() matches free-form English names, not ISO 639-2
|
||||
// codes — so for the typical MPLS payload ("eng", "fra", ...)
|
||||
// it returns None and we keep the trimmed code.
|
||||
if let Some(LangInfo { code, .. }) = vocab::lang(&trimmed) {
|
||||
return code.to_string();
|
||||
}
|
||||
trimmed
|
||||
}
|
||||
|
||||
/// Human-readable English name for an ISO 639-2 code, or empty if
|
||||
/// the code is unknown. Kept inline rather than in vocab because
|
||||
/// vocab is the *reverse* mapping (name → code).
|
||||
pub(crate) fn language_display_name(iso: &str) -> String {
|
||||
match iso {
|
||||
"eng" => "English",
|
||||
"fra" | "fre" => "French",
|
||||
"spa" => "Spanish",
|
||||
"deu" | "ger" => "German",
|
||||
"ita" => "Italian",
|
||||
"jpn" => "Japanese",
|
||||
"zho" | "chi" => "Chinese",
|
||||
"kor" => "Korean",
|
||||
"por" => "Portuguese",
|
||||
"pol" => "Polish",
|
||||
"ces" | "cze" => "Czech",
|
||||
"hun" => "Hungarian",
|
||||
"nld" | "dut" => "Dutch",
|
||||
"ara" => "Arabic",
|
||||
"hin" => "Hindi",
|
||||
"tur" => "Turkish",
|
||||
"tha" => "Thai",
|
||||
"swe" => "Swedish",
|
||||
"nor" => "Norwegian",
|
||||
"dan" => "Danish",
|
||||
"fin" => "Finnish",
|
||||
"heb" => "Hebrew",
|
||||
"rus" => "Russian",
|
||||
"ell" | "gre" => "Greek",
|
||||
"vie" => "Vietnamese",
|
||||
"ind" => "Indonesian",
|
||||
"msa" | "may" => "Malay",
|
||||
"ukr" => "Ukrainian",
|
||||
"ron" | "rum" => "Romanian",
|
||||
"bul" => "Bulgarian",
|
||||
"hrv" => "Croatian",
|
||||
"srp" => "Serbian",
|
||||
"slk" | "slo" => "Slovak",
|
||||
"slv" => "Slovenian",
|
||||
"est" => "Estonian",
|
||||
"lav" => "Latvian",
|
||||
"lit" => "Lithuanian",
|
||||
"isl" | "ice" => "Icelandic",
|
||||
"eus" | "baq" => "Basque",
|
||||
"cat" => "Catalan",
|
||||
"glg" => "Galician",
|
||||
_ => "",
|
||||
}
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Map BD coding_type byte → codec name. Returns empty for unknown
|
||||
/// bytes (the table covers everything the spec defines, but unknown
|
||||
/// values are still possible on malformed discs).
|
||||
pub(crate) fn codec_name(coding_type: u8) -> &'static str {
|
||||
match coding_type {
|
||||
0x02 => "MPEG-2",
|
||||
0x1B => "H.264",
|
||||
0x24 => "HEVC",
|
||||
0x80 => "LPCM",
|
||||
0x81 => "AC-3",
|
||||
0x82 => "DTS",
|
||||
0x83 => "TrueHD",
|
||||
0x84 => "AC-3+",
|
||||
0x85 => "DTS-HD",
|
||||
0x86 => "DTS-HD MA",
|
||||
0x90 => "PG",
|
||||
0x91 => "IG",
|
||||
0xA1 => "AC-3+ Secondary",
|
||||
0xA2 => "DTS-HD Secondary",
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the final `codec_hint`. For audio streams, optionally
|
||||
/// append " <channels>" and/or " <rate>" suffixes. Sample rate is
|
||||
/// only spelled out for non-48k (the universal default).
|
||||
fn build_codec_hint(label_type: StreamLabelType, entry: &crate::mpls::StreamEntry) -> String {
|
||||
let base = codec_name(entry.coding_type);
|
||||
if base.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
|
||||
if label_type != StreamLabelType::Audio {
|
||||
return base.to_string();
|
||||
}
|
||||
|
||||
let mut out = base.to_string();
|
||||
|
||||
let channels = match entry.audio_format {
|
||||
1 => Some("mono"),
|
||||
3 => Some("2.0"),
|
||||
6 => Some("5.1"),
|
||||
12 => Some("7.1"),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(ch) = channels {
|
||||
out.push(' ');
|
||||
out.push_str(ch);
|
||||
}
|
||||
|
||||
// 1 = 48 kHz (universal default, omit). Only call out higher rates.
|
||||
let rate = match entry.audio_rate {
|
||||
4 => Some("96kHz"),
|
||||
5 => Some("192kHz"),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(r) = rate {
|
||||
out.push(' ');
|
||||
out.push_str(r);
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
/// Dedup-tag for the label type. `u8` instead of `StreamLabelType`
|
||||
/// itself because the enum does not derive `Hash` / `Eq`-by-discriminant
|
||||
/// in a way that we want to couple to (and `==` works fine for the
|
||||
/// linear `Vec::contains` lookup we do).
|
||||
fn type_tag(t: StreamLabelType) -> u8 {
|
||||
match t {
|
||||
StreamLabelType::Audio => 1,
|
||||
StreamLabelType::Subtitle => 2,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::mpls::{Playlist, StreamEntry};
|
||||
|
||||
fn audio_entry(pid: u16, coding: u8, fmt: u8, rate: u8, lang: &str) -> StreamEntry {
|
||||
StreamEntry {
|
||||
stream_type: 2,
|
||||
pid,
|
||||
coding_type: coding,
|
||||
video_format: 0,
|
||||
video_rate: 0,
|
||||
audio_format: fmt,
|
||||
audio_rate: rate,
|
||||
language: lang.to_string(),
|
||||
dynamic_range: 0,
|
||||
color_space: 0,
|
||||
secondary: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn pg_entry(pid: u16, lang: &str) -> StreamEntry {
|
||||
StreamEntry {
|
||||
stream_type: 3,
|
||||
pid,
|
||||
coding_type: 0x90,
|
||||
video_format: 0,
|
||||
video_rate: 0,
|
||||
audio_format: 0,
|
||||
audio_rate: 0,
|
||||
language: lang.to_string(),
|
||||
dynamic_range: 0,
|
||||
color_space: 0,
|
||||
secondary: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn playlist_with(streams: Vec<StreamEntry>) -> Playlist {
|
||||
Playlist {
|
||||
version: "0200".to_string(),
|
||||
play_items: Vec::new(),
|
||||
streams,
|
||||
marks: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Drive the same conversion logic that `parse()` runs on real
|
||||
/// disc data, but starting from already-parsed Playlists so we
|
||||
/// don't have to synthesize valid MPLS bytes.
|
||||
fn labels_from_playlists(playlists: &[Playlist]) -> Vec<StreamLabel> {
|
||||
let mut labels: Vec<StreamLabel> = Vec::new();
|
||||
let mut seen: Vec<(u8, String, String, u16)> = Vec::new();
|
||||
|
||||
for playlist in playlists {
|
||||
let mut audio_idx: u16 = 0;
|
||||
let mut sub_idx: u16 = 0;
|
||||
|
||||
for entry in &playlist.streams {
|
||||
let label_type = match entry.stream_type {
|
||||
2 | 5 => StreamLabelType::Audio,
|
||||
3 => StreamLabelType::Subtitle,
|
||||
_ => continue,
|
||||
};
|
||||
let stream_number = match label_type {
|
||||
StreamLabelType::Audio => {
|
||||
audio_idx += 1;
|
||||
audio_idx
|
||||
}
|
||||
StreamLabelType::Subtitle => {
|
||||
sub_idx += 1;
|
||||
sub_idx
|
||||
}
|
||||
};
|
||||
let language = normalize_language(&entry.language);
|
||||
let name = language_display_name(&language);
|
||||
let codec_hint = build_codec_hint(label_type, entry);
|
||||
let key = (
|
||||
type_tag(label_type),
|
||||
language.clone(),
|
||||
codec_hint.clone(),
|
||||
entry.pid,
|
||||
);
|
||||
if seen.contains(&key) {
|
||||
continue;
|
||||
}
|
||||
seen.push(key);
|
||||
labels.push(StreamLabel {
|
||||
stream_number,
|
||||
stream_type: label_type,
|
||||
language,
|
||||
name,
|
||||
purpose: LabelPurpose::Normal,
|
||||
qualifier: LabelQualifier::None,
|
||||
codec_hint,
|
||||
variant: String::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
labels
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mpls_audio_streams_become_labels() {
|
||||
// Two audio streams: English TrueHD 7.1 48k, French AC-3 5.1 48k.
|
||||
let pl = playlist_with(vec![
|
||||
audio_entry(0x1100, 0x83, 12, 1, "eng"),
|
||||
audio_entry(0x1101, 0x81, 6, 1, "fra"),
|
||||
]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert_eq!(labels.len(), 2);
|
||||
|
||||
// English TrueHD 7.1
|
||||
let a = &labels[0];
|
||||
assert_eq!(a.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(a.stream_number, 1);
|
||||
assert_eq!(a.language, "eng");
|
||||
assert_eq!(a.name, "English");
|
||||
assert_eq!(a.codec_hint, "TrueHD 7.1");
|
||||
assert_eq!(a.purpose, LabelPurpose::Normal);
|
||||
assert_eq!(a.qualifier, LabelQualifier::None);
|
||||
assert_eq!(a.variant, "");
|
||||
|
||||
// French AC-3 5.1
|
||||
let b = &labels[1];
|
||||
assert_eq!(b.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(b.stream_number, 2);
|
||||
assert_eq!(b.language, "fra");
|
||||
assert_eq!(b.name, "French");
|
||||
assert_eq!(b.codec_hint, "AC-3 5.1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mpls_pg_streams_become_subtitle_labels() {
|
||||
let pl = playlist_with(vec![
|
||||
pg_entry(0x1200, "eng"),
|
||||
pg_entry(0x1201, "spa"),
|
||||
pg_entry(0x1202, "fra"),
|
||||
]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert_eq!(labels.len(), 3);
|
||||
for label in &labels {
|
||||
assert_eq!(label.stream_type, StreamLabelType::Subtitle);
|
||||
assert_eq!(label.codec_hint, "PG");
|
||||
}
|
||||
assert_eq!(labels[0].stream_number, 1);
|
||||
assert_eq!(labels[0].language, "eng");
|
||||
assert_eq!(labels[0].name, "English");
|
||||
assert_eq!(labels[1].stream_number, 2);
|
||||
assert_eq!(labels[1].language, "spa");
|
||||
assert_eq!(labels[1].name, "Spanish");
|
||||
assert_eq!(labels[2].stream_number, 3);
|
||||
assert_eq!(labels[2].language, "fra");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_streams_across_playlists() {
|
||||
// Two playlists, same English TrueHD 7.1 PID 0x1100 in both.
|
||||
// Expect one Audio label, not two.
|
||||
let pl1 = playlist_with(vec![
|
||||
audio_entry(0x1100, 0x83, 12, 1, "eng"),
|
||||
audio_entry(0x1101, 0x81, 6, 1, "fra"),
|
||||
]);
|
||||
let pl2 = playlist_with(vec![
|
||||
audio_entry(0x1100, 0x83, 12, 1, "eng"), // duplicate
|
||||
audio_entry(0x1102, 0x82, 6, 1, "deu"), // new
|
||||
]);
|
||||
let labels = labels_from_playlists(&[pl1, pl2]);
|
||||
// Expected: eng@0x1100, fra@0x1101, deu@0x1102 — three uniques.
|
||||
assert_eq!(labels.len(), 3);
|
||||
// PID isn't stored on StreamLabel, so assert on the surviving
|
||||
// language set instead.
|
||||
let mut langs: Vec<String> = labels.iter().map(|l| l.language.clone()).collect();
|
||||
langs.sort();
|
||||
assert_eq!(langs, vec!["deu", "eng", "fra"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coding_type_to_codec_hint_table() {
|
||||
// Spot-check every entry in the spec table. Audio entries
|
||||
// come back bare (no channels/rate set) so codec_hint is the
|
||||
// codec name alone.
|
||||
let cases: &[(u8, &str)] = &[
|
||||
(0x02, "MPEG-2"),
|
||||
(0x1B, "H.264"),
|
||||
(0x24, "HEVC"),
|
||||
(0x80, "LPCM"),
|
||||
(0x81, "AC-3"),
|
||||
(0x82, "DTS"),
|
||||
(0x83, "TrueHD"),
|
||||
(0x84, "AC-3+"),
|
||||
(0x85, "DTS-HD"),
|
||||
(0x86, "DTS-HD MA"),
|
||||
(0x90, "PG"),
|
||||
(0x91, "IG"),
|
||||
(0xA1, "AC-3+ Secondary"),
|
||||
(0xA2, "DTS-HD Secondary"),
|
||||
];
|
||||
for (ct, expected) in cases {
|
||||
assert_eq!(
|
||||
codec_name(*ct),
|
||||
*expected,
|
||||
"coding_type 0x{:02X} should map to {}",
|
||||
ct,
|
||||
expected
|
||||
);
|
||||
}
|
||||
// Unknown bytes return empty.
|
||||
assert_eq!(codec_name(0x00), "");
|
||||
assert_eq!(codec_name(0xFF), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_format_appends_channel_layout() {
|
||||
let mono = audio_entry(1, 0x83, 1, 1, "eng");
|
||||
let stereo = audio_entry(2, 0x83, 3, 1, "eng");
|
||||
let surround_51 = audio_entry(3, 0x83, 6, 1, "eng");
|
||||
let surround_71 = audio_entry(4, 0x83, 12, 1, "eng");
|
||||
let unknown = audio_entry(5, 0x83, 0, 1, "eng");
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &mono),
|
||||
"TrueHD mono"
|
||||
);
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &stereo),
|
||||
"TrueHD 2.0"
|
||||
);
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &surround_51),
|
||||
"TrueHD 5.1"
|
||||
);
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &surround_71),
|
||||
"TrueHD 7.1"
|
||||
);
|
||||
assert_eq!(build_codec_hint(StreamLabelType::Audio, &unknown), "TrueHD");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_rate_only_shows_above_48k() {
|
||||
// 48 kHz (rate=1) is the universal default → not surfaced.
|
||||
let r48 = audio_entry(1, 0x83, 6, 1, "eng");
|
||||
// 96 kHz (rate=4) → surfaced.
|
||||
let r96 = audio_entry(2, 0x83, 6, 4, "eng");
|
||||
// 192 kHz (rate=5) → surfaced.
|
||||
let r192 = audio_entry(3, 0x83, 6, 5, "eng");
|
||||
assert_eq!(build_codec_hint(StreamLabelType::Audio, &r48), "TrueHD 5.1");
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &r96),
|
||||
"TrueHD 5.1 96kHz"
|
||||
);
|
||||
assert_eq!(
|
||||
build_codec_hint(StreamLabelType::Audio, &r192),
|
||||
"TrueHD 5.1 192kHz"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_iso_code_passes_through_without_display_name() {
|
||||
// Made-up code: keep the raw lowercase code as `language`,
|
||||
// but `name` is empty because we don't know it.
|
||||
let pl = playlist_with(vec![audio_entry(0x1100, 0x83, 6, 1, "xyz")]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].language, "xyz");
|
||||
assert_eq!(labels[0].name, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ig_and_dv_streams_are_skipped() {
|
||||
// stream_type 4 = IG, 7 = DV EL — both must not surface.
|
||||
let mut ig = pg_entry(0x1400, "eng");
|
||||
ig.stream_type = 4;
|
||||
let mut dv = audio_entry(0x1011, 0x24, 0, 0, "");
|
||||
dv.stream_type = 7;
|
||||
let pl = playlist_with(vec![ig, dv]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert!(labels.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secondary_audio_becomes_audio_label() {
|
||||
// stream_type 5 = secondary audio. The conversion should
|
||||
// still produce an Audio label (the registry's apply path
|
||||
// can ignore secondary if it wants — this module just
|
||||
// surfaces what's there).
|
||||
let mut sec = audio_entry(0x1A00, 0x83, 3, 1, "eng");
|
||||
sec.stream_type = 5;
|
||||
sec.secondary = true;
|
||||
let pl = playlist_with(vec![sec]);
|
||||
let labels = labels_from_playlists(&[pl]);
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(labels[0].codec_hint, "TrueHD 2.0");
|
||||
}
|
||||
}
|
||||
+20
-31
@@ -12,15 +12,15 @@
|
||||
//! sub_com1_idx="23,24,25" />
|
||||
//! ```
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, xml};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "playlists.xml")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let data = super::read_jar_file(reader, udf, "playlists.xml")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
@@ -30,8 +30,8 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
let mut labels = Vec::new();
|
||||
|
||||
// Parse audio streams
|
||||
if let Some(aud) = extract_attr(&feature, "aud") {
|
||||
let com_idx = extract_attr(&feature, "aud_com1_idx").and_then(|s| s.parse::<usize>().ok());
|
||||
if let Some(aud) = xml::attr(&feature, "aud") {
|
||||
let com_idx = xml::attr(&feature, "aud_com1_idx").and_then(|s| s.parse::<usize>().ok());
|
||||
|
||||
for (i, lang) in aud.split(',').enumerate() {
|
||||
let lang = lang.trim();
|
||||
@@ -57,12 +57,12 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
}
|
||||
|
||||
// Parse subtitle streams
|
||||
if let Some(sub) = extract_attr(&feature, "sub") {
|
||||
let forced: Vec<bool> = extract_attr(&feature, "forced_sub")
|
||||
if let Some(sub) = xml::attr(&feature, "sub") {
|
||||
let forced: Vec<bool> = xml::attr(&feature, "forced_sub")
|
||||
.map(|s| s.split(',').map(|f| f.trim() == "1").collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
let com_indices: Vec<usize> = extract_attr(&feature, "sub_com1_idx")
|
||||
let com_indices: Vec<usize> = xml::attr(&feature, "sub_com1_idx")
|
||||
.map(|s| s.split(',').filter_map(|i| i.trim().parse().ok()).collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
@@ -100,32 +100,29 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(labels)
|
||||
// High confidence: paramount's playlists.xml is fully structured
|
||||
// and we extract every documented field.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
|
||||
/// Find the feature playlist element (the one with the most audio tracks).
|
||||
fn find_feature_playlist(xml: &str) -> Option<String> {
|
||||
fn find_feature_playlist(text: &str) -> Option<String> {
|
||||
let mut best: Option<String> = None;
|
||||
let mut best_aud_count = 0;
|
||||
let mut from = 0;
|
||||
|
||||
let mut pos = 0;
|
||||
while let Some(start) = xml[pos..].find("<playlist ") {
|
||||
let abs_start = pos + start;
|
||||
let end = match xml[abs_start..].find("/>") {
|
||||
Some(p) => abs_start + p + 2,
|
||||
None => break,
|
||||
};
|
||||
let element = &xml[abs_start..end];
|
||||
while let Some((start, end)) = xml::find_element(text, "playlist", from) {
|
||||
let element = &text[start..end];
|
||||
|
||||
// Prefer name="Feature" explicitly
|
||||
if let Some(name) = extract_attr(element, "name") {
|
||||
// Prefer name="Feature" explicitly.
|
||||
if let Some(name) = xml::attr(element, "name") {
|
||||
if name.eq_ignore_ascii_case("Feature") {
|
||||
return Some(element.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise pick the one with the most audio streams
|
||||
if let Some(aud) = extract_attr(element, "aud") {
|
||||
// Otherwise pick the one with the most audio streams.
|
||||
if let Some(aud) = xml::attr(element, "aud") {
|
||||
let count = aud.split(',').count();
|
||||
if count > best_aud_count {
|
||||
best_aud_count = count;
|
||||
@@ -133,15 +130,7 @@ fn find_feature_playlist(xml: &str) -> Option<String> {
|
||||
}
|
||||
}
|
||||
|
||||
pos = end;
|
||||
from = end;
|
||||
}
|
||||
best
|
||||
}
|
||||
|
||||
/// Extract an XML attribute value from an element string.
|
||||
fn extract_attr(element: &str, name: &str) -> Option<String> {
|
||||
let needle = format!("{name}=\"");
|
||||
let start = element.find(&needle)? + needle.len();
|
||||
let end = element[start..].find('"')? + start;
|
||||
Some(element[start..end].to_string())
|
||||
}
|
||||
|
||||
+221
-28
@@ -5,9 +5,13 @@
|
||||
//!
|
||||
//! Token format: `{lang}_{codec?}_{purpose?}_{region?}_`
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType, vocab};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{
|
||||
Confidence, LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, text,
|
||||
vocab,
|
||||
};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
/// Known audio codec tokens
|
||||
const AUDIO_CODECS: &[&str] = &["MLP", "AC3", "DTS", "DDL", "WAV", "AC"];
|
||||
@@ -20,18 +24,60 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "bluray_project.bin")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let data = super::read_jar_file(reader, udf, "bluray_project.bin")?;
|
||||
let strings = extract_strings(&data);
|
||||
// min_len=4 matches the prior local extract_strings impl. The token
|
||||
// grammar is `{lang3}_{codec?}_{purpose?}_{region?}_` so the
|
||||
// shortest meaningful run is 4 chars (lang + underscore).
|
||||
let strings = text::extract_ascii_strings(&data, 4);
|
||||
|
||||
// Tracked across all parse_token calls in this run: did any stream
|
||||
// hit an unrecognized token component (skip-unknown path)? If yes
|
||||
// we downgrade confidence to Medium — the labels are still valid
|
||||
// but the corpus surfaced something we don't catalogue.
|
||||
let saw_unknown = AtomicBool::new(false);
|
||||
|
||||
let labels = assign_labels(&strings, &saw_unknown);
|
||||
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let confidence = if saw_unknown.load(Ordering::Relaxed) {
|
||||
Confidence::Medium
|
||||
} else {
|
||||
Confidence::High
|
||||
};
|
||||
Some(ParseResult { labels, confidence })
|
||||
}
|
||||
|
||||
/// Walk the extracted token strings of the feature section and emit a
|
||||
/// `StreamLabel` per editorial token, numbered in STN order. Split out
|
||||
/// from `parse` so the section/numbering logic is unit-testable without
|
||||
/// a `SectorSource`/`UdfFs`.
|
||||
fn assign_labels(strings: &[String], saw_unknown: &AtomicBool) -> Vec<StreamLabel> {
|
||||
// The authoritative per-feature stream list lives in the `FPL_`
|
||||
// (FeaturePLaylist) section, in STN order. `SEG_*` entries are menu
|
||||
// segments (intros, logos, disclaimers, previews) that can also carry
|
||||
// stray stream tokens — e.g. a `SEG_MainFeature` preview segment that
|
||||
// lists only a commentary track. Anchoring on such a segment grabs the
|
||||
// wrong streams and misnumbers them. So when the project ships any
|
||||
// `FPL_` playlist, anchor exclusively on it; only fall back to
|
||||
// `SEG_MainFeature` on discs that have no `FPL_` section at all.
|
||||
let has_fpl = strings.iter().any(|s| s.starts_with("FPL_"));
|
||||
|
||||
let mut labels = Vec::new();
|
||||
let mut in_feature = false;
|
||||
let mut audio_num: u16 = 0;
|
||||
let mut sub_num: u16 = 0;
|
||||
|
||||
for s in &strings {
|
||||
for s in strings {
|
||||
// Detect feature section start
|
||||
if s.starts_with("FPL_") || s.starts_with("SEG_MainFeature") {
|
||||
let is_start = if has_fpl {
|
||||
s.starts_with("FPL_")
|
||||
} else {
|
||||
s.starts_with("SEG_MainFeature")
|
||||
};
|
||||
if is_start {
|
||||
if in_feature {
|
||||
break;
|
||||
}
|
||||
@@ -50,7 +96,25 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(label) = parse_token(s) {
|
||||
// Generic audio-slot placeholder. Pixelogic lists each audio stream
|
||||
// in the playlist as either an editorial `{lang}_{codec}_…` token OR
|
||||
// a bare `Audio Stream N` placeholder when no editorial label was
|
||||
// authored. The placeholder carries nothing to label, but it DOES
|
||||
// occupy an STN slot — so it must advance `audio_num`. Otherwise a
|
||||
// later editorial token (e.g. a lone `eng_ACOM_` commentary sitting
|
||||
// at STN slot 4, behind three unlabelled main tracks) collapses onto
|
||||
// slot 1 and its Commentary purpose lands on the main feature track.
|
||||
//
|
||||
// Only audio is corrected here: subtitle (`PG Stream N`) numbering is
|
||||
// left exactly as-is — the corpus snapshots show forced/commentary
|
||||
// subtitle tokens already align with STN without counting the
|
||||
// placeholders, and counting them regresses several discs.
|
||||
if s.starts_with("Audio Stream") {
|
||||
audio_num += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(label) = parse_token_inner(s, Some(saw_unknown)) {
|
||||
match label.stream_type {
|
||||
StreamLabelType::Audio => {
|
||||
audio_num += 1;
|
||||
@@ -70,13 +134,10 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
}
|
||||
}
|
||||
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(labels)
|
||||
labels
|
||||
}
|
||||
|
||||
fn parse_token(s: &str) -> Option<StreamLabel> {
|
||||
fn parse_token_inner(s: &str, saw_unknown: Option<&AtomicBool>) -> Option<StreamLabel> {
|
||||
let clean = s.trim().trim_start_matches('\t').trim_end_matches('_');
|
||||
let parts: Vec<&str> = clean.split('_').collect();
|
||||
if parts.len() < 2 {
|
||||
@@ -127,7 +188,18 @@ fn parse_token(s: &str) -> Option<StreamLabel> {
|
||||
} else if part.starts_with("PGStream") {
|
||||
is_subtitle = true;
|
||||
} else {
|
||||
return None;
|
||||
// Unknown token component — skip this single part rather
|
||||
// than discarding the entire stream record. Pre-refactor
|
||||
// behavior was `return None` here, which silently dropped
|
||||
// any stream containing a single uncatalogued token (e.g.
|
||||
// a new codec ID or framework variant). Better to surface
|
||||
// what we know than discard a whole stream over one part,
|
||||
// but flag the parse as Medium-confidence so callers know
|
||||
// some data was elided.
|
||||
tracing::debug!(part = %part, "pixelogic: unrecognized token component, skipping");
|
||||
if let Some(flag) = saw_unknown {
|
||||
flag.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,22 +225,143 @@ fn parse_token(s: &str) -> Option<StreamLabel> {
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_strings(data: &[u8]) -> Vec<String> {
|
||||
let mut strings = Vec::new();
|
||||
let mut current = String::new();
|
||||
// extract_strings removed — replaced by super::text::extract_ascii_strings(data, 4).
|
||||
|
||||
for &b in data {
|
||||
if (0x20..0x7f).contains(&b) {
|
||||
current.push(b as char);
|
||||
} else {
|
||||
if current.len() > 3 {
|
||||
strings.push(current.clone());
|
||||
}
|
||||
current.clear();
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_token_basic_audio() {
|
||||
let l = parse_token_inner("eng_MLP_", None).unwrap();
|
||||
assert_eq!(l.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.codec_hint, "TrueHD");
|
||||
assert_eq!(l.purpose, LabelPurpose::Normal);
|
||||
}
|
||||
if current.len() > 3 {
|
||||
strings.push(current);
|
||||
|
||||
#[test]
|
||||
fn parse_token_basic_subtitle_sdh() {
|
||||
let l = parse_token_inner("eng_SDH_", None).unwrap();
|
||||
assert_eq!(l.stream_type, StreamLabelType::Subtitle);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.qualifier, LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_commentary() {
|
||||
let l = parse_token_inner("eng_MLP_ACOM_", None).unwrap();
|
||||
assert_eq!(l.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(l.purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_descriptive() {
|
||||
let l = parse_token_inner("eng_AC3_ADES_", None).unwrap();
|
||||
assert_eq!(l.purpose, LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_with_region() {
|
||||
let l = parse_token_inner("eng_MLP_US_", None).unwrap();
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.variant, "US");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_unknown_component_does_not_kill_stream() {
|
||||
// Regression: pre-refactor, an unrecognized token part returned
|
||||
// None for the whole stream, silently dropping it. New
|
||||
// behavior: skip the unknown part, surface what we know.
|
||||
let l = parse_token_inner("eng_MLP_FUTUREFLAG_FOR_", None).unwrap();
|
||||
assert_eq!(l.stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(l.language, "eng");
|
||||
assert_eq!(l.codec_hint, "TrueHD");
|
||||
assert_eq!(l.qualifier, LabelQualifier::Forced);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_no_audio_or_subtitle_signal_returns_none() {
|
||||
// A token that has only a language and an unknown part with
|
||||
// no audio/subtitle classifier should still return None —
|
||||
// there's no way to file it as a stream.
|
||||
assert!(parse_token_inner("eng_UNKNOWN_", None).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_token_rejects_non_lang_prefix() {
|
||||
assert!(parse_token_inner("XX_MLP_", None).is_none());
|
||||
assert!(parse_token_inner("ENG_MLP_", None).is_none()); // uppercase not accepted as ISO 639-2
|
||||
}
|
||||
|
||||
fn strs(v: &[&str]) -> Vec<String> {
|
||||
v.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assign_labels_numbers_commentary_behind_placeholders() {
|
||||
// Observed case: the FPL_MainFeature playlist lists three unlabelled main
|
||||
// audio tracks as `Audio Stream N` placeholders, then a lone
|
||||
// `eng_ACOM_` commentary at STN slot 4. The commentary must land on
|
||||
// audio #4, not collapse onto #1 (which would tag the main feature
|
||||
// track as commentary).
|
||||
let flag = AtomicBool::new(false);
|
||||
let tokens = strs(&[
|
||||
"FPL_MainFeature",
|
||||
"Audio Stream 1",
|
||||
"Audio Stream 2",
|
||||
"Audio Stream 3",
|
||||
"eng_ACOM_",
|
||||
]);
|
||||
let labels = assign_labels(&tokens, &flag);
|
||||
let audio: Vec<_> = labels
|
||||
.iter()
|
||||
.filter(|l| l.stream_type == StreamLabelType::Audio)
|
||||
.collect();
|
||||
assert_eq!(audio.len(), 1, "only the commentary carries a label");
|
||||
assert_eq!(audio[0].stream_number, 4, "commentary is STN slot 4");
|
||||
assert_eq!(audio[0].purpose, LabelPurpose::Commentary);
|
||||
assert_eq!(audio[0].language, "eng");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assign_labels_prefers_fpl_over_seg_mainfeature() {
|
||||
// A `SEG_MainFeature` menu/preview segment carries a stray commentary
|
||||
// token, but the real playlist is `FPL_MainFeature`. When an FPL_
|
||||
// section exists, the SEG_ one must be ignored as an anchor — so we
|
||||
// number from the FPL playlist, putting the commentary at slot 2.
|
||||
let flag = AtomicBool::new(false);
|
||||
let tokens = strs(&[
|
||||
"SEG_MainFeature",
|
||||
"eng_ACOM_", // stray token in the menu segment — must be ignored
|
||||
"FPL_MainFeature",
|
||||
"Audio Stream 1",
|
||||
"eng_ACOM_",
|
||||
]);
|
||||
let labels = assign_labels(&tokens, &flag);
|
||||
let audio: Vec<_> = labels
|
||||
.iter()
|
||||
.filter(|l| l.stream_type == StreamLabelType::Audio)
|
||||
.collect();
|
||||
assert_eq!(audio.len(), 1);
|
||||
assert_eq!(audio[0].stream_number, 2, "numbered from the FPL playlist");
|
||||
assert_eq!(audio[0].purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assign_labels_falls_back_to_seg_without_fpl() {
|
||||
// Discs with no FPL_ playlist still anchor on SEG_MainFeature.
|
||||
let flag = AtomicBool::new(false);
|
||||
let tokens = strs(&["SEG_MainFeature", "eng_MLP_", "spa_AC3_"]);
|
||||
let labels = assign_labels(&tokens, &flag);
|
||||
let audio: Vec<_> = labels
|
||||
.iter()
|
||||
.filter(|l| l.stream_type == StreamLabelType::Audio)
|
||||
.collect();
|
||||
assert_eq!(audio.len(), 2);
|
||||
assert_eq!(audio[0].stream_number, 1);
|
||||
assert_eq!(audio[0].language, "eng");
|
||||
assert_eq!(audio[1].stream_number, 2);
|
||||
assert_eq!(audio[1].language, "spa");
|
||||
}
|
||||
strings
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
//! Text-extraction helpers used by parsers that scan binary blobs for
|
||||
//! embedded label strings.
|
||||
//!
|
||||
//! Promoted from two near-duplicate implementations:
|
||||
//! - `pixelogic::extract_strings` (`bluray_project.bin`, min_len=4)
|
||||
//! - `dbp::extract_printable` (`.class` files in jars, min_len=5)
|
||||
//!
|
||||
//! Single implementation, threshold passed in. Callers that have a
|
||||
//! more structured parse path (e.g. `class_reader` for .class) should
|
||||
//! prefer that — this helper is for genuinely unstructured input.
|
||||
|
||||
/// Walk `data`, emit every maximal run of printable-ASCII bytes
|
||||
/// (`0x20..=0x7E`) whose length is at least `min_len`.
|
||||
///
|
||||
/// Non-printable bytes (including `\t`, `\n`, NUL) terminate the
|
||||
/// current run. Output strings are guaranteed valid UTF-8 (they're
|
||||
/// pure 7-bit ASCII). Strings shorter than `min_len` are dropped.
|
||||
pub fn extract_ascii_strings(data: &[u8], min_len: usize) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
let mut current = String::new();
|
||||
for &b in data {
|
||||
if (0x20..=0x7E).contains(&b) {
|
||||
current.push(b as char);
|
||||
} else if current.len() >= min_len {
|
||||
out.push(std::mem::take(&mut current));
|
||||
} else {
|
||||
current.clear();
|
||||
}
|
||||
}
|
||||
if current.len() >= min_len {
|
||||
out.push(current);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn extracts_simple_runs() {
|
||||
let got = extract_ascii_strings(b"hello\0world\0", 3);
|
||||
assert_eq!(got, vec!["hello", "world"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn applies_minimum_length() {
|
||||
let got = extract_ascii_strings(b"hi\0ok\0longer\0", 5);
|
||||
assert_eq!(got, vec!["longer"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn treats_tab_and_newline_as_separators() {
|
||||
// \t (0x09) and \n (0x0A) are below 0x20, so they break runs.
|
||||
let got = extract_ascii_strings(b"alpha\tbeta\ngamma", 3);
|
||||
assert_eq!(got, vec!["alpha", "beta", "gamma"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handles_trailing_run_without_terminator() {
|
||||
// Run at the very end of the buffer should still be emitted.
|
||||
let got = extract_ascii_strings(b"prefix\0tail", 3);
|
||||
assert_eq!(got, vec!["prefix", "tail"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_high_bit_bytes() {
|
||||
// 0x80+ is non-printable per this helper's definition.
|
||||
let mut buf = b"good".to_vec();
|
||||
buf.push(0xC3);
|
||||
buf.push(0xA9);
|
||||
buf.extend_from_slice(b"more");
|
||||
let got = extract_ascii_strings(&buf, 3);
|
||||
assert_eq!(got, vec!["good", "more"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_returns_empty() {
|
||||
let got = extract_ascii_strings(&[], 1);
|
||||
assert!(got.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn min_len_zero_emits_singletons() {
|
||||
// Pathological but well-defined.
|
||||
let got = extract_ascii_strings(b"a\0b", 0);
|
||||
assert_eq!(got, vec!["a", "b"]);
|
||||
}
|
||||
}
|
||||
+431
-9
@@ -1,19 +1,39 @@
|
||||
//! Shared label vocabulary — values we are 100% confident about.
|
||||
//! Shared label vocabulary — canonical mappings used by ≥1 label parser.
|
||||
//!
|
||||
//! Labels come from BD-J authoring tool files (bluray_project.bin,
|
||||
//! playlists.xml, menu_base.prop, etc.) — NOT from BD spec fields.
|
||||
//! This is NOT for MPLS/CLPI/STN data. Those follow the BD spec directly.
|
||||
//! playlists.xml, menu_base.prop, .class string pools, etc.) — NOT
|
||||
//! from BD spec fields. This module is the central, regression-tested
|
||||
//! source of truth for:
|
||||
//!
|
||||
//! Rules:
|
||||
//! - Only map values we are 100% certain about (published codec names).
|
||||
//! - Unknown codes (csp, eda, cf, etc.) pass through raw from disc.
|
||||
//! - The app/CLI handles display text, not the lib.
|
||||
//! - Codec brand name aliases (`MLP` → `TrueHD`).
|
||||
//! - English / multi-word language name → ISO 639-2 code.
|
||||
//! - English text → [`LabelPurpose`] (Commentary / Descriptive / etc.).
|
||||
//! - English text → [`LabelQualifier`] (SDH / Forced / Descriptive Service).
|
||||
//!
|
||||
//! Rules of engagement:
|
||||
//!
|
||||
//! 1. Only map values we are 100% certain about — published codec
|
||||
//! names, well-known ISO 639-2 mappings, vendor-documented purpose
|
||||
//! keywords.
|
||||
//! 2. Unknown codes / unrecognized phrases pass through raw or return
|
||||
//! `None`. We never guess.
|
||||
//! 3. Matching is case-insensitive and word-boundary-aware where
|
||||
//! relevant (so "Commenter" doesn't match "commentary"). Anchoring
|
||||
//! on whole tokens is the responsibility of this module — callers
|
||||
//! pass raw text, we handle it.
|
||||
//!
|
||||
//! This module is NOT for BD spec STN codec IDs; those decode in
|
||||
//! `mpls.rs` separately.
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier};
|
||||
|
||||
// ── Codec aliases ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Map a codec identifier found in label data to its display name.
|
||||
///
|
||||
/// These are well-known codec identifiers used across multiple BD-J
|
||||
/// authoring tools. NOT BD spec STN codec IDs — those are decoded
|
||||
/// separately in mpls.rs.
|
||||
/// authoring tools. Unknown codes pass through unchanged so callers
|
||||
/// can still surface vendor-specific tokens we haven't catalogued.
|
||||
pub fn codec(code: &str) -> &str {
|
||||
match code {
|
||||
"MLP" => "TrueHD",
|
||||
@@ -25,3 +45,405 @@ pub fn codec(code: &str) -> &str {
|
||||
_ => code,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Language: English / multi-word names → ISO 639-2 ─────────────────────────
|
||||
|
||||
/// Result of [`lang`] — ISO code + human-readable regional variant.
|
||||
///
|
||||
/// `code` is ISO 639-2 (always 3 lowercase letters).
|
||||
/// `variant` is the regional dialect as a human-readable English word
|
||||
/// (`"Brazilian"`, `"Castilian"`, `"Canadian"`, `"Simplified"`, ...)
|
||||
/// or `""` when the input names just a bare language without
|
||||
/// dialect ("Spanish" → variant=""). The variant matches the
|
||||
/// convention pixelogic / ctrm / criterion already use for their
|
||||
/// `StreamLabel::variant` field: a short display token the UI can
|
||||
/// surface verbatim.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct LangInfo {
|
||||
pub code: &'static str,
|
||||
pub variant: &'static str,
|
||||
}
|
||||
|
||||
/// Map a free-form language label fragment to an ISO 639-2 code AND
|
||||
/// (where applicable) its regional variant.
|
||||
///
|
||||
/// Handles both bare English names ("English", "Spanish") and the
|
||||
/// multi-word vendor variants we've seen in the corpus ("Brazilian
|
||||
/// Portuguese", "Castilian Spanish", "Canadian French"). Match is
|
||||
/// case-insensitive; longer compound phrases win over their bare
|
||||
/// counterparts (so "Brazilian Portuguese" returns
|
||||
/// `LangInfo { code: "por", variant: "Brazilian" }`, not consumed by
|
||||
/// the bare "Portuguese" entry).
|
||||
///
|
||||
/// Bare-name matches return `variant: ""`.
|
||||
///
|
||||
/// Returns `None` for unrecognized input — callers decide whether to
|
||||
/// fall back to MPLS spec codes, pass through raw, or drop the stream.
|
||||
/// Never guesses.
|
||||
///
|
||||
/// Why the variant: the prior `lang() -> Option<&str>` shape silently
|
||||
/// dropped regional dialect info. "Brazilian Portuguese 5.1" became
|
||||
/// `language="por", variant=""` — UI displayed plain "Portuguese"
|
||||
/// even though the disc had explicitly labeled this stream Brazilian.
|
||||
/// Capturing the variant here parallels how pixelogic and ctrm
|
||||
/// populate `StreamLabel::variant` from their own region tables.
|
||||
pub fn lang(text: &str) -> Option<LangInfo> {
|
||||
let lower = text.to_lowercase();
|
||||
// Multi-word compounds first — longest-match wins.
|
||||
for (needle, code, variant) in COMPOUND_LANGS {
|
||||
if lower.contains(needle) {
|
||||
return Some(LangInfo { code, variant });
|
||||
}
|
||||
}
|
||||
// Bare names: word-boundary match (avoid "english" inside "englishman"
|
||||
// or any other accidental substring).
|
||||
for (needle, code) in BARE_LANGS {
|
||||
if has_word(&lower, needle) {
|
||||
return Some(LangInfo { code, variant: "" });
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
const COMPOUND_LANGS: &[(&str, &str, &str)] = &[
|
||||
("brazilian portuguese", "por", "Brazilian"),
|
||||
("euro portuguese", "por", "European"),
|
||||
("european portuguese", "por", "European"),
|
||||
("castilian spanish", "spa", "Castilian"),
|
||||
("latin american spanish", "spa", "Latin American"),
|
||||
("latin spanish", "spa", "Latin American"),
|
||||
("canadian french", "fra", "Canadian"),
|
||||
("parisian french", "fra", "Parisian"),
|
||||
("australian english", "eng", "Australian"),
|
||||
("austrailian english", "eng", "Australian"), // disc-corpus typo, keep matching
|
||||
("british english", "eng", "British"),
|
||||
("simplified chinese", "zho", "Simplified"),
|
||||
("traditional chinese", "zho", "Traditional"),
|
||||
("mandarin chinese", "zho", "Mandarin"),
|
||||
("cantonese chinese", "zho", "Cantonese"),
|
||||
];
|
||||
|
||||
const BARE_LANGS: &[(&str, &str)] = &[
|
||||
("english", "eng"),
|
||||
("spanish", "spa"),
|
||||
("french", "fra"),
|
||||
("german", "deu"),
|
||||
("italian", "ita"),
|
||||
("japanese", "jpn"),
|
||||
("chinese", "zho"),
|
||||
("mandarin", "zho"),
|
||||
("cantonese", "zho"),
|
||||
("portuguese", "por"),
|
||||
("polish", "pol"),
|
||||
("czech", "ces"),
|
||||
("hungarian", "hun"),
|
||||
("dutch", "nld"),
|
||||
("korean", "kor"),
|
||||
("arabic", "ara"),
|
||||
("hindi", "hin"),
|
||||
("turkish", "tur"),
|
||||
("thai", "tha"),
|
||||
("swedish", "swe"),
|
||||
("norwegian", "nor"),
|
||||
("danish", "dan"),
|
||||
("finnish", "fin"),
|
||||
("hebrew", "heb"),
|
||||
("russian", "rus"),
|
||||
("greek", "ell"),
|
||||
("vietnamese", "vie"),
|
||||
("indonesian", "ind"),
|
||||
("malay", "msa"),
|
||||
("ukrainian", "ukr"),
|
||||
("romanian", "ron"),
|
||||
("bulgarian", "bul"),
|
||||
("croatian", "hrv"),
|
||||
("serbian", "srp"),
|
||||
("slovak", "slk"),
|
||||
("slovenian", "slv"),
|
||||
("estonian", "est"),
|
||||
("latvian", "lav"),
|
||||
("lithuanian", "lit"),
|
||||
("icelandic", "isl"),
|
||||
("basque", "eus"),
|
||||
("catalan", "cat"),
|
||||
("galician", "glg"),
|
||||
];
|
||||
|
||||
// ── Purpose ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Classify a free-form English label string into a [`LabelPurpose`].
|
||||
///
|
||||
/// Recognized keywords (case-insensitive, word-boundary matched):
|
||||
/// - "commentary", "director's commentary" → `Commentary`
|
||||
/// - "descriptive", "description", "audio description", "described" → `Descriptive`
|
||||
/// - "score", "music only" → `Score`
|
||||
/// - "ime" (alternate music for closing themes etc.) → `Ime`
|
||||
/// - anything else → `Normal`
|
||||
///
|
||||
/// Word-boundary matching means "Commentary track" matches but
|
||||
/// "Commenter Pro audio" does not.
|
||||
pub fn purpose(text: &str) -> LabelPurpose {
|
||||
let lower = text.to_lowercase();
|
||||
// Multi-word compounds first — they're more specific.
|
||||
if lower.contains("audio description") || lower.contains("descriptive service") {
|
||||
return LabelPurpose::Descriptive;
|
||||
}
|
||||
if lower.contains("music only") {
|
||||
return LabelPurpose::Score;
|
||||
}
|
||||
if has_word(&lower, "commentary") {
|
||||
return LabelPurpose::Commentary;
|
||||
}
|
||||
if has_word(&lower, "descriptive")
|
||||
|| has_word(&lower, "description")
|
||||
|| has_word(&lower, "described")
|
||||
{
|
||||
return LabelPurpose::Descriptive;
|
||||
}
|
||||
if has_word(&lower, "score") {
|
||||
return LabelPurpose::Score;
|
||||
}
|
||||
if has_word(&lower, "ime") {
|
||||
return LabelPurpose::Ime;
|
||||
}
|
||||
LabelPurpose::Normal
|
||||
}
|
||||
|
||||
// ── Qualifier ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Classify a free-form English label string into a [`LabelQualifier`].
|
||||
///
|
||||
/// Recognized keywords (case-insensitive, word-boundary matched):
|
||||
/// - "sdh", "captions" → `Sdh`
|
||||
/// - "forced", "forced narrative" → `Forced`
|
||||
/// - "rnib", "descriptive service" → `DescriptiveService`
|
||||
/// - anything else → `None`
|
||||
///
|
||||
/// SDH (Subtitles for the Deaf and Hard of hearing) wins over Forced
|
||||
/// when both keywords are present, because an SDH track is its own
|
||||
/// stream regardless of whether the player flags it as "forced".
|
||||
pub fn qualifier(text: &str) -> LabelQualifier {
|
||||
let lower = text.to_lowercase();
|
||||
if has_word(&lower, "sdh") || has_word(&lower, "captions") {
|
||||
return LabelQualifier::Sdh;
|
||||
}
|
||||
if lower.contains("descriptive service") || has_word(&lower, "rnib") {
|
||||
return LabelQualifier::DescriptiveService;
|
||||
}
|
||||
if has_word(&lower, "forced") {
|
||||
return LabelQualifier::Forced;
|
||||
}
|
||||
LabelQualifier::None
|
||||
}
|
||||
|
||||
// ── Internal: word-boundary matching ────────────────────────────────────────
|
||||
|
||||
/// True if `needle` appears in `haystack` surrounded by non-alphanumeric
|
||||
/// boundaries (or string ends). `haystack` is assumed lowercase already.
|
||||
///
|
||||
/// This is the load-bearing primitive for `lang` / `purpose` /
|
||||
/// `qualifier`: bare-token matchers MUST use it, otherwise we match
|
||||
/// "english" inside "englishman" and "sdh" inside "lambdash". The
|
||||
/// existing parsers used `.contains()` and got lucky on the corpus;
|
||||
/// vocab guarantees the boundary.
|
||||
fn has_word(haystack: &str, needle: &str) -> bool {
|
||||
if needle.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let bytes = haystack.as_bytes();
|
||||
let nb = needle.as_bytes();
|
||||
let mut i = 0;
|
||||
while i + nb.len() <= bytes.len() {
|
||||
if &bytes[i..i + nb.len()] == nb {
|
||||
let before = if i == 0 { None } else { Some(bytes[i - 1]) };
|
||||
let after = bytes.get(i + nb.len()).copied();
|
||||
let bound = |c: Option<u8>| match c {
|
||||
None => true,
|
||||
Some(b) => !b.is_ascii_alphanumeric(),
|
||||
};
|
||||
if bound(before) && bound(after) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn codec_known_aliases() {
|
||||
assert_eq!(codec("MLP"), "TrueHD");
|
||||
assert_eq!(codec("AC3"), "Dolby Digital");
|
||||
assert_eq!(codec("AC"), "Dolby Digital");
|
||||
assert_eq!(codec("DDL"), "Dolby Digital Plus");
|
||||
assert_eq!(codec("atmos"), "Dolby Atmos");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_unknown_passes_through() {
|
||||
assert_eq!(codec("FX9"), "FX9");
|
||||
assert_eq!(codec(""), "");
|
||||
}
|
||||
|
||||
fn li(code: &'static str, variant: &'static str) -> LangInfo {
|
||||
LangInfo { code, variant }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_bare_names_have_empty_variant() {
|
||||
assert_eq!(lang("English"), Some(li("eng", "")));
|
||||
assert_eq!(lang("english"), Some(li("eng", "")));
|
||||
assert_eq!(lang("Spanish 5.1 Dolby Digital"), Some(li("spa", "")));
|
||||
assert_eq!(lang("japanese"), Some(li("jpn", "")));
|
||||
assert_eq!(lang("Italian"), Some(li("ita", "")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_compounds_carry_variant() {
|
||||
assert_eq!(
|
||||
lang("Brazilian Portuguese 5.1"),
|
||||
Some(li("por", "Brazilian"))
|
||||
);
|
||||
assert_eq!(lang("Castilian Spanish"), Some(li("spa", "Castilian")));
|
||||
assert_eq!(
|
||||
lang("Canadian French Dolby Digital"),
|
||||
Some(li("fra", "Canadian"))
|
||||
);
|
||||
assert_eq!(
|
||||
lang("Latin American Spanish"),
|
||||
Some(li("spa", "Latin American"))
|
||||
);
|
||||
assert_eq!(lang("Simplified Chinese"), Some(li("zho", "Simplified")));
|
||||
assert_eq!(lang("British English"), Some(li("eng", "British")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_compounds_win_over_bare() {
|
||||
// Brazilian Portuguese must map to (por, Brazilian) via the
|
||||
// compound rule, not be intercepted by bare "portuguese"
|
||||
// (which would yield (por, "") and lose the variant).
|
||||
assert_eq!(lang("Brazilian Portuguese").unwrap().variant, "Brazilian");
|
||||
assert_eq!(lang("Canadian French").unwrap().variant, "Canadian");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_unknown_returns_none() {
|
||||
assert_eq!(lang("Klingon Dolby Atmos"), None);
|
||||
assert_eq!(lang(""), None);
|
||||
assert_eq!(lang("eng"), None); // 3-letter codes are not English names
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_word_boundary_avoids_substring_false_positive() {
|
||||
// No false positive — "engineering" must NOT match "english".
|
||||
assert_eq!(lang("Audio Engineering Demo"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_recognizes_commentary() {
|
||||
assert_eq!(purpose("English Commentary"), LabelPurpose::Commentary);
|
||||
assert_eq!(purpose("Director's Commentary"), LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_recognizes_descriptive() {
|
||||
assert_eq!(
|
||||
purpose("English Descriptive Audio"),
|
||||
LabelPurpose::Descriptive
|
||||
);
|
||||
assert_eq!(purpose("Audio Description"), LabelPurpose::Descriptive);
|
||||
assert_eq!(purpose("Described Video"), LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_descriptive_service_routes_to_descriptive() {
|
||||
// "Descriptive Service" is qualifier territory but the purpose
|
||||
// implication is Descriptive — vocab::purpose treats it as such.
|
||||
assert_eq!(
|
||||
purpose("English Descriptive Service"),
|
||||
LabelPurpose::Descriptive
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_word_boundary_avoids_commenter_false_positive() {
|
||||
// "Commenter Pro audio" does NOT match "commentary" — the
|
||||
// existing dbp/ctrm hand-rolls would have. Vocab is stricter.
|
||||
assert_eq!(purpose("Commenter Pro audio track"), LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_recognizes_score() {
|
||||
assert_eq!(purpose("Music Only"), LabelPurpose::Score);
|
||||
assert_eq!(purpose("Isolated Score"), LabelPurpose::Score);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purpose_unknown_is_normal() {
|
||||
assert_eq!(purpose("English Dolby Atmos"), LabelPurpose::Normal);
|
||||
assert_eq!(purpose(""), LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_recognizes_sdh() {
|
||||
assert_eq!(qualifier("English SDH"), LabelQualifier::Sdh);
|
||||
assert_eq!(qualifier("English Captions"), LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_recognizes_forced() {
|
||||
assert_eq!(qualifier("English Forced"), LabelQualifier::Forced);
|
||||
assert_eq!(qualifier("Forced Narrative"), LabelQualifier::Forced);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_recognizes_descriptive_service() {
|
||||
assert_eq!(
|
||||
qualifier("English RNIB"),
|
||||
LabelQualifier::DescriptiveService
|
||||
);
|
||||
assert_eq!(
|
||||
qualifier("English Descriptive Service"),
|
||||
LabelQualifier::DescriptiveService
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_sdh_wins_over_forced_when_both_present() {
|
||||
// SDH track is its own stream regardless of forced flag.
|
||||
assert_eq!(qualifier("English Forced SDH"), LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifier_unknown_is_none() {
|
||||
assert_eq!(qualifier("English"), LabelQualifier::None);
|
||||
assert_eq!(qualifier(""), LabelQualifier::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_word_basic() {
|
||||
assert!(has_word("english forced", "english"));
|
||||
assert!(has_word("english forced", "forced"));
|
||||
assert!(has_word("english", "english"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_word_rejects_substring() {
|
||||
assert!(!has_word("engineering", "english"));
|
||||
assert!(!has_word("englishman", "english"));
|
||||
assert!(!has_word("aenglish", "english"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_word_punctuation_boundary() {
|
||||
// "(SDH)" is a valid boundary — parentheses count as non-alphanum.
|
||||
assert!(has_word("english (sdh)", "sdh"));
|
||||
assert!(has_word("commentary,extra,info", "commentary"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,452 @@
|
||||
//! Tolerant XML scraping helpers — promoted from two near-duplicate
|
||||
//! hand-rolls in `paramount.rs` (attribute extraction) and `criterion.rs`
|
||||
//! (tag-text extraction).
|
||||
//!
|
||||
//! These are NOT a full XML parser. They handle the subset of XML the
|
||||
//! BD-J authoring tools we've seen actually emit: ASCII tag/attr
|
||||
//! names, no entity references inside label strings, optional XML
|
||||
//! namespaces. Hardening goals over the prior `find("<tag>")` /
|
||||
//! `find(r#"name=""#)` matchers:
|
||||
//!
|
||||
//! 1. **Case-insensitive** tag and attribute names — vendors casing
|
||||
//! is inconsistent across authoring-tool revisions.
|
||||
//! 2. **Namespace-aware** — strip an optional `ns:` prefix so
|
||||
//! `<ns:playlist>` and `<playlist>` both match.
|
||||
//! 3. **Whitespace-tolerant** — multiple/tab/newline characters
|
||||
//! around `=` between attribute name and value; whitespace inside
|
||||
//! the opening tag.
|
||||
//! 4. **Quote-style tolerant** — both `"value"` and `'value'`.
|
||||
//! 5. **Self-closing tag handling** — `<tag />` and `<tag/>` both
|
||||
//! work; [`text`] returns `Some("")` for empty content.
|
||||
//!
|
||||
//! Out of scope (intentionally simple): XML entity decoding
|
||||
//! (`&`, `<`, etc.), CDATA sections, comments, processing
|
||||
//! instructions, DTD declarations. None of the BD-J authored disc
|
||||
//! data we've observed exercises any of those — labels are plain
|
||||
//! ASCII/Latin-1 in attribute values.
|
||||
|
||||
/// Extract the value of attribute `name` from one XML element
|
||||
/// fragment (e.g. `<playlist name="Feature" id="00222" />`).
|
||||
///
|
||||
/// Returns the raw attribute text (no entity decoding) or `None` if
|
||||
/// the attribute isn't present. Empty string for `name=""` is
|
||||
/// represented as `Some("")`.
|
||||
pub fn attr(element: &str, name: &str) -> Option<String> {
|
||||
let bytes = element.as_bytes();
|
||||
let name_lower = name.to_ascii_lowercase();
|
||||
let name_bytes = name_lower.as_bytes();
|
||||
let mut i = 0;
|
||||
while i + name_bytes.len() < bytes.len() {
|
||||
// Find the next position where `name=` could start. We need
|
||||
// a word boundary before the name (whitespace or `<` or `:`).
|
||||
if i > 0 && is_name_char(bytes[i - 1]) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if !slice_eq_ignore_case(&bytes[i..i + name_bytes.len()], name_bytes) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
let after_name = i + name_bytes.len();
|
||||
// The character immediately after the name must not be a
|
||||
// name-continuation (otherwise we matched a prefix like
|
||||
// `lang_id` when looking for `lang`).
|
||||
if after_name < bytes.len() && is_name_char(bytes[after_name]) {
|
||||
i = after_name;
|
||||
continue;
|
||||
}
|
||||
// Walk past whitespace, then `=`, then more whitespace, then
|
||||
// the opening quote.
|
||||
let mut j = after_name;
|
||||
while j < bytes.len() && is_ws(bytes[j]) {
|
||||
j += 1;
|
||||
}
|
||||
if j >= bytes.len() || bytes[j] != b'=' {
|
||||
i = j.max(i + 1);
|
||||
continue;
|
||||
}
|
||||
j += 1; // past '='
|
||||
while j < bytes.len() && is_ws(bytes[j]) {
|
||||
j += 1;
|
||||
}
|
||||
if j >= bytes.len() {
|
||||
return None;
|
||||
}
|
||||
let quote = bytes[j];
|
||||
if quote != b'"' && quote != b'\'' {
|
||||
// Unquoted attribute values aren't part of well-formed
|
||||
// XML (HTML5 allows them, XML doesn't). Skip.
|
||||
i = j;
|
||||
continue;
|
||||
}
|
||||
let value_start = j + 1;
|
||||
let close = bytes[value_start..].iter().position(|&b| b == quote)?;
|
||||
let value = &element[value_start..value_start + close];
|
||||
return Some(value.to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Extract the trimmed text content of the first occurrence of
|
||||
/// `<tag>...</tag>` in `xml`. Returns `None` if the tag isn't found
|
||||
/// or its opening tag is malformed.
|
||||
///
|
||||
/// Whitespace around the inner text is stripped. Self-closing
|
||||
/// `<tag />` yields `Some("")`. Nested same-name tags are NOT
|
||||
/// handled — the first close encountered wins (this matches the
|
||||
/// prior behavior in criterion.rs).
|
||||
pub fn text(xml: &str, tag: &str) -> Option<String> {
|
||||
let (open_end, body_start) = find_open_tag(xml, tag, 0)?;
|
||||
// Self-closing — already consumed in find_open_tag if `/>`.
|
||||
if open_end == body_start {
|
||||
// Means find_open_tag returned the same offset twice for
|
||||
// self-closing form. (Not currently the case in our impl,
|
||||
// but defensive.)
|
||||
return Some(String::new());
|
||||
}
|
||||
// For self-closing tags, body_start is past `/>` and we have no
|
||||
// content. Detect by checking the char at body_start - 1 was `/`.
|
||||
if body_start >= 2 && &xml[body_start - 2..body_start] == "/>" {
|
||||
return Some(String::new());
|
||||
}
|
||||
// Find the matching close tag. Case-insensitive + namespace-aware.
|
||||
let close_start = find_close_tag(xml, tag, body_start)?;
|
||||
Some(xml[body_start..close_start].trim().to_string())
|
||||
}
|
||||
|
||||
/// Locate the next `<tag>` opening AND its closing `</tag>` in
|
||||
/// `xml`, starting at byte offset `from`. Returns `(element_start,
|
||||
/// element_end)` — `element_start` is the `<` of the opening tag,
|
||||
/// `element_end` is one past the `>` of the closing tag. Useful for
|
||||
/// iterating over repeated elements like `<playlist>` blocks in
|
||||
/// `paramount`.
|
||||
///
|
||||
/// Self-closing elements return the same offset for body_end as the
|
||||
/// element_end (i.e. `element_end - element_start` includes only the
|
||||
/// `<tag .../>` text).
|
||||
pub fn find_element(xml: &str, tag: &str, from: usize) -> Option<(usize, usize)> {
|
||||
let bytes = xml.as_bytes();
|
||||
let mut i = from;
|
||||
while i < bytes.len() {
|
||||
if bytes[i] != b'<' {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
// Try matching tag name at i+1 (after `<`).
|
||||
let after_lt = i + 1;
|
||||
if !matches_tag_name_at(bytes, after_lt, tag) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
// Found an open tag at offset i. Walk to find the closing `>`
|
||||
// of the open tag itself.
|
||||
let mut j = after_lt;
|
||||
// Skip past the tag name (and optional namespace prefix).
|
||||
while j < bytes.len() && (is_name_char(bytes[j]) || bytes[j] == b':') {
|
||||
j += 1;
|
||||
}
|
||||
// Walk attributes — track quoting state.
|
||||
let mut self_closing = false;
|
||||
while j < bytes.len() {
|
||||
match bytes[j] {
|
||||
b'>' => {
|
||||
j += 1;
|
||||
break;
|
||||
}
|
||||
b'/' if j + 1 < bytes.len() && bytes[j + 1] == b'>' => {
|
||||
self_closing = true;
|
||||
j += 2;
|
||||
break;
|
||||
}
|
||||
b'"' | b'\'' => {
|
||||
let q = bytes[j];
|
||||
j += 1;
|
||||
while j < bytes.len() && bytes[j] != q {
|
||||
j += 1;
|
||||
}
|
||||
if j < bytes.len() {
|
||||
j += 1;
|
||||
}
|
||||
}
|
||||
_ => j += 1,
|
||||
}
|
||||
}
|
||||
if self_closing {
|
||||
return Some((i, j));
|
||||
}
|
||||
// Find matching close. Doesn't handle nested same-name; OK
|
||||
// for our authoring-tool subset.
|
||||
let close_start = find_close_tag(xml, tag, j)?;
|
||||
let close_end = find_byte(bytes, b'>', close_start)? + 1;
|
||||
return Some((i, close_end));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
// ── Internal helpers ───────────────────────────────────────────────────────
|
||||
|
||||
/// True if `bytes[start..]` opens a tag named `tag`, allowing an
|
||||
/// optional `ns:` namespace prefix. Comparison is case-insensitive.
|
||||
/// The character after the tag name must not be a name-continuation
|
||||
/// (so `<player>` doesn't match `<play>`).
|
||||
fn matches_tag_name_at(bytes: &[u8], start: usize, tag: &str) -> bool {
|
||||
let tag_lower = tag.to_ascii_lowercase();
|
||||
let tag_bytes = tag_lower.as_bytes();
|
||||
// Skip optional `prefix:` (one or more name chars + `:`).
|
||||
let mut name_start = start;
|
||||
let mut scan = start;
|
||||
while scan < bytes.len() && is_name_char(bytes[scan]) {
|
||||
scan += 1;
|
||||
}
|
||||
if scan < bytes.len() && bytes[scan] == b':' {
|
||||
name_start = scan + 1;
|
||||
}
|
||||
if name_start + tag_bytes.len() > bytes.len() {
|
||||
return false;
|
||||
}
|
||||
if !slice_eq_ignore_case(&bytes[name_start..name_start + tag_bytes.len()], tag_bytes) {
|
||||
return false;
|
||||
}
|
||||
// Boundary: char after the tag name must be `>`, `/`, whitespace.
|
||||
let after = name_start + tag_bytes.len();
|
||||
if after >= bytes.len() {
|
||||
return false;
|
||||
}
|
||||
matches!(bytes[after], b'>' | b'/' | b' ' | b'\t' | b'\n' | b'\r')
|
||||
}
|
||||
|
||||
/// Find the offset of the next `</tag>` (or `</ns:tag>`) in `xml`
|
||||
/// starting at `from`. Case-insensitive; returns the offset of the
|
||||
/// `<`. None if not found.
|
||||
fn find_close_tag(xml: &str, tag: &str, from: usize) -> Option<usize> {
|
||||
let bytes = xml.as_bytes();
|
||||
let mut i = from;
|
||||
while i + 2 < bytes.len() {
|
||||
if bytes[i] == b'<' && bytes[i + 1] == b'/' {
|
||||
// Check tag name (with optional namespace).
|
||||
if matches_tag_name_at(bytes, i + 2, tag) {
|
||||
return Some(i);
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Find the open tag of `<tag>` in `xml` starting at `from`. Returns
|
||||
/// `(after_open_lt, after_open_gt)` — the offsets are: just past
|
||||
/// the `<` of the open tag, and just past the `>` of the open tag.
|
||||
fn find_open_tag(xml: &str, tag: &str, from: usize) -> Option<(usize, usize)> {
|
||||
let bytes = xml.as_bytes();
|
||||
let (elem_start, _) = find_element(xml, tag, from)?;
|
||||
let after_lt = elem_start + 1;
|
||||
// Find the `>` that ends the open tag (handling quoted attrs).
|
||||
let mut j = after_lt;
|
||||
while j < bytes.len() {
|
||||
match bytes[j] {
|
||||
b'>' => return Some((after_lt, j + 1)),
|
||||
b'/' if j + 1 < bytes.len() && bytes[j + 1] == b'>' => {
|
||||
return Some((after_lt, j + 2));
|
||||
}
|
||||
b'"' | b'\'' => {
|
||||
let q = bytes[j];
|
||||
j += 1;
|
||||
while j < bytes.len() && bytes[j] != q {
|
||||
j += 1;
|
||||
}
|
||||
if j < bytes.len() {
|
||||
j += 1;
|
||||
}
|
||||
}
|
||||
_ => j += 1,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn find_byte(bytes: &[u8], target: u8, from: usize) -> Option<usize> {
|
||||
bytes[from..]
|
||||
.iter()
|
||||
.position(|&b| b == target)
|
||||
.map(|p| p + from)
|
||||
}
|
||||
|
||||
/// True if `c` can be part of an XML name token (rough). We accept
|
||||
/// alphanumerics, `_`, `-`, `.`.
|
||||
fn is_name_char(c: u8) -> bool {
|
||||
c.is_ascii_alphanumeric() || c == b'_' || c == b'-' || c == b'.'
|
||||
}
|
||||
|
||||
fn is_ws(c: u8) -> bool {
|
||||
matches!(c, b' ' | b'\t' | b'\n' | b'\r')
|
||||
}
|
||||
|
||||
fn slice_eq_ignore_case(a: &[u8], b_lower: &[u8]) -> bool {
|
||||
if a.len() != b_lower.len() {
|
||||
return false;
|
||||
}
|
||||
a.iter()
|
||||
.zip(b_lower.iter())
|
||||
.all(|(&x, &y)| x.to_ascii_lowercase() == y)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn attr_basic() {
|
||||
assert_eq!(
|
||||
attr(r#"<playlist name="Feature" id="00222" />"#, "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
assert_eq!(
|
||||
attr(r#"<playlist name="Feature" id="00222" />"#, "id"),
|
||||
Some("00222".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_case_insensitive_name() {
|
||||
assert_eq!(
|
||||
attr(r#"<playlist Name="Feature" />"#, "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
assert_eq!(
|
||||
attr(r#"<playlist NAME="Feature" />"#, "Name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_accepts_single_quotes() {
|
||||
assert_eq!(
|
||||
attr(r#"<playlist name='Feature' />"#, "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_whitespace_around_equals() {
|
||||
assert_eq!(
|
||||
attr(r#"<playlist name = "Feature" />"#, "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
assert_eq!(
|
||||
attr("<playlist name\n=\n\"Feature\" />", "name"),
|
||||
Some("Feature".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_missing_returns_none() {
|
||||
assert_eq!(attr(r#"<playlist name="X" />"#, "id"), None);
|
||||
assert_eq!(attr("", "name"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_no_substring_false_positive() {
|
||||
// Looking for "lang" should NOT match "lang_id" or
|
||||
// "language" because of the name-char boundary check.
|
||||
assert_eq!(attr(r#"<x lang_id="fra" language="eng" />"#, "lang"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attr_empty_value() {
|
||||
assert_eq!(attr(r#"<x name="" id="1" />"#, "name"), Some("".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_basic() {
|
||||
assert_eq!(text("<x>hello</x>", "x"), Some("hello".into()));
|
||||
assert_eq!(
|
||||
text("<x> hello world </x>", "x"),
|
||||
Some("hello world".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_case_insensitive_tag() {
|
||||
assert_eq!(text("<X>foo</X>", "x"), Some("foo".into()));
|
||||
assert_eq!(text("<Foo>bar</foo>", "foo"), Some("bar".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_namespace_prefix() {
|
||||
assert_eq!(text("<ns:tag>value</ns:tag>", "tag"), Some("value".into()));
|
||||
assert_eq!(text("<foo:Bar>v</foo:Bar>", "bar"), Some("v".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_self_closing() {
|
||||
assert_eq!(text("<x/>", "x"), Some("".into()));
|
||||
assert_eq!(text("<x />", "x"), Some("".into()));
|
||||
assert_eq!(text("<x attr=\"y\" />", "x"), Some("".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_with_attrs() {
|
||||
assert_eq!(
|
||||
text(r#"<x id="1" name="y">hello</x>"#, "x"),
|
||||
Some("hello".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_missing_close_returns_none() {
|
||||
assert_eq!(text("<x>hello", "x"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_skips_inner_tags_naively() {
|
||||
// Limitation noted: nested same-name tags aren't handled.
|
||||
// Different-name nesting works (we just return everything
|
||||
// between the open and close).
|
||||
assert_eq!(
|
||||
text("<x><y>nested</y></x>", "x"),
|
||||
Some("<y>nested</y>".into())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_basic() {
|
||||
let xml = r#"<x /> <y attr="1">body</y>"#;
|
||||
let (s, e) = find_element(xml, "y", 0).unwrap();
|
||||
assert_eq!(&xml[s..e], r#"<y attr="1">body</y>"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_self_closing() {
|
||||
let xml = r#"<x />"#;
|
||||
let (s, e) = find_element(xml, "x", 0).unwrap();
|
||||
assert_eq!(&xml[s..e], "<x />");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_handles_quoted_gt_in_attr() {
|
||||
// A `>` inside a quoted attribute value should not terminate
|
||||
// the open tag prematurely.
|
||||
let xml = r#"<x attr="foo>bar">body</x>"#;
|
||||
let (s, e) = find_element(xml, "x", 0).unwrap();
|
||||
assert_eq!(&xml[s..e], r#"<x attr="foo>bar">body</x>"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_iteration() {
|
||||
let xml = "<p>a</p><p>b</p><p>c</p>";
|
||||
let mut positions = Vec::new();
|
||||
let mut from = 0;
|
||||
while let Some((s, e)) = find_element(xml, "p", from) {
|
||||
positions.push(&xml[s..e]);
|
||||
from = e;
|
||||
}
|
||||
assert_eq!(positions, vec!["<p>a</p>", "<p>b</p>", "<p>c</p>"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_element_with_namespace() {
|
||||
let xml = r#"<root><ns:item id="1" /></root>"#;
|
||||
let (s, e) = find_element(xml, "item", 0).unwrap();
|
||||
assert_eq!(&xml[s..e], r#"<ns:item id="1" />"#);
|
||||
}
|
||||
}
|
||||
+59
-10
@@ -77,12 +77,16 @@ pub mod css;
|
||||
pub mod decrypt;
|
||||
pub mod disc;
|
||||
pub mod drive;
|
||||
pub mod drm;
|
||||
pub mod error;
|
||||
pub mod event;
|
||||
pub mod halt;
|
||||
pub(crate) mod identity;
|
||||
pub(crate) mod ifo;
|
||||
pub mod io;
|
||||
pub mod keydb;
|
||||
pub(crate) mod labels;
|
||||
pub mod keysource;
|
||||
pub mod labels;
|
||||
pub(crate) mod mpls;
|
||||
pub mod mux;
|
||||
pub mod pes;
|
||||
@@ -95,6 +99,9 @@ pub(crate) mod speed;
|
||||
pub(crate) mod udf;
|
||||
pub mod verify;
|
||||
|
||||
// Re-export verify types at the crate root for ergonomic imports.
|
||||
pub use verify::{SectorRange, SectorStatus, VerifyResult, verify_title};
|
||||
|
||||
// ─── Drive lifecycle ────────────────────────────────────────────────────────
|
||||
//
|
||||
// `Drive::open(path)` → `wait_ready()` → `init()` → `Disc::scan()`. `Drive`
|
||||
@@ -112,6 +119,31 @@ pub use drive::{Drive, DriveStatus, find_drive};
|
||||
// codes to localized messages. See `error.rs` for the full taxonomy.
|
||||
pub use error::{Error, Result};
|
||||
|
||||
// ─── 0.18 primitives ────────────────────────────────────────────────────────
|
||||
//
|
||||
// One-bit cooperative cancellation token, shared by every long-running loop
|
||||
// in libfreemkv (sweep, patch, mux). Replaces the ad-hoc `Arc<AtomicBool>`
|
||||
// flags scattered across 0.17 (`DiscStream::set_halt`, autorip's
|
||||
// `HALT_FLAGS` registry). Clone it cheaply; pass it by value into each
|
||||
// component; poll `is_cancelled()` inside the loop body.
|
||||
pub use halt::Halt;
|
||||
|
||||
// Generic bounded producer/consumer primitive used by sweep, patch, and
|
||||
// mux to overlap reads with writes via a dedicated consumer thread.
|
||||
// `Pipeline::spawn(name, depth, sink)` spawns a named consumer; `pipe.send(item)`
|
||||
// pushes one item with back-pressure; `pipe.finish()` joins the
|
||||
// consumer and surfaces its `close()` output. Callers implement `Sink`
|
||||
// to define per-item behaviour and end-of-stream finalisation.
|
||||
//
|
||||
// `DEFAULT_PIPELINE_DEPTH` (=4) is for callers without specific needs;
|
||||
// most should use READ_PIPELINE_DEPTH or WRITE_PIPELINE_DEPTH instead.
|
||||
// Patch uses `WRITE_THROUGH_DEPTH` (=1). Returning `Flow::Stop` from
|
||||
// `apply` ends the consumer cleanly (still calls `close()`).
|
||||
pub use io::pipeline::{
|
||||
DEFAULT_PIPELINE_DEPTH, Flow, Pipeline, READ_PIPELINE_DEPTH, Sink, WRITE_PIPELINE_DEPTH,
|
||||
WRITE_THROUGH_DEPTH,
|
||||
};
|
||||
|
||||
// ─── Drive events (low-level callbacks) ─────────────────────────────────────
|
||||
pub use event::{Event, EventKind};
|
||||
pub use identity::DriveId;
|
||||
@@ -124,7 +156,7 @@ pub use profile::DriveProfile;
|
||||
// don't touch `DecryptKeys` directly — `DiscStream::new(reader, title, keys, …)`
|
||||
// accepts whatever `Disc::decrypt_keys()` returned. `decrypt_sectors()` is
|
||||
// for callers that operate on raw sector buffers (e.g. ISO patching).
|
||||
pub use decrypt::{DecryptKeys, decrypt_sectors};
|
||||
pub use decrypt::{DecryptKeys, decrypt_sectors, decrypt_threads, set_decrypt_threads};
|
||||
|
||||
// ─── Disc structure ─────────────────────────────────────────────────────────
|
||||
//
|
||||
@@ -133,19 +165,24 @@ pub use decrypt::{DecryptKeys, decrypt_sectors};
|
||||
// for displaying disc name + format quickly while a full scan runs in the
|
||||
// background. The codec / channel / resolution enums are the canonical
|
||||
// structured representation; never compare against display strings.
|
||||
// Note: `disc::Stream` here is the codec enum (audio / video / sub kind)
|
||||
// — not the `pes::Stream` trait re-exported below as `PesStream`. Two
|
||||
// different concepts, the same short name; the trait gets the `Pes`
|
||||
// prefix at the crate root to keep both addressable.
|
||||
pub use disc::{
|
||||
AacsState, AudioChannels, AudioStream, Clip, Codec, ColorSpace, ContentFormat, DamageSeverity,
|
||||
Disc, DiscFormat, DiscId, DiscTitle, Extent, FrameRate, HdrFormat, KeySource, LabelPurpose,
|
||||
LabelQualifier, Resolution, SampleRate, ScanOptions, Stream, SubtitleStream, VideoStream,
|
||||
classify_damage,
|
||||
Disc, DiscFormat, DiscId, DiscTitle, DriveCredentials, Extent, FrameRate, HdrFormat, Key,
|
||||
KeyOrigin, LabelPurpose, LabelQualifier, PatchOptions, PatchOutcome, Resolution, SampleRate,
|
||||
ScanOptions, Stream, SubtitleStream, SweepOptions, VideoStream, classify_damage,
|
||||
};
|
||||
pub use keysource::{DiscInputs, KeySource};
|
||||
|
||||
// ─── Streams ────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// All stream types implement `pes::Stream` — read PES frames from a source,
|
||||
// write PES frames to a sink. Pick the right type at construction:
|
||||
//
|
||||
// - `DiscStream` — physical drive or ISO (any `SectorReader`). Always read.
|
||||
// - `DiscStream` — physical drive or ISO (any `SectorSource`). Read-only.
|
||||
// - `MkvStream` — Matroska container. Read on `open()`, write on `create()`.
|
||||
// - `M2tsStream` — Blu-ray Transport Stream. Read on `open()`, write on `create()`.
|
||||
// - `NetworkStream` — TCP. Read on `listen()`, write on `connect()`.
|
||||
@@ -155,6 +192,12 @@ pub use disc::{
|
||||
// Most consumers use the URL resolvers (`input()` / `output()`) which pick
|
||||
// the right type from a scheme:// URL. Direct construction is for callers
|
||||
// that need to wire custom readers (e.g. autorip's drive-session reuse).
|
||||
// The trait is re-exported as `PesStream` here to disambiguate from
|
||||
// `disc::Stream` (the codec-kind enum re-exported above), which would
|
||||
// otherwise collide at the crate root.
|
||||
pub use pes::PesFrame;
|
||||
pub use pes::Stream as PesStream;
|
||||
|
||||
pub use mux::DiscStream;
|
||||
pub use mux::M2tsStream;
|
||||
pub use mux::MkvStream;
|
||||
@@ -166,10 +209,16 @@ pub use mux::{InputOptions, StreamUrl, input, output, parse_url};
|
||||
// ─── Lower-level surfaces ───────────────────────────────────────────────────
|
||||
//
|
||||
// `ScsiTransport` is the platform-abstraction trait Drive uses; expose for
|
||||
// out-of-tree platform backends. `SectorReader` lets callers feed any byte
|
||||
// source (test harness, network image, SMB share) into the disc scan
|
||||
// pipeline; `FileSectorReader` is the standard ISO-on-disk implementation.
|
||||
// out-of-tree platform backends. `SectorSource` / `SectorSink` are the
|
||||
// direction-typed read/write traits; `FileSectorSource` and `FileSectorSink`
|
||||
// are the ISO-on-disk implementations. [`DecryptingSectorSource`] is the
|
||||
// single decrypt-on-read decorator (AACS / CSS / none) — wrap any
|
||||
// `SectorSource` to get plaintext sectors out.
|
||||
pub use mux::build_iso_pipeline;
|
||||
pub use scsi::{DriveInfo, ScsiSense, ScsiTransport, drive_has_disc, list_drives};
|
||||
pub use sector::{FileSectorReader, SectorReader};
|
||||
pub use sector::{
|
||||
DecryptingSectorSource, FileSectorSink, FileSectorSource, PrefetchedSectorSource, SectorSink,
|
||||
SectorSource,
|
||||
};
|
||||
pub use speed::DriveSpeed;
|
||||
pub use udf::{UdfFs, read_filesystem};
|
||||
|
||||
+15
-3
@@ -301,9 +301,21 @@ fn parse_stream_entry(item: &[u8], pos: usize, stream_type: u8) -> Option<(Strea
|
||||
return None;
|
||||
}
|
||||
|
||||
// PID from stream entry (type 0x01 = PlayItem stream: PID at bytes 2-3)
|
||||
let pid = if item[pos + 1] == 0x01 && pos + 4 <= item.len() {
|
||||
u16::from_be_bytes([item[pos + 2], item[pos + 3]])
|
||||
// PID location depends on the stream-entry type (BD spec stream_entry()):
|
||||
// type 1 (stream in the PlayItem's Clip): PID at +2
|
||||
// type 2 (stream in a SubPath SubClip): +subpath_id(1)+subclip_id(1) → PID at +4
|
||||
// type 3 / 4 (SubPath clip; type 4 = Dolby Vision +subpath_id(1) → PID at +3
|
||||
// enhancement layer, e.g. PID 0x1015):
|
||||
// Previously only type 1 was handled, so the DV EL (type 4) and any
|
||||
// sub-path stream fell through to PID 0 and were dropped by the mux.
|
||||
let pid_off = match item[pos + 1] {
|
||||
0x01 => 2,
|
||||
0x02 => 4,
|
||||
0x03 | 0x04 => 3,
|
||||
_ => 0,
|
||||
};
|
||||
let pid = if pid_off != 0 && pos + pid_off + 2 <= item.len() {
|
||||
u16::from_be_bytes([item[pos + pid_off], item[pos + pid_off + 1]])
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
+120
-7
@@ -6,6 +6,14 @@
|
||||
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
/// Hard cap on the carry-over buffer. An AC-3/E-AC-3 frame is at most 8192
|
||||
/// bytes (the `frame_size > 8192` reject below), so a single straddling frame
|
||||
/// plus a little slack never needs more than this. If the buffer grows past
|
||||
/// the cap without yielding a frame (pathological / never-syncing input) we
|
||||
/// drop it and resync rather than accumulate one PES worth of data per call
|
||||
/// for the whole title.
|
||||
const MAX_AC3_BUF: usize = 64 * 1024;
|
||||
|
||||
pub struct Ac3Parser {
|
||||
/// Leftover bytes from previous PES (incomplete frame at end).
|
||||
buf: Vec<u8>,
|
||||
@@ -76,23 +84,43 @@ impl CodecParser for Ac3Parser {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: data[start..start + frame_size].to_vec(),
|
||||
duration_ns: None,
|
||||
});
|
||||
pos = start + frame_size;
|
||||
}
|
||||
|
||||
// Keep unconsumed data for next call
|
||||
// `pos` points to the start of unconsumed data (either a partial sync or leftover)
|
||||
// Keep unconsumed data for the next call. `pos` is the start of the
|
||||
// unconsumed region: either a partial frame that straddles this PES
|
||||
// boundary — which, by construction, begins at a syncword (every byte
|
||||
// before `pos` was emitted as a frame or skipped as pre-sync junk) — or
|
||||
// trailing bytes too short to size/complete a frame. Carry from `pos`,
|
||||
// NOT from the next syncword: discarding bytes between `pos` and the
|
||||
// next sync would drop the partial frame we are deliberately keeping
|
||||
// across the boundary.
|
||||
let keep_from = if pos < data.len() {
|
||||
// Find the last sync word position in the unconsumed region
|
||||
find_ac3_sync(&data[pos..])
|
||||
.map(|o| pos + o)
|
||||
.unwrap_or(data.len())
|
||||
// A syncword at/after `pos` marks the carry-over start (anything
|
||||
// before it is junk with no sync). With no full sync, retain the
|
||||
// whole tail — including a lone trailing 0x0B that may be the first
|
||||
// half of a syncword split across the PES boundary.
|
||||
match find_ac3_sync(&data[pos..]) {
|
||||
Some(o) => pos + o,
|
||||
None if data.last() == Some(&0x0B) => data.len() - 1,
|
||||
None => data.len(),
|
||||
}
|
||||
} else {
|
||||
data.len()
|
||||
};
|
||||
|
||||
if keep_from < data.len() {
|
||||
self.buf = data[keep_from..].to_vec();
|
||||
let tail = &data[keep_from..];
|
||||
if tail.len() > MAX_AC3_BUF {
|
||||
// No frame could be parsed out of a buffer this large — this is
|
||||
// not valid AC-3 here. Drop it and resync on the next PES rather
|
||||
// than grow without bound on pathological input.
|
||||
self.buf.clear();
|
||||
} else {
|
||||
self.buf = tail.to_vec();
|
||||
}
|
||||
} else {
|
||||
self.buf.clear();
|
||||
}
|
||||
@@ -275,6 +303,91 @@ mod tests {
|
||||
assert_eq!(frames[0].data.len(), 160);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_word_split_across_pes_is_preserved() {
|
||||
// A frame whose 0x0B77 syncword straddles the PES boundary (0x0B at the
|
||||
// tail of PES 1, 0x77 at the head of PES 2) must still be emitted whole.
|
||||
// Previously the lone trailing 0x0B was dropped and the frame lost.
|
||||
let mut parser = Ac3Parser::new();
|
||||
let frame_data = make_ac3_frame(0, 2); // 160 bytes, starts with 0x0B 0x77
|
||||
|
||||
// PES 1: a complete frame, then a single 0x0B (first half of next sync).
|
||||
let mut pes1_data = frame_data.clone();
|
||||
pes1_data.push(0x0B);
|
||||
let pes1 = PesPacket {
|
||||
pid: 0,
|
||||
pts: Some(90000),
|
||||
dts: None,
|
||||
data: pes1_data,
|
||||
};
|
||||
let frames1 = parser.parse(&pes1);
|
||||
assert_eq!(frames1.len(), 1, "first complete frame emitted");
|
||||
|
||||
// PES 2: 0x77 (second half of sync) + rest of the second frame.
|
||||
let mut pes2_data = vec![0x77];
|
||||
pes2_data.extend_from_slice(&frame_data[2..]);
|
||||
let pes2 = PesPacket {
|
||||
pid: 0,
|
||||
pts: Some(93000),
|
||||
dts: None,
|
||||
data: pes2_data,
|
||||
};
|
||||
let frames2 = parser.parse(&pes2);
|
||||
assert_eq!(frames2.len(), 1, "split-sync frame must be recovered");
|
||||
assert_eq!(frames2[0].data.len(), 160);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn buffer_stays_bounded_across_many_garbage_pes() {
|
||||
// Finding 14: the carry-over buffer must never grow without bound. Feed
|
||||
// many large PES packets that contain no usable frame and assert the
|
||||
// retained buffer stays tiny — carry-from-`pos` drops all pre-sync junk,
|
||||
// and a never-completing frame is bounded by the 8192-byte frame cap and
|
||||
// the MAX_AC3_BUF resync guard.
|
||||
let mut parser = Ac3Parser::new();
|
||||
for i in 0..256 {
|
||||
// Vary the trailing byte so we also exercise the lone-0x0B retain.
|
||||
let mut data = vec![0x55u8; 8192];
|
||||
if i % 3 == 0 {
|
||||
*data.last_mut().unwrap() = 0x0B;
|
||||
}
|
||||
let pes = PesPacket {
|
||||
pid: 0,
|
||||
pts: None,
|
||||
dts: None,
|
||||
data,
|
||||
};
|
||||
let frames = parser.parse(&pes);
|
||||
assert!(frames.is_empty());
|
||||
assert!(
|
||||
parser.buf.len() <= MAX_AC3_BUF,
|
||||
"buffer grew to {} (cap {})",
|
||||
parser.buf.len(),
|
||||
MAX_AC3_BUF
|
||||
);
|
||||
}
|
||||
// After all that garbage the retained tail is at most a single partial
|
||||
// syncword byte — never an accumulation of whole PES packets.
|
||||
assert!(parser.buf.len() <= 1, "retained {} bytes", parser.buf.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_sync_below_cap_is_still_retained() {
|
||||
// The cap must not break the normal split-sync straddle: a short tail
|
||||
// ending in 0x0B (well under the cap) is retained so the next PES can
|
||||
// complete the syncword.
|
||||
let mut parser = Ac3Parser::new();
|
||||
let data = vec![0x00, 0x00, 0x0B];
|
||||
let pes = PesPacket {
|
||||
pid: 0,
|
||||
pts: None,
|
||||
dts: None,
|
||||
data,
|
||||
};
|
||||
assert!(parser.parse(&pes).is_empty());
|
||||
assert_eq!(parser.buf, vec![0x0B], "lone trailing 0x0B retained");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ac3_frame_size_table() {
|
||||
// fscod=0 (48kHz), frmsizecod=0: 64 words = 128 bytes
|
||||
|
||||
+410
-77
@@ -8,10 +8,20 @@
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
const DTS_CORE_SYNC: [u8; 4] = [0x7F, 0xFE, 0x80, 0x01];
|
||||
/// DTS-HD extension substream syncword. The parser delimits an access unit by
|
||||
/// the next CORE sync (so every extension between two cores is captured), and
|
||||
/// never needs to locate or size the extension itself — so this is referenced
|
||||
/// only by the tests that synthesize extension substreams.
|
||||
#[cfg(test)]
|
||||
const DTS_HD_EXT_SYNC: [u8; 4] = [0x64, 0x58, 0x20, 0x25];
|
||||
|
||||
pub struct DtsParser {
|
||||
buf: Vec<u8>,
|
||||
/// PTS of the access unit currently being assembled in `buf` (the unit
|
||||
/// starting at the first buffered core sync). Captured when that core
|
||||
/// frame's PES first arrived; the trailing extension-substream PES
|
||||
/// packets carry their own (later) PTS which must NOT override it.
|
||||
pending_pts: i64,
|
||||
}
|
||||
|
||||
impl Default for DtsParser {
|
||||
@@ -24,10 +34,29 @@ impl DtsParser {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
buf: Vec::with_capacity(32768),
|
||||
pending_pts: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hard cap on a buffered access unit (core + all its extension substreams).
|
||||
/// A DTS-HD MA frame is at most a few tens of KB; if the buffer grows past
|
||||
/// this without a clean boundary we resync rather than stall or balloon.
|
||||
const MAX_AU_BYTES: usize = 65536;
|
||||
|
||||
/// Minimum plausible DTS core frame size. The core header alone is ~10-14
|
||||
/// bytes; a decoded `core_size` below this means we matched a false/corrupt
|
||||
/// core sync (the 14-bit `fsize` field decoded to a tiny value) rather than a
|
||||
/// real frame, so we resync instead of emitting a junk access unit.
|
||||
const MIN_CORE_FRAME_BYTES: usize = 10;
|
||||
|
||||
/// Sentinel for "no valid PTS base captured yet". Real PTS-in-ns values are
|
||||
/// non-negative (derived from the unsigned 90 kHz PES timestamp), so a negative
|
||||
/// value can never collide with a genuine timestamp. Used to mark the PTS base
|
||||
/// invalid after a forced flush so the next PES sets it regardless of buffer
|
||||
/// state.
|
||||
const PTS_UNSET: i64 = -1;
|
||||
|
||||
impl CodecParser for DtsParser {
|
||||
fn parse(&mut self, pes: &PesPacket) -> Vec<Frame> {
|
||||
if pes.data.is_empty() {
|
||||
@@ -35,79 +64,162 @@ impl CodecParser for DtsParser {
|
||||
}
|
||||
let pts_ns = pes.pts.map(pts_to_ns).unwrap_or(0);
|
||||
|
||||
// On Blu-ray, a DTS-HD MA/HRA access unit is a DTS core frame
|
||||
// (sync 0x7FFE8001) followed by one or more DTS extension substreams
|
||||
// (sync 0x64582025). The m2ts demuxer hands those out as SEPARATE PES
|
||||
// packets on the same PID — the core in one PES, then the extension
|
||||
// substreams in following PES packets (with their own, later PTS). The
|
||||
// lossless audio lives entirely in the extension substreams, so an
|
||||
// access unit is only complete once all of its trailing extensions
|
||||
// have been buffered. We assemble across PES boundaries here: an access
|
||||
// unit runs from its core sync up to (but not including) the NEXT core
|
||||
// sync. Emitting on the core boundary keeps the core + every following
|
||||
// extension substream together (the lossless data), instead of the
|
||||
// old per-PES emit that dropped the extension PES packets and
|
||||
// downgraded the track to lossy DTS core (the Dunkirk / Fight Club
|
||||
// bug). The PTS is the core frame's PTS, captured when the unit began.
|
||||
// Capture the access unit's PTS base on a fresh buffer, or whenever a
|
||||
// prior forced (safety-valve) flush left it invalidated — in the
|
||||
// forced case the bytes still in `buf` are not a real core frame, so
|
||||
// the first PES to arrive after the flush carries the correct base.
|
||||
if self.buf.is_empty() || self.pending_pts == PTS_UNSET {
|
||||
self.pending_pts = pts_ns;
|
||||
}
|
||||
self.buf.extend_from_slice(&pes.data);
|
||||
|
||||
let data = &self.buf;
|
||||
let mut frames = Vec::new();
|
||||
let mut pos = 0;
|
||||
|
||||
while pos < data.len() {
|
||||
// Find DTS core sync
|
||||
let start = match find_sync(&data[pos..], &DTS_CORE_SYNC) {
|
||||
Some(offset) => pos + offset,
|
||||
None => break,
|
||||
loop {
|
||||
// Resync to the first core sync; drop any leading junk.
|
||||
let Some(start) = find_sync(&self.buf, &DTS_CORE_SYNC) else {
|
||||
// No core sync at all yet — keep at most a 3-byte tail so a
|
||||
// sync split across PES packets can still be found next time.
|
||||
if self.buf.len() > 3 {
|
||||
let tail = self.buf.len() - 3;
|
||||
self.buf.drain(..tail);
|
||||
}
|
||||
break;
|
||||
};
|
||||
|
||||
// Need at least 10 bytes for core header to get frame size
|
||||
if start + 10 > data.len() {
|
||||
break;
|
||||
}
|
||||
|
||||
let core_size = dts_core_frame_size(&data[start..]);
|
||||
if core_size == 0 || core_size > 32768 {
|
||||
pos = start + 4;
|
||||
continue;
|
||||
}
|
||||
|
||||
if start + core_size > data.len() {
|
||||
// Incomplete core frame
|
||||
break;
|
||||
}
|
||||
|
||||
// Check for DTS-HD extension after core
|
||||
let mut total_size = core_size;
|
||||
if start + core_size + 4 <= data.len() {
|
||||
if let Some(0) = find_sync(
|
||||
&data[start + core_size..start + core_size + 4],
|
||||
&DTS_HD_EXT_SYNC,
|
||||
) {
|
||||
let ext = &data[start + core_size..];
|
||||
if ext.len() >= 9 {
|
||||
let ext_size = dts_hd_ext_frame_size(ext);
|
||||
if start + core_size + ext_size <= data.len() {
|
||||
total_size = core_size + ext_size;
|
||||
}
|
||||
// If ext incomplete, just emit core
|
||||
}
|
||||
if start > 0 {
|
||||
self.buf.drain(..start);
|
||||
if find_sync(&self.buf, &DTS_CORE_SYNC) != Some(0) {
|
||||
// Shouldn't happen, but never loop forever.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Need the core header to size the core frame.
|
||||
if self.buf.len() < 10 {
|
||||
break;
|
||||
}
|
||||
let core_size = dts_core_frame_size(&self.buf);
|
||||
// `dts_core_frame_size` returns a 14-bit `fsize + 1`, so it is
|
||||
// always in [1, 16384]; the bare `== 0` / `> MAX_AU_BYTES` checks
|
||||
// can never fire. A real DTS core header is at least ~10-14 bytes,
|
||||
// so any decoded size below that came from a false/corrupt sync.
|
||||
// Reject it (drain the 4 syncword bytes and resync) instead of
|
||||
// letting a tiny bogus size close the current access unit at a junk
|
||||
// boundary and drop the trailing extension substreams. The
|
||||
// `> MAX_AU_BYTES` upper bound is kept as a harmless guard.
|
||||
if !(MIN_CORE_FRAME_BYTES..=MAX_AU_BYTES).contains(&core_size) {
|
||||
// Bogus core sync — skip past it and resync.
|
||||
self.buf.drain(..4);
|
||||
continue;
|
||||
}
|
||||
if self.buf.len() < core_size {
|
||||
break; // core frame not fully buffered yet — wait
|
||||
}
|
||||
|
||||
// The access unit ends at the next *valid* core sync. The search
|
||||
// begins after this core's syncword so we don't re-match it.
|
||||
// Anything between the core and that next sync is this unit's
|
||||
// extension substream(s) — which can themselves contain byte
|
||||
// sequences matching the core syncword, so a raw `find_sync` match
|
||||
// is not enough: a candidate is only a real boundary if its decoded
|
||||
// core size is plausible. `next_core_boundary` skips bogus matches.
|
||||
//
|
||||
// `forced` distinguishes a real next-core boundary from a forced
|
||||
// safety-valve flush. On a forced flush the access unit was NOT
|
||||
// closed by a new core sync, so the bytes following it are not a
|
||||
// fresh core frame and the current PES's PTS (which on a forced
|
||||
// flush is an extension-substream PES, carrying its own later
|
||||
// timestamp) must NOT become the next unit's PTS base.
|
||||
let mut forced = false;
|
||||
let au_end = match next_core_boundary(&self.buf, core_size) {
|
||||
NextCore::Found(end) => end,
|
||||
NextCore::NeedMore => break, // candidate sync needs more header
|
||||
NextCore::None => {
|
||||
// No next core sync buffered yet. The trailing extension
|
||||
// substream PES packets may still be arriving, so WAIT for
|
||||
// them rather than emit a core-only (lossy) frame — unless
|
||||
// the buffer has grown unreasonably large, in which case
|
||||
// emit what we have to guarantee forward progress.
|
||||
if self.buf.len() <= MAX_AU_BYTES {
|
||||
break;
|
||||
}
|
||||
forced = true;
|
||||
self.buf.len()
|
||||
}
|
||||
};
|
||||
|
||||
let au: Vec<u8> = self.buf[..au_end].to_vec();
|
||||
frames.push(Frame {
|
||||
pts_ns,
|
||||
pts_ns: self.pending_pts,
|
||||
keyframe: true,
|
||||
data: data[start..start + total_size].to_vec(),
|
||||
data: au,
|
||||
duration_ns: None,
|
||||
});
|
||||
pos = start + total_size;
|
||||
}
|
||||
|
||||
// Keep unconsumed data
|
||||
let keep_from = if pos < data.len() {
|
||||
find_sync(&data[pos..], &DTS_CORE_SYNC)
|
||||
.map(|o| pos + o)
|
||||
.unwrap_or(data.len())
|
||||
} else {
|
||||
data.len()
|
||||
};
|
||||
|
||||
if keep_from < data.len() {
|
||||
self.buf = data[keep_from..].to_vec();
|
||||
} else {
|
||||
self.buf.clear();
|
||||
self.buf.drain(..au_end);
|
||||
if forced {
|
||||
// Safety-valve flush: the next access unit's real core PES has
|
||||
// not arrived. Invalidate the PTS so the next PES sets it
|
||||
// regardless of buffer state, rather than inheriting this
|
||||
// (non-core) PES's timestamp.
|
||||
self.pending_pts = PTS_UNSET;
|
||||
} else {
|
||||
// Real boundary: the next access unit (now at buf start) begins
|
||||
// at a core sync that arrived inside this PES, so this PES's PTS
|
||||
// is the correct base for it.
|
||||
self.pending_pts = pts_ns;
|
||||
}
|
||||
}
|
||||
|
||||
frames
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> Vec<Frame> {
|
||||
// End of stream: emit the final access unit still buffered (the last
|
||||
// core + its extension substreams, which had no following core sync to
|
||||
// close it during streaming). Require a complete core frame; drop a
|
||||
// bare partial sync tail.
|
||||
if find_sync(&self.buf, &DTS_CORE_SYNC) != Some(0) || self.buf.len() < 10 {
|
||||
self.buf.clear();
|
||||
return Vec::new();
|
||||
}
|
||||
let core_size = dts_core_frame_size(&self.buf);
|
||||
// `dts_core_frame_size` returns a 14-bit `fsize + 1` (never 0), so the
|
||||
// old `== 0` check was dead; reject a sub-minimum core like `parse()`.
|
||||
if core_size < MIN_CORE_FRAME_BYTES || self.buf.len() < core_size {
|
||||
self.buf.clear();
|
||||
return Vec::new();
|
||||
}
|
||||
let au = std::mem::take(&mut self.buf);
|
||||
// A non-empty buffer here means a PES arrived after any prior forced
|
||||
// flush (which fully drains `buf`), so `pending_pts` was reset to that
|
||||
// PES's real PTS. Clamp the sentinel to 0 defensively all the same.
|
||||
let pts_ns = if self.pending_pts == PTS_UNSET {
|
||||
0
|
||||
} else {
|
||||
self.pending_pts
|
||||
};
|
||||
vec![Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: au,
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
fn codec_private(&self) -> Option<Vec<u8>> {
|
||||
None
|
||||
}
|
||||
@@ -120,6 +232,40 @@ fn find_sync(data: &[u8], pattern: &[u8; 4]) -> Option<usize> {
|
||||
(0..=data.len() - 4).find(|&i| data[i..i + 4] == *pattern)
|
||||
}
|
||||
|
||||
/// Result of scanning for the next valid core sync that closes an access unit.
|
||||
enum NextCore {
|
||||
/// A valid next core sync was found; the access unit ends at this offset.
|
||||
Found(usize),
|
||||
/// A candidate core sync was found but its header isn't fully buffered yet,
|
||||
/// so its validity can't be decided — wait for more data.
|
||||
NeedMore,
|
||||
/// No (further) core sync found in the buffer.
|
||||
None,
|
||||
}
|
||||
|
||||
/// Find the next *valid* core sync after the current core frame, to delimit the
|
||||
/// access unit. Extension-substream payload can contain byte sequences that
|
||||
/// match the core syncword, so each candidate is validated by decoding its
|
||||
/// core size: a match whose decoded size is implausible (< MIN_CORE_FRAME_BYTES
|
||||
/// or > MAX_AU_BYTES) is a false sync and is skipped, continuing the search.
|
||||
fn next_core_boundary(buf: &[u8], core_size: usize) -> NextCore {
|
||||
let mut from = core_size;
|
||||
while let Some(rel) = find_sync(&buf[from..], &DTS_CORE_SYNC) {
|
||||
let pos = from + rel;
|
||||
// Need the candidate's core header to judge it.
|
||||
if buf.len() - pos < 10 {
|
||||
return NextCore::NeedMore;
|
||||
}
|
||||
let sz = dts_core_frame_size(&buf[pos..]);
|
||||
if (MIN_CORE_FRAME_BYTES..=MAX_AU_BYTES).contains(&sz) {
|
||||
return NextCore::Found(pos);
|
||||
}
|
||||
// False sync inside extension payload — skip it and keep searching.
|
||||
from = pos + 4;
|
||||
}
|
||||
NextCore::None
|
||||
}
|
||||
|
||||
/// DTS core frame size from header bits.
|
||||
/// fsize is at bits 46-59 (14 bits) of the header: bytes 5-7.
|
||||
fn dts_core_frame_size(data: &[u8]) -> usize {
|
||||
@@ -133,20 +279,6 @@ fn dts_core_frame_size(data: &[u8]) -> usize {
|
||||
fsize + 1
|
||||
}
|
||||
|
||||
/// DTS-HD extension frame size from extension header.
|
||||
pub fn dts_hd_ext_frame_size(ext: &[u8]) -> usize {
|
||||
if ext.len() < 9 {
|
||||
return 0;
|
||||
}
|
||||
let raw =
|
||||
((ext[6] as usize & 0x1F) << 11) | ((ext[7] as usize) << 3) | ((ext[8] as usize) >> 5);
|
||||
raw + 1
|
||||
}
|
||||
|
||||
pub fn find_dts_hd_ext_sync(data: &[u8]) -> Option<usize> {
|
||||
find_sync(data, &DTS_HD_EXT_SYNC)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -180,12 +312,16 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn parse_single_frame() {
|
||||
// A single core frame with no following core sync is the LAST access
|
||||
// unit — held during streaming (can't know an extension won't follow),
|
||||
// then drained on flush() at EOF.
|
||||
let mut parser = DtsParser::new();
|
||||
let frame = make_dts_core(512);
|
||||
let pes = make_pes(frame, Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data.len(), 512);
|
||||
assert!(parser.parse(&pes).is_empty());
|
||||
let tail = parser.flush();
|
||||
assert_eq!(tail.len(), 1);
|
||||
assert_eq!(tail[0].data.len(), 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -198,9 +334,206 @@ mod tests {
|
||||
assert!(parser.parse(&pes1).is_empty());
|
||||
|
||||
let pes2 = make_pes(frame[mid..].to_vec(), Some(93000));
|
||||
let frames = parser.parse(&pes2);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data.len(), 512);
|
||||
assert!(parser.parse(&pes2).is_empty());
|
||||
let tail = parser.flush();
|
||||
assert_eq!(tail.len(), 1);
|
||||
assert_eq!(tail[0].data.len(), 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_cores_back_to_back_emit_first_on_boundary() {
|
||||
// The first complete unit is emitted as soon as the next core sync is
|
||||
// seen; the second is held until flush.
|
||||
let mut parser = DtsParser::new();
|
||||
let mut stream = make_dts_core(512);
|
||||
stream.extend_from_slice(&make_dts_core(640));
|
||||
let f = parser.parse(&make_pes(stream, Some(90000)));
|
||||
assert_eq!(f.len(), 1);
|
||||
assert_eq!(f[0].data.len(), 512);
|
||||
let tail = parser.flush();
|
||||
assert_eq!(tail.len(), 1);
|
||||
assert_eq!(tail[0].data.len(), 640);
|
||||
}
|
||||
|
||||
/// Build a minimal DTS-HD extension substream of `size` bytes (just the
|
||||
/// sync + zero-padding). The parser delimits extensions by the next CORE
|
||||
/// sync, not by the extension's own size header, so a valid header isn't
|
||||
/// required — only that the bytes carry no spurious core sync.
|
||||
fn make_dts_ext(size: usize) -> Vec<u8> {
|
||||
let mut e = vec![0u8; size];
|
||||
e[0..4].copy_from_slice(&DTS_HD_EXT_SYNC);
|
||||
e
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_dts_hd_extension_in_separate_pes_packets() {
|
||||
// The real Blu-ray layout (ground-truthed on Dunkirk): the DTS core
|
||||
// arrives in one PES, then its DTS-HD MA extension substreams arrive
|
||||
// in SEPARATE following PES packets on the same PID. The parser must
|
||||
// stitch core + all trailing extensions into one access unit — not
|
||||
// emit a core-only (lossy) frame and drop the extension PES packets
|
||||
// (the Dunkirk / Fight Club lossy-core bug).
|
||||
let mut parser = DtsParser::new();
|
||||
|
||||
// Frame 1: core (512) + two extension substreams (256 + 200).
|
||||
assert!(
|
||||
parser
|
||||
.parse(&make_pes(make_dts_core(512), Some(90000)))
|
||||
.is_empty(),
|
||||
"core alone: must wait for any following extension"
|
||||
);
|
||||
assert!(
|
||||
parser
|
||||
.parse(&make_pes(make_dts_ext(256), Some(91000)))
|
||||
.is_empty(),
|
||||
"first extension PES: still waiting for the unit to close"
|
||||
);
|
||||
assert!(
|
||||
parser
|
||||
.parse(&make_pes(make_dts_ext(200), Some(91500)))
|
||||
.is_empty(),
|
||||
"second extension PES: unit still not closed (no next core yet)"
|
||||
);
|
||||
|
||||
// Frame 2's core PES arrives — that closes frame 1. The emitted unit
|
||||
// must be core + BOTH extensions (lossless preserved), and keep the
|
||||
// core's PTS, not the extension PES timestamps.
|
||||
let f = parser.parse(&make_pes(make_dts_core(512), Some(93000)));
|
||||
assert_eq!(f.len(), 1);
|
||||
assert_eq!(
|
||||
f[0].data.len(),
|
||||
512 + 256 + 200,
|
||||
"frame must include core + every extension substream"
|
||||
);
|
||||
|
||||
// EOF drains frame 2.
|
||||
let tail = parser.flush();
|
||||
assert_eq!(tail.len(), 1);
|
||||
assert_eq!(tail[0].data.len(), 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_split_across_pes_is_preserved() {
|
||||
// An extension substream straddling a PES boundary must still be fully
|
||||
// attached to its core.
|
||||
let mut parser = DtsParser::new();
|
||||
let ext = make_dts_ext(300);
|
||||
assert!(
|
||||
parser
|
||||
.parse(&make_pes(make_dts_core(512), Some(90000)))
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
parser
|
||||
.parse(&make_pes(ext[..150].to_vec(), Some(91000)))
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
parser
|
||||
.parse(&make_pes(ext[150..].to_vec(), Some(91000)))
|
||||
.is_empty()
|
||||
);
|
||||
let tail = parser.flush();
|
||||
assert_eq!(tail.len(), 1);
|
||||
assert_eq!(tail[0].data.len(), 512 + 300);
|
||||
}
|
||||
|
||||
/// Build 4 bytes that look like a DTS core sync but whose `fsize` field
|
||||
/// decodes to a tiny `core_size` (< MIN_CORE_FRAME_BYTES). With the
|
||||
/// dead-code guards this passed validation and could close an access unit
|
||||
/// at a junk boundary; with the fix it must be drained and resynced past.
|
||||
fn bogus_tiny_core_sync() -> Vec<u8> {
|
||||
// Core sync + zero header bytes. fsize = 0 → core_size = 1 (< 10).
|
||||
let mut v = vec![0u8; 10];
|
||||
v[0..4].copy_from_slice(&DTS_CORE_SYNC);
|
||||
// bytes 5,6,7 left zero → fsize = 0 → dts_core_frame_size = 1.
|
||||
assert_eq!(dts_core_frame_size(&v), 1);
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bogus_tiny_core_sync_does_not_split_or_drop_real_au() {
|
||||
// A real core frame followed by an extension substream that happens to
|
||||
// contain a false core sync whose fsize decodes tiny. The bogus sync
|
||||
// must NOT close the real access unit early (dropping the rest of the
|
||||
// extension) nor emit a junk few-byte frame — it must be skipped, and
|
||||
// the whole core + extension preserved as one access unit.
|
||||
let mut parser = DtsParser::new();
|
||||
|
||||
// Frame 1: core(512) + an extension whose body embeds a bogus tiny
|
||||
// core sync midway through.
|
||||
let mut ext = make_dts_ext(256);
|
||||
// Embed the bogus core sync inside the extension body (offset 64).
|
||||
let bogus = bogus_tiny_core_sync();
|
||||
ext[64..64 + bogus.len()].copy_from_slice(&bogus);
|
||||
|
||||
let mut frame1 = make_dts_core(512);
|
||||
frame1.extend_from_slice(&ext);
|
||||
|
||||
// No next REAL core yet → frame 1 held.
|
||||
assert!(
|
||||
parser.parse(&make_pes(frame1, Some(90000))).is_empty(),
|
||||
"bogus tiny core sync must not close the AU; wait for a real core"
|
||||
);
|
||||
|
||||
// Frame 2's real core arrives — closes frame 1.
|
||||
let f = parser.parse(&make_pes(make_dts_core(640), Some(93000)));
|
||||
assert_eq!(f.len(), 1, "exactly one real access unit emitted");
|
||||
assert_eq!(
|
||||
f[0].data.len(),
|
||||
512 + 256,
|
||||
"AU must be the full core + extension, not split at the bogus sync"
|
||||
);
|
||||
assert_eq!(f[0].pts_ns, pts_to_ns(90000), "AU keeps the core's PTS");
|
||||
|
||||
let tail = parser.flush();
|
||||
assert_eq!(tail.len(), 1);
|
||||
assert_eq!(tail[0].data.len(), 640);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forced_emit_does_not_corrupt_next_au_pts() {
|
||||
// When the buffer exceeds MAX_AU_BYTES with no next core sync, the
|
||||
// parser force-emits for forward progress. The current PES at that
|
||||
// point is an extension-substream PES (later PTS). The forced path must
|
||||
// NOT make that extension PTS the base of the NEXT access unit.
|
||||
let mut parser = DtsParser::new();
|
||||
|
||||
// Core PES at the real PTS, then a giant extension (no next core) that
|
||||
// pushes the buffer past MAX_AU_BYTES, forcing an emit.
|
||||
let core_pts = 90000i64;
|
||||
assert!(
|
||||
parser
|
||||
.parse(&make_pes(make_dts_core(512), Some(core_pts)))
|
||||
.is_empty()
|
||||
);
|
||||
let ext_pts = 120000i64; // later extension-PES timestamp
|
||||
let big_ext = make_dts_ext(MAX_AU_BYTES + 1024);
|
||||
let f = parser.parse(&make_pes(big_ext, Some(ext_pts)));
|
||||
assert_eq!(f.len(), 1, "oversized buffer force-emits one AU");
|
||||
assert_eq!(
|
||||
f[0].pts_ns,
|
||||
pts_to_ns(core_pts),
|
||||
"forced AU keeps the core PTS"
|
||||
);
|
||||
|
||||
// The next REAL core PES arrives with its own PTS. Its AU must inherit
|
||||
// THIS core's PTS, not the prior extension PES timestamp.
|
||||
let next_core_pts = 150000i64;
|
||||
assert!(
|
||||
parser
|
||||
.parse(&make_pes(make_dts_core(512), Some(next_core_pts)))
|
||||
.is_empty()
|
||||
);
|
||||
let next_next_pts = 180000i64;
|
||||
let f2 = parser.parse(&make_pes(make_dts_core(512), Some(next_next_pts)));
|
||||
assert_eq!(f2.len(), 1);
|
||||
assert_eq!(
|
||||
f2[0].pts_ns,
|
||||
pts_to_ns(next_core_pts),
|
||||
"AU after a forced emit must use the next core's PTS, not the \
|
||||
stale extension PTS"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -30,6 +30,7 @@ impl CodecParser for DvdSubParser {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: pes.data.clone(),
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
|
||||
+18
-12
@@ -38,8 +38,11 @@ impl CodecParser for H264Parser {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Use DTS when available (monotonic for B-frame content), fall back to PTS
|
||||
let pts_ns = pes.dts.or(pes.pts).map(pts_to_ns).unwrap_or(0);
|
||||
// MKV block timecodes are PRESENTATION timestamps; frames are stored in
|
||||
// decode order and the player reorders by timecode. Use PTS, not DTS —
|
||||
// DTS presents B-frames in decode order (visible judder) and breaks
|
||||
// PTS-based seeking. Fall back to DTS only if PTS is absent.
|
||||
let pts_ns = pes.pts.or(pes.dts).map(pts_to_ns).unwrap_or(0);
|
||||
|
||||
// Scan NAL units for SPS, PPS, and IDR detection
|
||||
let mut keyframe = false;
|
||||
@@ -85,6 +88,7 @@ impl CodecParser for H264Parser {
|
||||
pts_ns,
|
||||
keyframe,
|
||||
data: frame_data,
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
@@ -176,14 +180,16 @@ impl<'a> Iterator for NalIterator<'a> {
|
||||
}
|
||||
|
||||
/// Find the position of the next start code (00 00 01) at or after `from`.
|
||||
///
|
||||
/// Backed by `memchr::memmem::find` for SIMD-accelerated bytestring
|
||||
/// search. On AVX2-capable x86_64 this runs ~5–10× the byte-by-byte
|
||||
/// scan that preceded it; on a 200 KB UHD HEVC frame the saving is
|
||||
/// in the hundreds of microseconds per call.
|
||||
pub fn find_start_code(data: &[u8], from: usize) -> Option<usize> {
|
||||
if data.len() < from + 3 {
|
||||
return None;
|
||||
}
|
||||
// Range excludes last 2 bytes since we read 3 bytes at each position.
|
||||
// data.len()-2 as exclusive upper bound means last checked index is data.len()-3,
|
||||
// which accesses data[len-3], data[len-2], data[len-1] — all valid.
|
||||
(from..data.len() - 2).find(|&i| data[i] == 0x00 && data[i + 1] == 0x00 && data[i + 2] == 0x01)
|
||||
memchr::memmem::find(&data[from..], b"\x00\x00\x01").map(|rel| from + rel)
|
||||
}
|
||||
|
||||
/// Skip past the start code at position `pos`, returning the first byte after it.
|
||||
@@ -448,10 +454,10 @@ mod tests {
|
||||
assert!(frames.is_empty());
|
||||
}
|
||||
|
||||
// --- DTS preferred over PTS when present ---
|
||||
// --- PTS (presentation) used for the MKV block timecode, not DTS ---
|
||||
|
||||
#[test]
|
||||
fn dts_preferred_over_pts() {
|
||||
fn pts_preferred_over_dts() {
|
||||
let mut parser = H264Parser::new();
|
||||
|
||||
let mut data = Vec::new();
|
||||
@@ -461,13 +467,13 @@ mod tests {
|
||||
|
||||
let pes = PesPacket {
|
||||
pid: 0x1011,
|
||||
pts: Some(180000), // 2 seconds
|
||||
dts: Some(90000), // 1 second
|
||||
pts: Some(180000), // 2 seconds (presentation)
|
||||
dts: Some(90000), // 1 second (decode)
|
||||
data,
|
||||
};
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
// DTS should be used, not PTS
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
// PTS must be used — MKV block timecodes are presentation timestamps.
|
||||
assert_eq!(frames[0].pts_ns, 2_000_000_000);
|
||||
}
|
||||
}
|
||||
|
||||
+171
-26
@@ -48,17 +48,30 @@ impl CodecParser for HevcParser {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Use DTS when available (monotonic for B-frame content), fall back to PTS
|
||||
let pts_ns = pes.dts.or(pes.pts).map(pts_to_ns).unwrap_or(0);
|
||||
// MKV block timecodes are PRESENTATION timestamps; frames are stored
|
||||
// in decode order (the order they arrive here) and the player reorders
|
||||
// for display by timecode. So use PTS, not DTS — using DTS makes the
|
||||
// block timecode monotonic in storage order, which presents B-frames in
|
||||
// decode order (visible judder / wrong frames) and breaks PTS-based
|
||||
// seeking. Fall back to DTS only if PTS is somehow absent.
|
||||
let pts_ns = pes.pts.or(pes.dts).map(pts_to_ns).unwrap_or(0);
|
||||
let data = &pes.data;
|
||||
let mut keyframe = false;
|
||||
let mut frame_data = Vec::new();
|
||||
// Pre-size: output is ~input bytes with a few 4-byte length
|
||||
// prefixes added. UHD frames are 150-300 KB; the unsized Vec
|
||||
// growth chain otherwise reallocs 5-7× per frame.
|
||||
let mut frame_data = Vec::with_capacity(data.len() + 64);
|
||||
|
||||
// Single-pass NAL scan: extract params, detect keyframes, build length-prefixed output
|
||||
let mut pos = 0;
|
||||
while let Some(sc_pos) = find_start_code(data, pos) {
|
||||
if let Some(nal_start) = skip_start_code(data, sc_pos) {
|
||||
let next = find_start_code(data, nal_start).unwrap_or(data.len());
|
||||
// Strip the leading zeros of the following start code. For a
|
||||
// conforming bitstream this is lossless: rbsp_trailing_bits()
|
||||
// sets a stop-one bit, so the final byte of any RBSP is never
|
||||
// 0x00 — the only trailing zeros here belong to the next
|
||||
// 00 00 (00) 01 prefix.
|
||||
let mut end = next;
|
||||
while end > nal_start && data[end - 1] == 0x00 {
|
||||
end -= 1;
|
||||
@@ -107,6 +120,7 @@ impl CodecParser for HevcParser {
|
||||
pts_ns,
|
||||
keyframe,
|
||||
data: frame_data,
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
@@ -120,34 +134,41 @@ impl CodecParser for HevcParser {
|
||||
// Full HEVCDecoderConfigurationRecord is complex — for now, concatenate
|
||||
let mut record = Vec::new();
|
||||
|
||||
// Minimal HEVCDecoderConfigurationRecord header
|
||||
// Minimal HEVCDecoderConfigurationRecord header.
|
||||
//
|
||||
// The stored SPS NAL is [2-byte HEVC NAL header][SPS RBSP...].
|
||||
// The RBSP begins at sps[2]; profile_tier_level() begins one byte
|
||||
// later, after sps_video_parameter_set_id u(4) +
|
||||
// sps_max_sub_layers_minus1 u(3) + sps_temporal_id_nesting_flag u(1)
|
||||
// (= sps[2], a full byte). So the profile_tier_level fields are:
|
||||
// sps[3] general_profile_space u(2)+tier u(1)+profile_idc u(5)
|
||||
// sps[4..8] general_profile_compatibility_flags u(32)
|
||||
// sps[8..14] general_constraint_indicator_flags 48 bits
|
||||
// sps[14] general_level_idc u(8)
|
||||
// (Byte-aligned read; emulation-prevention bytes within the first
|
||||
// 15 SPS bytes are not handled — extremely rare and matches the
|
||||
// pre-existing simplification.)
|
||||
record.push(1); // configurationVersion
|
||||
// General profile space, tier flag, profile IDC from SPS
|
||||
if sps.len() > 3 {
|
||||
record.push(sps[1]); // general_profile_space + general_tier_flag + general_profile_idc
|
||||
// general_profile_space + general_tier_flag + general_profile_idc
|
||||
record.push(if sps.len() > 3 { sps[3] } else { 0 });
|
||||
// general_profile_compatibility_flags (4 bytes) — SPS bytes 4..8
|
||||
if sps.len() > 7 {
|
||||
record.extend_from_slice(&sps[4..8]);
|
||||
} else {
|
||||
record.push(0);
|
||||
let avail = sps.len().saturating_sub(4).min(4);
|
||||
record.extend_from_slice(&sps[sps.len().min(4)..sps.len().min(8)]);
|
||||
record.extend_from_slice(&vec![0u8; 4 - avail]);
|
||||
}
|
||||
// general_profile_compatibility_flags (4 bytes) — from SPS bytes 2..6
|
||||
if sps.len() > 5 {
|
||||
record.extend_from_slice(&sps[2..6]);
|
||||
// general_constraint_indicator_flags (6 bytes) — SPS bytes 8..14
|
||||
if sps.len() > 13 {
|
||||
record.extend_from_slice(&sps[8..14]);
|
||||
} else {
|
||||
record.extend_from_slice(&[0, 0, 0, 0]);
|
||||
let avail = sps.len().saturating_sub(8).min(6);
|
||||
record.extend_from_slice(&sps[sps.len().min(8)..sps.len().min(14)]);
|
||||
record.extend_from_slice(&vec![0u8; 6 - avail]);
|
||||
}
|
||||
// general_constraint_indicator_flags (6 bytes) — from SPS bytes 6..12
|
||||
if sps.len() > 11 {
|
||||
record.extend_from_slice(&sps[6..12]);
|
||||
} else {
|
||||
let avail = sps.len().saturating_sub(6).min(6);
|
||||
if avail > 0 {
|
||||
record.extend_from_slice(&sps[6..6 + avail]);
|
||||
record.extend_from_slice(&vec![0u8; 6 - avail]);
|
||||
} else {
|
||||
record.extend_from_slice(&[0, 0, 0, 0, 0, 0]);
|
||||
}
|
||||
}
|
||||
// general_level_idc
|
||||
record.push(if sps.len() > 12 { sps[12] } else { 0 });
|
||||
// general_level_idc — SPS byte 14
|
||||
record.push(if sps.len() > 14 { sps[14] } else { 0 });
|
||||
// min_spatial_segmentation_idc (4 + 12 bits)
|
||||
record.extend_from_slice(&[0xF0, 0x00]);
|
||||
// parallelismType (6 + 2 bits)
|
||||
@@ -264,6 +285,103 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hvcc_profile_tier_level_offsets() {
|
||||
// The hvcC fixed header must read profile_tier_level from the SPS
|
||||
// RBSP, not from the NAL header. Stored SPS = [2-byte NAL header][RBSP].
|
||||
// RBSP layout (byte-aligned):
|
||||
// sps[2] sps_vps_id/max_sub_layers/temporal_nesting
|
||||
// sps[3] general_profile_space+tier+profile_idc
|
||||
// sps[4..8] general_profile_compatibility_flags
|
||||
// sps[8..14] general_constraint_indicator_flags
|
||||
// sps[14] general_level_idc
|
||||
let mut parser = HevcParser::new();
|
||||
|
||||
// Distinct, recognizable values for each field.
|
||||
let sps_rbsp: [u8; 13] = [
|
||||
0xAB, // sps[2] (vps_id etc.) — must NOT leak into profile fields
|
||||
0x21, // sps[3] profile byte: space=0, tier=0, profile_idc=1
|
||||
0x60, 0x00, 0x00, 0x00, // sps[4..8] compat flags
|
||||
0x90, 0x00, 0x00, 0x00, 0x00, 0x00, // sps[8..14] constraint flags
|
||||
0x7B, // sps[14] level_idc = 123
|
||||
];
|
||||
|
||||
let mut data = Vec::new();
|
||||
// VPS
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01]);
|
||||
data.extend_from_slice(&hevc_nal_header(32));
|
||||
data.extend_from_slice(&[0xAA, 0xBB, 0xCC]);
|
||||
// SPS — 2-byte header + the structured RBSP above
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01]);
|
||||
data.extend_from_slice(&hevc_nal_header(33));
|
||||
data.extend_from_slice(&sps_rbsp);
|
||||
// PPS
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01]);
|
||||
data.extend_from_slice(&hevc_nal_header(34));
|
||||
data.extend_from_slice(&[0xDD, 0xEE]);
|
||||
|
||||
let pes = make_pes(data, Some(0));
|
||||
parser.parse(&pes);
|
||||
|
||||
let cp = parser
|
||||
.codec_private()
|
||||
.expect("codec_private should be Some");
|
||||
|
||||
// record[0] = configurationVersion
|
||||
assert_eq!(cp[0], 1, "configurationVersion");
|
||||
// record[1] = general_profile_space+tier+profile_idc <- sps[3]
|
||||
assert_eq!(
|
||||
cp[1], 0x21,
|
||||
"profile byte must come from SPS RBSP, not NAL hdr"
|
||||
);
|
||||
// record[2..6] = general_profile_compatibility_flags <- sps[4..8]
|
||||
assert_eq!(&cp[2..6], &[0x60, 0x00, 0x00, 0x00], "compatibility flags");
|
||||
// record[6..12] = general_constraint_indicator_flags <- sps[8..14]
|
||||
assert_eq!(
|
||||
&cp[6..12],
|
||||
&[0x90, 0x00, 0x00, 0x00, 0x00, 0x00],
|
||||
"constraint flags"
|
||||
);
|
||||
// record[12] = general_level_idc <- sps[14]
|
||||
assert_eq!(cp[12], 0x7B, "level_idc must come from sps[14]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hvcc_short_sps_does_not_panic() {
|
||||
// A truncated SPS must still produce a fixed header without panicking
|
||||
// and zero-pad the missing profile/level bytes.
|
||||
let mut parser = HevcParser::new();
|
||||
|
||||
let mut data = Vec::new();
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01]);
|
||||
data.extend_from_slice(&hevc_nal_header(32));
|
||||
data.extend_from_slice(&[0xAA]);
|
||||
// SPS with only 3 RBSP bytes (stored len = 5): forces every guard path
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01]);
|
||||
data.extend_from_slice(&hevc_nal_header(33));
|
||||
data.extend_from_slice(&[0x11, 0x22, 0x33]);
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01]);
|
||||
data.extend_from_slice(&hevc_nal_header(34));
|
||||
data.extend_from_slice(&[0xDD]);
|
||||
|
||||
let pes = make_pes(data, Some(0));
|
||||
parser.parse(&pes);
|
||||
|
||||
let cp = parser
|
||||
.codec_private()
|
||||
.expect("codec_private should be Some");
|
||||
// sps stored = [hdr0, hdr1, 0x11, 0x22, 0x33], len 5.
|
||||
// profile byte = sps[3] = 0x22; everything past sps[4]=0x33 is absent.
|
||||
assert_eq!(cp[0], 1);
|
||||
assert_eq!(cp[1], 0x22, "profile byte = sps[3]");
|
||||
// compat flags: only sps[4]=0x33 present, rest zero-padded.
|
||||
assert_eq!(&cp[2..6], &[0x33, 0x00, 0x00, 0x00]);
|
||||
// constraint flags: none present, all zero.
|
||||
assert_eq!(&cp[6..12], &[0x00, 0x00, 0x00, 0x00, 0x00, 0x00]);
|
||||
// level_idc: absent, zero.
|
||||
assert_eq!(cp[12], 0x00);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_private_none_before_params() {
|
||||
let parser = HevcParser::new();
|
||||
@@ -470,6 +588,33 @@ mod tests {
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
}
|
||||
|
||||
// --- PTS (presentation), not DTS, drives the MKV block timecode ---
|
||||
// Regression for B-frame presentation: writing DTS as the block timecode
|
||||
// presents frames in decode order (visible judder) and breaks seeking.
|
||||
|
||||
#[test]
|
||||
fn pts_preferred_over_dts() {
|
||||
let mut parser = HevcParser::new();
|
||||
|
||||
let mut data = Vec::new();
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01]);
|
||||
data.extend_from_slice(&hevc_nal_header(1)); // TRAIL_R slice
|
||||
data.extend_from_slice(&[0x10, 0x20]);
|
||||
|
||||
let pes = PesPacket {
|
||||
pid: 0x1011,
|
||||
pts: Some(180000), // 2 s (presentation)
|
||||
dts: Some(90000), // 1 s (decode)
|
||||
data,
|
||||
};
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(
|
||||
frames[0].pts_ns, 2_000_000_000,
|
||||
"block timecode must be PTS"
|
||||
);
|
||||
}
|
||||
|
||||
// --- Dolby Vision enhancement layer ---
|
||||
|
||||
#[test]
|
||||
|
||||
+89
-10
@@ -1,24 +1,39 @@
|
||||
//! BD/DVD LPCM (Linear PCM) audio parser.
|
||||
//!
|
||||
//! BD LPCM PES packets have a 4-byte header:
|
||||
//! BD LPCM PES packets (TS stream type 0x80) carry a 4-byte header on the
|
||||
//! elementary-stream payload:
|
||||
//! Bytes 0-1: audio frame number
|
||||
//! Byte 2: reserved
|
||||
//! Byte 3: quantization (bits 7-6), sample rate (bits 5-4), channel assignment (bits 3-0)
|
||||
//! This header is part of the ES payload, so the BD parser must strip it.
|
||||
//!
|
||||
//! DVD LPCM (private stream 1, sub-stream 0xA0-0xA7) has a 3-byte header.
|
||||
//! DVD LPCM lives in private stream 1 (sub-stream 0xA0-0xA7). Its 7-byte
|
||||
//! private sub-header (sub_id + frames + first-access-unit-ptr(2) + emphasis +
|
||||
//! quant/freq + channels) is stripped by `PsDemuxer` while demuxing the
|
||||
//! Program Stream. By the time a DVD LPCM `PesPacket` reaches this parser its
|
||||
//! `data` is already raw PCM, so the parser must NOT strip any further bytes —
|
||||
//! doing so drops one sample pair per PES and drifts the audio.
|
||||
//!
|
||||
//! The raw PCM data follows the header. No framing is needed — each PES
|
||||
//! payload minus its header is one complete audio frame.
|
||||
//! The two origins are distinguished by the `strip_header` flag: BD = strip,
|
||||
//! DVD = leave intact. The raw PCM data is otherwise one complete audio frame
|
||||
//! per PES; no framing is needed.
|
||||
//!
|
||||
//! For MKV: codec ID "A_PCM/INT/BIG" (BD) or "A_PCM/INT/LIT" (DVD).
|
||||
//! All frames are keyframes (uncompressed audio).
|
||||
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
/// BD LPCM header size in bytes.
|
||||
/// BD LPCM header size in bytes (present on BD-TS LPCM, absent on DVD-PS LPCM
|
||||
/// because `PsDemuxer` already stripped the private sub-header).
|
||||
const BD_LPCM_HEADER_SIZE: usize = 4;
|
||||
|
||||
pub struct LpcmParser;
|
||||
pub struct LpcmParser {
|
||||
/// Whether to strip the 4-byte BD LPCM header from each PES payload.
|
||||
///
|
||||
/// `true` for BD-TS LPCM (header still present), `false` for DVD-PS LPCM
|
||||
/// (header already removed by `PsDemuxer`).
|
||||
strip_header: bool,
|
||||
}
|
||||
|
||||
impl Default for LpcmParser {
|
||||
fn default() -> Self {
|
||||
@@ -27,23 +42,37 @@ impl Default for LpcmParser {
|
||||
}
|
||||
|
||||
impl LpcmParser {
|
||||
/// BD-TS LPCM parser: strips the 4-byte BD LPCM header from each PES.
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
Self { strip_header: true }
|
||||
}
|
||||
|
||||
/// DVD-PS LPCM parser: `PsDemuxer` already stripped the private sub-header,
|
||||
/// so the payload is raw PCM and no further bytes are removed.
|
||||
pub fn new_dvd() -> Self {
|
||||
Self {
|
||||
strip_header: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CodecParser for LpcmParser {
|
||||
fn parse(&mut self, pes: &PesPacket) -> Vec<Frame> {
|
||||
// Skip the BD LPCM header (4 bytes).
|
||||
let offset = if self.strip_header {
|
||||
BD_LPCM_HEADER_SIZE
|
||||
} else {
|
||||
0
|
||||
};
|
||||
// If the PES is too short to contain header + data, return nothing.
|
||||
if pes.data.len() <= BD_LPCM_HEADER_SIZE {
|
||||
if pes.data.len() <= offset {
|
||||
return Vec::new();
|
||||
}
|
||||
let pts_ns = pes.pts.map(pts_to_ns).unwrap_or(0);
|
||||
vec![Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: pes.data[BD_LPCM_HEADER_SIZE..].to_vec(),
|
||||
data: pes.data[offset..].to_vec(),
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
@@ -83,6 +112,56 @@ mod tests {
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000); // 90000 ticks = 1 second
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bd_lpcm_strips_4_byte_header() {
|
||||
// BD-TS LPCM: the 4-byte BD header is part of the ES payload and must
|
||||
// be stripped, leaving exactly the PCM bytes.
|
||||
let mut parser = LpcmParser::new();
|
||||
let header = vec![0x00, 0x01, 0x00, 0b1001_0001];
|
||||
let pcm = vec![0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88];
|
||||
let mut data = header;
|
||||
data.extend_from_slice(&pcm);
|
||||
|
||||
let frames = parser.parse(&make_pes(data, Some(0)));
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data, pcm, "BD must strip exactly 4 header bytes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dvd_lpcm_preserves_all_pcm_bytes() {
|
||||
// DVD-PS LPCM: PsDemuxer already removed the 7-byte private sub-header,
|
||||
// so the payload handed to this parser is raw PCM. The DVD parser must
|
||||
// NOT strip any further bytes (the round-2 audit Finding 3 bug: the BD
|
||||
// 4-byte strip dropped one sample pair per PES, drifting the audio).
|
||||
let mut parser = LpcmParser::new_dvd();
|
||||
let pcm = vec![0xDE, 0xAD, 0xBE, 0xEF, 0xCA, 0xFE, 0x01, 0x02];
|
||||
let frames = parser.parse(&make_pes(pcm.clone(), Some(90000)));
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(
|
||||
frames[0].data, pcm,
|
||||
"DVD must preserve every PCM byte — no second strip"
|
||||
);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dvd_lpcm_emits_short_payload_bd_would_drop() {
|
||||
// A 4-byte raw-PCM DVD payload (1 sample pair at 16-bit stereo). The BD
|
||||
// parser drops <= 4 bytes as "header only"; the DVD parser must emit it.
|
||||
let mut bd = LpcmParser::new();
|
||||
let mut dvd = LpcmParser::new_dvd();
|
||||
let pcm = vec![0xAA, 0xBB, 0xCC, 0xDD];
|
||||
|
||||
assert!(
|
||||
bd.parse(&make_pes(pcm.clone(), Some(0))).is_empty(),
|
||||
"BD treats 4 bytes as header-only"
|
||||
);
|
||||
let frames = dvd.parse(&make_pes(pcm.clone(), Some(0)));
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data, pcm);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn always_keyframe() {
|
||||
let mut parser = LpcmParser::new();
|
||||
|
||||
+31
-1
@@ -29,6 +29,13 @@ pub struct Frame {
|
||||
pub keyframe: bool,
|
||||
/// Frame data (elementary stream bytes).
|
||||
pub data: Vec<u8>,
|
||||
/// Optional duration in nanoseconds — only set by parsers that
|
||||
/// can compute one (currently PGS, which pairs a display PCS
|
||||
/// with the following empty PCS). When `Some`, the MKV muxer
|
||||
/// emits a `BlockGroup` with `BlockDuration` instead of a
|
||||
/// `SimpleBlock`; without it players guess the display interval
|
||||
/// (subtitles linger past their end-time).
|
||||
pub duration_ns: Option<u64>,
|
||||
}
|
||||
|
||||
/// Convert 90kHz PTS to nanoseconds (round to nearest).
|
||||
@@ -45,6 +52,18 @@ pub trait CodecParser: Send {
|
||||
/// Some (TrueHD): multiple access units per PES.
|
||||
fn parse(&mut self, pes: &PesPacket) -> Vec<Frame>;
|
||||
|
||||
/// Drain any access unit still buffered after the last PES.
|
||||
///
|
||||
/// Parsers that buffer across PES boundaries to assemble a complete
|
||||
/// access unit (e.g. DTS-HD, whose extension substreams arrive in
|
||||
/// separate PES packets) hold the final unit until they can prove it's
|
||||
/// complete. At end-of-stream there is no following packet to prove it,
|
||||
/// so the demuxer calls `flush()` once after the last PES to emit it.
|
||||
/// Default: nothing buffered, no tail.
|
||||
fn flush(&mut self) -> Vec<Frame> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// Get codec initialization data (e.g., SPS+PPS for H.264).
|
||||
/// Returns None until enough data has been seen.
|
||||
fn codec_private(&self) -> Option<Vec<u8>>;
|
||||
@@ -71,6 +90,7 @@ impl CodecParser for PassthroughParser {
|
||||
pts_ns,
|
||||
keyframe: self.keyframe,
|
||||
data: pes.data.clone(),
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
@@ -82,7 +102,16 @@ impl CodecParser for PassthroughParser {
|
||||
/// Create the appropriate parser for a codec, with optional codec private data.
|
||||
///
|
||||
/// For DvdSub, `codec_data` should be the pre-formatted VobSub .idx palette header.
|
||||
pub fn parser_for_codec(codec: Codec, codec_data: Option<Vec<u8>>) -> Box<dyn CodecParser> {
|
||||
///
|
||||
/// `is_dvd_ps` selects the DVD program-stream variant where it matters: DVD
|
||||
/// LPCM arrives with its private sub-header already stripped by the
|
||||
/// `PsDemuxer`, so the LPCM parser must NOT strip the 4-byte BD LPCM header
|
||||
/// again (that would drop one PCM sample pair per PES → progressive drift).
|
||||
pub fn parser_for_codec(
|
||||
codec: Codec,
|
||||
codec_data: Option<Vec<u8>>,
|
||||
is_dvd_ps: bool,
|
||||
) -> Box<dyn CodecParser> {
|
||||
match codec {
|
||||
Codec::H264 => Box::new(h264::H264Parser::new()),
|
||||
Codec::Hevc => Box::new(hevc::HevcParser::new()),
|
||||
@@ -92,6 +121,7 @@ pub fn parser_for_codec(codec: Codec, codec_data: Option<Vec<u8>>) -> Box<dyn Co
|
||||
Codec::DtsHdMa | Codec::DtsHdHr | Codec::Dts => Box::new(dts::DtsParser::new()),
|
||||
Codec::TrueHd => Box::new(truehd::TrueHdParser::new()),
|
||||
Codec::Pgs => Box::new(pgs::PgsParser::new()),
|
||||
Codec::Lpcm if is_dvd_ps => Box::new(lpcm::LpcmParser::new_dvd()),
|
||||
Codec::Lpcm => Box::new(lpcm::LpcmParser::new()),
|
||||
Codec::DvdSub => Box::new(dvdsub::DvdSubParser::new(codec_data)),
|
||||
_ => Box::new(PassthroughParser::new(true)),
|
||||
|
||||
+80
-4
@@ -96,9 +96,14 @@ impl CodecParser for Mpeg2Parser {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let pts_ns = pes.dts.or(pes.pts).map(pts_to_ns).unwrap_or(0);
|
||||
// MKV block timecodes are PRESENTATION timestamps; frames are stored in
|
||||
// decode order and the player reorders by timecode. Use PTS, not DTS —
|
||||
// DTS presents B-frames in decode order (visible judder) and breaks
|
||||
// PTS-based seeking. Fall back to DTS only if PTS is absent.
|
||||
let pts_ns = pes.pts.or(pes.dts).map(pts_to_ns).unwrap_or(0);
|
||||
let data = &pes.data;
|
||||
let mut keyframe = false;
|
||||
let mut has_picture = false;
|
||||
|
||||
// Scan for start codes in the elementary stream data.
|
||||
let mut pos = 0;
|
||||
@@ -176,6 +181,7 @@ impl CodecParser for Mpeg2Parser {
|
||||
PICTURE_CODE => {
|
||||
// Picture header: bytes after start code contain temporal_reference
|
||||
// (10 bits) + picture_coding_type (3 bits).
|
||||
has_picture = true;
|
||||
if sc + 5 < data.len() {
|
||||
let picture_coding_type = (data[sc + 5] >> 3) & 0x07;
|
||||
if picture_coding_type == PICTURE_TYPE_I {
|
||||
@@ -190,10 +196,28 @@ impl CodecParser for Mpeg2Parser {
|
||||
}
|
||||
}
|
||||
|
||||
// A PES that carried a sequence header but no picture start code is a
|
||||
// parameter-set-only access unit: it has no coded picture to emit.
|
||||
// Emitting it as a standalone keyframe would put bare sequence-header
|
||||
// bytes into frame data with no picture. The sequence header is
|
||||
// captured into codec_private above and is re-emitted in-band on the
|
||||
// next real picture's PES, so dropping the empty access unit loses
|
||||
// nothing. Mirrors how the H.264/HEVC parsers skip parameter-set-only
|
||||
// access units.
|
||||
//
|
||||
// Conservative: only drop when this PES actually contained a sequence
|
||||
// header and no picture. A PES with neither (e.g. a slice
|
||||
// continuation) still passes through unchanged, preserving real
|
||||
// keyframe detection.
|
||||
if !has_picture && contains_seq_header(data) {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
vec![Frame {
|
||||
pts_ns,
|
||||
keyframe,
|
||||
data: pes.data.clone(),
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
@@ -240,6 +264,21 @@ fn parse_aspect_ratio(hdr: &[u8]) -> Option<(u8, u8)> {
|
||||
Some(ASPECT_RATIOS[ar_code])
|
||||
}
|
||||
|
||||
/// Returns true if `data` contains a sequence-header start code (00 00 01 B3).
|
||||
fn contains_seq_header(data: &[u8]) -> bool {
|
||||
let mut pos = 0;
|
||||
while let Some(sc) = find_start_code(data, pos) {
|
||||
if sc + 3 >= data.len() {
|
||||
break;
|
||||
}
|
||||
if data[sc + 3] == SEQ_HEADER_CODE {
|
||||
return true;
|
||||
}
|
||||
pos = sc + 4;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Find the position of the next start code (00 00 01) at or after `from`.
|
||||
fn find_start_code(data: &[u8], from: usize) -> Option<usize> {
|
||||
if data.len() < from + 3 {
|
||||
@@ -428,15 +467,15 @@ mod tests {
|
||||
assert!(has_ext, "codec_private should include sequence extension");
|
||||
}
|
||||
|
||||
// --- I-frame with sequence header = keyframe ---
|
||||
// --- sequence header + picture = keyframe ---
|
||||
|
||||
#[test]
|
||||
fn sequence_header_implies_keyframe() {
|
||||
fn sequence_header_with_picture_is_keyframe() {
|
||||
let mut parser = Mpeg2Parser::new();
|
||||
|
||||
let mut data = Vec::new();
|
||||
data.extend_from_slice(&make_seq_header(720, 480, 3, 4));
|
||||
// Even without an explicit picture header, a sequence header implies I-frame.
|
||||
data.extend_from_slice(&make_picture_header(PICTURE_TYPE_I));
|
||||
data.extend_from_slice(&[0xFF; 16]);
|
||||
|
||||
let pes = make_pes(data, Some(0));
|
||||
@@ -444,6 +483,43 @@ mod tests {
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert!(frames[0].keyframe);
|
||||
// codecPrivate is still captured.
|
||||
assert!(parser.codec_private().is_some());
|
||||
}
|
||||
|
||||
// --- parameter-set-only PES (seq header, no picture) emits no frame ---
|
||||
|
||||
#[test]
|
||||
fn sequence_header_only_pes_emits_no_frame() {
|
||||
let mut parser = Mpeg2Parser::new();
|
||||
|
||||
// A PES carrying only a sequence header (+ extension), no picture.
|
||||
let mut data = Vec::new();
|
||||
data.extend_from_slice(&make_seq_header(1920, 1080, 3, 4));
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01, SEQ_EXT_CODE]);
|
||||
data.extend_from_slice(&[0x14, 0x8A, 0x00, 0x01, 0x00, 0x00]);
|
||||
|
||||
let pes = make_pes(data, Some(0));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
// No coded picture → no frame emitted, but the sequence header is
|
||||
// still captured for codecPrivate.
|
||||
assert!(
|
||||
frames.is_empty(),
|
||||
"parameter-set-only PES should not emit a frame"
|
||||
);
|
||||
assert!(
|
||||
parser.codec_private().is_some(),
|
||||
"sequence header should still be captured into codec_private"
|
||||
);
|
||||
|
||||
// A following picture-bearing PES emits the real keyframe.
|
||||
let mut data2 = Vec::new();
|
||||
data2.extend_from_slice(&make_picture_header(PICTURE_TYPE_I));
|
||||
data2.extend_from_slice(&[0xFF; 16]);
|
||||
let frames2 = parser.parse(&make_pes(data2, Some(3600)));
|
||||
assert_eq!(frames2.len(), 1);
|
||||
assert!(frames2[0].keyframe);
|
||||
}
|
||||
|
||||
// --- PTS conversion ---
|
||||
|
||||
+169
-28
@@ -1,12 +1,39 @@
|
||||
//! HDMV PGS (Presentation Graphics Stream) subtitle parser.
|
||||
//!
|
||||
//! PGS segments: PCS, WDS, PDS, ODS, END.
|
||||
//! Each PES packet contains one or more segments.
|
||||
//! All segments are keyframes (no inter-segment dependencies).
|
||||
//! PGS segments: PCS, WDS, PDS, ODS, END. Each PES packet starts with
|
||||
//! one of those (segment_type byte at offset 0).
|
||||
//!
|
||||
//! Subtitle display lifecycle (BD spec):
|
||||
//! - A "display" PCS (number_of_composition_objects > 0) starts a
|
||||
//! visible subtitle. Its WDS/PDS/ODS follow.
|
||||
//! - A later "empty" PCS (number_of_composition_objects == 0) clears
|
||||
//! the screen.
|
||||
//!
|
||||
//! For Matroska output we collapse that pair into one block with
|
||||
//! `BlockDuration` set to (clear_pts - display_pts). Without a
|
||||
//! duration, hardware players linger on the last bitmap until the
|
||||
//! next subtitle replaces it — which can be many seconds, and on a
|
||||
//! disc where the final subtitle has no follower, until end of file.
|
||||
|
||||
use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
|
||||
pub struct PgsParser;
|
||||
const SEGMENT_PCS: u8 = 0x16;
|
||||
// Upper bound on a pending display set's accumulated bytes. Real PGS
|
||||
// display sets are small (a 1080p RLE bitmap plus palette is well under
|
||||
// 1 MB); a stream that keeps appending non-PCS segments without ever
|
||||
// emitting a PCS is malformed. Cap accumulation to bound memory and
|
||||
// drop further appends until the next PCS resyncs the parser. Mirrors
|
||||
// the MAX_*_BYTES / MAX_*_BUF caps in the DTS and AC-3 parsers.
|
||||
const MAX_PGS_PENDING_BYTES: usize = 4 * 1024 * 1024;
|
||||
// Offset within the PES payload at which number_of_composition_objects
|
||||
// lives in a PCS: 3-byte segment header + 10 bytes of PCS fields
|
||||
// (video_w/h, frame_rate, comp_num, comp_state, palette_update,
|
||||
// palette_id_ref) = 13.
|
||||
const PCS_NUM_OBJECTS_OFFSET: usize = 13;
|
||||
|
||||
pub struct PgsParser {
|
||||
pending: Option<(i64, Vec<u8>)>,
|
||||
}
|
||||
|
||||
impl Default for PgsParser {
|
||||
fn default() -> Self {
|
||||
@@ -16,7 +43,7 @@ impl Default for PgsParser {
|
||||
|
||||
impl PgsParser {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
Self { pending: None }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,11 +53,68 @@ impl CodecParser for PgsParser {
|
||||
return Vec::new();
|
||||
}
|
||||
let pts_ns = pes.pts.map(pts_to_ns).unwrap_or(0);
|
||||
vec![Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: pes.data.clone(),
|
||||
}]
|
||||
|
||||
let is_pcs = pes.data[0] == SEGMENT_PCS;
|
||||
let pcs_num_objects = if is_pcs && pes.data.len() > PCS_NUM_OBJECTS_OFFSET {
|
||||
Some(pes.data[PCS_NUM_OBJECTS_OFFSET])
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut out = Vec::new();
|
||||
match pcs_num_objects {
|
||||
// Clear/empty PCS — closes any pending display. Drop the
|
||||
// clear segment itself; BlockDuration covers the screen
|
||||
// wipe.
|
||||
Some(0) => {
|
||||
if let Some((start_pts, data)) = self.pending.take() {
|
||||
let duration = pts_ns.saturating_sub(start_pts).max(0) as u64;
|
||||
out.push(Frame {
|
||||
pts_ns: start_pts,
|
||||
keyframe: true,
|
||||
data,
|
||||
duration_ns: Some(duration),
|
||||
});
|
||||
}
|
||||
}
|
||||
// Display PCS — start a new pending. If a prior display
|
||||
// was never explicitly cleared (replace-without-clear),
|
||||
// emit it with the new PCS's PTS as its end.
|
||||
Some(_) => {
|
||||
if let Some((start_pts, data)) = self.pending.take() {
|
||||
let duration = pts_ns.saturating_sub(start_pts).max(0) as u64;
|
||||
out.push(Frame {
|
||||
pts_ns: start_pts,
|
||||
keyframe: true,
|
||||
data,
|
||||
duration_ns: Some(duration),
|
||||
});
|
||||
}
|
||||
self.pending = Some((pts_ns, pes.data.clone()));
|
||||
}
|
||||
// Non-PCS first segment — either a continuation of the
|
||||
// current display set, or non-standard layout. If we have
|
||||
// a pending display, append; otherwise emit as-is.
|
||||
None => {
|
||||
if let Some((_, ref mut buf)) = self.pending {
|
||||
// Bound accumulation: a well-formed display set is small.
|
||||
// Past the cap, drop further appends (malformed stream);
|
||||
// the next PCS will take/replace `pending` and resync.
|
||||
if buf.len() + pes.data.len() <= MAX_PGS_PENDING_BYTES {
|
||||
buf.extend_from_slice(&pes.data);
|
||||
}
|
||||
} else {
|
||||
out.push(Frame {
|
||||
pts_ns,
|
||||
keyframe: true,
|
||||
data: pes.data.clone(),
|
||||
duration_ns: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
fn codec_private(&self) -> Option<Vec<u8>> {
|
||||
@@ -52,29 +136,86 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_basic_segment() {
|
||||
let mut parser = PgsParser::new();
|
||||
// PGS segment data (PCS = presentation composition segment)
|
||||
let data = vec![0x16, 0x00, 0x00, 0x11, 0x01, 0x02, 0x03];
|
||||
let pes = make_pes(data.clone(), Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].data, data);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
// Minimum-viable PCS bytes: type 0x16, segment_length (2 bytes),
|
||||
// then 11 bytes of PCS fields ending in number_of_composition_objects.
|
||||
fn pcs_bytes(num_objects: u8) -> Vec<u8> {
|
||||
let mut v = vec![SEGMENT_PCS, 0x00, 0x0B];
|
||||
v.extend_from_slice(&[0x07, 0x80, 0x04, 0x38]); // 1920x1080
|
||||
v.push(0x10); // frame_rate
|
||||
v.extend_from_slice(&[0x00, 0x01]); // composition_number
|
||||
v.push(0x80); // composition_state = EpochStart
|
||||
v.push(0x00); // palette_update + reserved
|
||||
v.push(0x00); // palette_id_ref
|
||||
v.push(num_objects);
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_keyframes() {
|
||||
fn display_then_clear_yields_duration() {
|
||||
let mut parser = PgsParser::new();
|
||||
for i in 0..3 {
|
||||
let data = vec![0x16, 0x00, i];
|
||||
let pes = make_pes(data, Some(90000 * i as i64));
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert!(frames[0].keyframe, "PGS segment should always be keyframe");
|
||||
|
||||
// Display PCS at PTS 90000 (= 1s)
|
||||
let display = pcs_bytes(1);
|
||||
let frames = parser.parse(&make_pes(display.clone(), Some(90000)));
|
||||
assert!(frames.is_empty(), "display PCS should be pending");
|
||||
|
||||
// Empty PCS at PTS 270000 (= 3s)
|
||||
let clear = pcs_bytes(0);
|
||||
let frames = parser.parse(&make_pes(clear, Some(270000)));
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
assert_eq!(frames[0].duration_ns, Some(2_000_000_000));
|
||||
assert_eq!(frames[0].data, display);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replace_without_clear_still_emits_prior_with_duration() {
|
||||
let mut parser = PgsParser::new();
|
||||
let _ = parser.parse(&make_pes(pcs_bytes(1), Some(90000)));
|
||||
let frames = parser.parse(&make_pes(pcs_bytes(1), Some(180000)));
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
assert_eq!(frames[0].duration_ns, Some(1_000_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_pcs_segment_appends_to_pending() {
|
||||
let mut parser = PgsParser::new();
|
||||
let _ = parser.parse(&make_pes(pcs_bytes(1), Some(90000)));
|
||||
// ODS-like segment (type 0x15)
|
||||
let frames = parser.parse(&make_pes(vec![0x15, 0x00, 0x02, 0xAA, 0xBB], Some(90000)));
|
||||
assert!(frames.is_empty());
|
||||
// Clear closes the set; data should include the appended bytes.
|
||||
let frames = parser.parse(&make_pes(pcs_bytes(0), Some(180000)));
|
||||
assert_eq!(frames.len(), 1);
|
||||
let data = &frames[0].data;
|
||||
assert!(data.windows(5).any(|w| w == [0x15, 0x00, 0x02, 0xAA, 0xBB]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_buffer_is_capped() {
|
||||
let mut parser = PgsParser::new();
|
||||
// Open a display set.
|
||||
let _ = parser.parse(&make_pes(pcs_bytes(1), Some(90000)));
|
||||
|
||||
// Flood with non-PCS segments far exceeding the cap.
|
||||
let chunk = vec![0x15u8; 256 * 1024]; // 256 KB ODS-like segment
|
||||
let floods = (MAX_PGS_PENDING_BYTES / chunk.len()) + 32;
|
||||
for _ in 0..floods {
|
||||
let frames = parser.parse(&make_pes(chunk.clone(), Some(90000)));
|
||||
assert!(frames.is_empty(), "non-PCS appends should not emit");
|
||||
}
|
||||
|
||||
// The pending buffer must not have grown without bound.
|
||||
let pending_len = parser.pending.as_ref().map(|(_, b)| b.len()).unwrap_or(0);
|
||||
assert!(
|
||||
pending_len <= MAX_PGS_PENDING_BYTES,
|
||||
"pending buffer {pending_len} exceeded cap {MAX_PGS_PENDING_BYTES}"
|
||||
);
|
||||
|
||||
// A following PCS still resyncs and emits the (capped) pending set.
|
||||
let frames = parser.parse(&make_pes(pcs_bytes(0), Some(180000)));
|
||||
assert_eq!(frames.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -16,6 +16,12 @@ use super::{CodecParser, Frame, PesPacket, pts_to_ns};
|
||||
/// Duration of one TrueHD access unit in nanoseconds (1/1200 second).
|
||||
const AU_DURATION_NS: i64 = 833_333;
|
||||
|
||||
/// Hard cap on the reassembly buffer. A valid TrueHD/MAT access unit is
|
||||
/// well under 32 KiB; if the buffer grows far past that without yielding a
|
||||
/// frame the stream is malformed, so we drop it and resync rather than grow
|
||||
/// without bound. Parity with the AC-3 / DTS / PGS caps.
|
||||
const MAX_TRUEHD_BUF: usize = 256 * 1024;
|
||||
|
||||
pub struct TrueHdParser {
|
||||
buf: Vec<u8>,
|
||||
next_pts_ns: i64,
|
||||
@@ -139,11 +145,18 @@ impl CodecParser for TrueHdParser {
|
||||
pts_ns: self.next_pts_ns,
|
||||
keyframe: is_major_sync,
|
||||
data: self.buf[..unit_bytes].to_vec(),
|
||||
duration_ns: None,
|
||||
});
|
||||
self.buf.drain(..unit_bytes);
|
||||
self.next_pts_ns += AU_DURATION_NS;
|
||||
}
|
||||
|
||||
// Bound memory on malformed input: a stream that never yields a
|
||||
// complete frame must not grow the buffer without limit.
|
||||
if self.buf.len() > MAX_TRUEHD_BUF {
|
||||
self.buf.clear();
|
||||
}
|
||||
|
||||
frames
|
||||
}
|
||||
|
||||
@@ -152,6 +165,52 @@ impl CodecParser for TrueHdParser {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-bit channel counts for the TrueHD 8-channel and 6-channel presentation
|
||||
/// channel-assignment masks (per the MLP spec / FFmpeg `thd_channels`). Some
|
||||
/// bits denote a stereo pair (2), others a single channel (1).
|
||||
const THD_8CH: [u8; 13] = [2, 1, 1, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1];
|
||||
const THD_6CH: [u8; 5] = [2, 1, 1, 2, 1];
|
||||
|
||||
/// Decode the true channel count from a TrueHD major-sync `format_info` word
|
||||
/// (the 32 bits immediately after the 0xF8726FBA sync). Returns the richest
|
||||
/// presentation's channel count — the 8-channel (e.g. 7.1) presentation when
|
||||
/// present, else the 6-channel (5.1) one. This is the real layout that the MPLS
|
||||
/// `audio_format` base field (often 5.1 even on a 7.1/Atmos track) understates.
|
||||
pub fn truehd_channels(format_info: u32) -> Option<u8> {
|
||||
let ch8 = (format_info & 0x1FFF) as u16; // 8ch_presentation_channel_assignment (13 bits)
|
||||
let ch6 = ((format_info >> 15) & 0x1F) as u16; // 6ch_presentation_channel_assignment (5 bits)
|
||||
let count = |mask: u16, tbl: &[u8]| -> u8 {
|
||||
tbl.iter()
|
||||
.enumerate()
|
||||
.filter(|(i, _)| mask & (1 << i) != 0)
|
||||
.map(|(_, &c)| c)
|
||||
.sum()
|
||||
};
|
||||
if ch8 != 0 {
|
||||
Some(count(ch8, &THD_8CH))
|
||||
} else if ch6 != 0 {
|
||||
Some(count(ch6, &THD_6CH))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Scan a demuxed TrueHD elementary-stream chunk for the first major sync and
|
||||
/// decode its true channel count. The stream may interleave AC-3; we scan for
|
||||
/// the major-sync word anywhere and read the following `format_info`.
|
||||
pub fn truehd_channels_from_stream(data: &[u8]) -> Option<u8> {
|
||||
let mut p = 0;
|
||||
while p + 8 <= data.len() {
|
||||
let w = u32::from_be_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]]);
|
||||
if (w & 0xFFFF_FFFE) == 0xF872_6FBA {
|
||||
let fi = u32::from_be_bytes([data[p + 4], data[p + 5], data[p + 6], data[p + 7]]);
|
||||
return truehd_channels(fi);
|
||||
}
|
||||
p += 1;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -246,4 +305,27 @@ mod tests {
|
||||
let parser = TrueHdParser::new();
|
||||
assert!(parser.codec_private().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truehd_channels_71_from_8ch_presentation() {
|
||||
// 8ch presentation assignment bits 0-4 (LR,C,LFE,LsRs,back-LR) = 2+1+1+2+2 = 8.
|
||||
let format_info = 0x1F; // low 13 bits = 0x1F
|
||||
assert_eq!(truehd_channels(format_info), Some(8));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truehd_channels_51_from_6ch_presentation() {
|
||||
// No 8ch presentation; 6ch bits 0-3 (LR,C,LFE,LsRs) = 2+1+1+2 = 6.
|
||||
let format_info = 0xF << 15; // 6ch field = 0xF, 8ch field = 0
|
||||
assert_eq!(truehd_channels(format_info), Some(6));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truehd_channels_scan_finds_major_sync() {
|
||||
// [junk][major sync 0xF8726FBA][format_info: 8ch=0x1F -> 7.1]
|
||||
let mut data = vec![0xAA, 0xBB];
|
||||
data.extend_from_slice(&0xF872_6FBAu32.to_be_bytes());
|
||||
data.extend_from_slice(&0x0000_001Fu32.to_be_bytes());
|
||||
assert_eq!(truehd_channels_from_stream(&data), Some(8));
|
||||
}
|
||||
}
|
||||
|
||||
+65
-10
@@ -41,9 +41,13 @@ impl CodecParser for Vc1Parser {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Use DTS when available (monotonic for B-frame content), fall back to PTS
|
||||
let ts_ns = pes.dts.or(pes.pts).map(pts_to_ns).unwrap_or(0);
|
||||
// MKV block timecodes are PRESENTATION timestamps; frames are stored in
|
||||
// decode order and the player reorders by timecode. Use PTS, not DTS —
|
||||
// DTS presents B-frames in decode order (visible judder) and breaks
|
||||
// PTS-based seeking. Fall back to DTS only if PTS is absent.
|
||||
let ts_ns = pes.pts.or(pes.dts).map(pts_to_ns).unwrap_or(0);
|
||||
let mut has_seq_header = false;
|
||||
let mut has_entry_point = false;
|
||||
let mut frame_start: Option<usize> = None;
|
||||
|
||||
// Scan for start codes (00 00 01 XX)
|
||||
@@ -67,6 +71,7 @@ impl CodecParser for Vc1Parser {
|
||||
SC_ENTRY_POINT => {
|
||||
let end = find_next_sc(data, i + 4).unwrap_or(data.len());
|
||||
self.entry_point = Some(data[i..end].to_vec());
|
||||
has_entry_point = true;
|
||||
}
|
||||
SC_FRAME => {
|
||||
// Frame data starts at this start code
|
||||
@@ -85,17 +90,30 @@ impl CodecParser for Vc1Parser {
|
||||
// Keyframe = this PES contains a sequence header (I-frame indicator in BD)
|
||||
let keyframe = has_seq_header;
|
||||
|
||||
// Strip sequence header + entry point from frame data — those are in codecPrivate.
|
||||
// Only include data from the frame start code onwards.
|
||||
// Strip sequence header + entry point from frame data — those are in
|
||||
// codecPrivate, not coded-picture data. Only include data from the
|
||||
// frame start code onwards.
|
||||
let frame_data = match frame_start {
|
||||
Some(start) => &data[start..],
|
||||
None => data, // no frame start code found, pass through entire PES
|
||||
None => {
|
||||
// No frame start code. If this PES carried only parameter sets
|
||||
// (sequence header / entry point, captured above into
|
||||
// codecPrivate), there is no coded picture to emit — drop it
|
||||
// rather than passing parameter bytes through as a bogus
|
||||
// keyframe. Mirrors how the H.264/HEVC parsers skip
|
||||
// parameter-set-only access units.
|
||||
if has_seq_header || has_entry_point {
|
||||
return Vec::new();
|
||||
}
|
||||
data // genuine picture payload with no leading 0x0D — pass through
|
||||
}
|
||||
};
|
||||
|
||||
vec![Frame {
|
||||
pts_ns: ts_ns,
|
||||
keyframe,
|
||||
data: frame_data.to_vec(),
|
||||
duration_ns: None,
|
||||
}]
|
||||
}
|
||||
|
||||
@@ -342,6 +360,42 @@ mod tests {
|
||||
assert_eq!(&frames[0].data[0..4], &[0x00, 0x00, 0x01, SC_FRAME]);
|
||||
}
|
||||
|
||||
// --- parameter-set-only PES (seq header + entry point, no frame SC) ---
|
||||
|
||||
#[test]
|
||||
fn param_set_only_pes_emits_no_frame() {
|
||||
let mut parser = Vc1Parser::new();
|
||||
|
||||
// Sequence header + entry point, but NO frame start code (0x0D).
|
||||
let mut data = Vec::new();
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01, SC_SEQUENCE_HEADER]);
|
||||
data.extend_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]);
|
||||
data.extend_from_slice(&[0x00, 0x00, 0x01, SC_ENTRY_POINT]);
|
||||
data.extend_from_slice(&[0x11, 0x22, 0x33, 0x44]);
|
||||
|
||||
let pes = make_pes(data, Some(90000));
|
||||
let frames = parser.parse(&pes);
|
||||
|
||||
// No coded picture → no frame emitted (parameter bytes must not be
|
||||
// passed through as a bogus keyframe).
|
||||
assert!(
|
||||
frames.is_empty(),
|
||||
"parameter-set-only PES should not emit a frame"
|
||||
);
|
||||
// But codecPrivate is still captured.
|
||||
assert!(parser.seq_header.is_some());
|
||||
assert!(parser.entry_point.is_some());
|
||||
assert!(parser.codec_private().is_some());
|
||||
|
||||
// A following frame-bearing PES still emits its picture.
|
||||
let mut data2 = Vec::new();
|
||||
data2.extend_from_slice(&[0x00, 0x00, 0x01, SC_FRAME]);
|
||||
data2.extend_from_slice(&[0x55, 0x66, 0x77]);
|
||||
let frames2 = parser.parse(&make_pes(data2, Some(180000)));
|
||||
assert_eq!(frames2.len(), 1);
|
||||
assert_eq!(&frames2[0].data[0..4], &[0x00, 0x00, 0x01, SC_FRAME]);
|
||||
}
|
||||
|
||||
// --- empty PES ---
|
||||
|
||||
#[test]
|
||||
@@ -368,10 +422,10 @@ mod tests {
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
}
|
||||
|
||||
// --- DTS preferred over PTS ---
|
||||
// --- PTS (presentation) used for the MKV block timecode, not DTS ---
|
||||
|
||||
#[test]
|
||||
fn dts_preferred_over_pts() {
|
||||
fn pts_preferred_over_dts() {
|
||||
let mut parser = Vc1Parser::new();
|
||||
|
||||
let mut data = Vec::new();
|
||||
@@ -380,13 +434,14 @@ mod tests {
|
||||
|
||||
let pes = PesPacket {
|
||||
pid: 0x1011,
|
||||
pts: Some(180000),
|
||||
dts: Some(90000),
|
||||
pts: Some(180000), // presentation
|
||||
dts: Some(90000), // decode
|
||||
data,
|
||||
};
|
||||
let frames = parser.parse(&pes);
|
||||
assert_eq!(frames.len(), 1);
|
||||
assert_eq!(frames[0].pts_ns, 1_000_000_000);
|
||||
// PTS must be used — MKV block timecodes are presentation timestamps.
|
||||
assert_eq!(frames[0].pts_ns, 2_000_000_000);
|
||||
}
|
||||
|
||||
// --- find_next_sc utility ---
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
//! `DemuxThread` — runs the read+decrypt+demux pipeline on a
|
||||
//! dedicated thread, feeding completed `PesPacket` batches to the
|
||||
//! caller via a bounded channel.
|
||||
//!
|
||||
//! ## Why a second worker thread
|
||||
//!
|
||||
//! With [`crate::sector::PrefetchedSectorSource`] alone, read+decrypt
|
||||
//! already runs on a producer thread; the *consumer* (main) thread
|
||||
//! still serialises `ts_demuxer.feed` (M2TS parsing) with the codec
|
||||
//! parsers. Profiling on the rip1 testbed showed feed at ~37 % and
|
||||
//! codec parse at ~44 % of consumer wall time — i.e. feed is heavy
|
||||
//! enough that pipelining it with parse pays for itself.
|
||||
//!
|
||||
//! Splitting them: feed runs in [`DemuxThread`]; the consumer thread
|
||||
//! receives `Vec<PesPacket>` batches and runs codec parse + frame
|
||||
//! emission only. Total throughput becomes `1/max(feed, parse)`
|
||||
//! instead of `1/(feed + parse)`.
|
||||
//!
|
||||
//! ## Lifecycle
|
||||
//!
|
||||
//! [`DemuxThread::spawn`] takes ownership of the inner reader and the
|
||||
//! demuxer state, returns a handle plus a `Receiver<DemuxBatch>`.
|
||||
//! Dropping the handle closes the channel which signals the thread
|
||||
//! to exit; the join in `Drop::drop` is bounded.
|
||||
|
||||
use crate::halt::Halt;
|
||||
use crate::sector::SectorSource;
|
||||
use crossbeam_channel::{Receiver, Sender, bounded};
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// Output channel depth. Two batches in flight keeps the consumer
|
||||
/// (codec parser) busy without piling up demuxed bytes if it stalls.
|
||||
const DEMUX_CHANNEL_DEPTH: usize = 2;
|
||||
|
||||
/// One demuxed batch flowing from the demux thread to the consumer.
|
||||
pub enum DemuxBatch {
|
||||
/// Successfully demuxed PesPackets — non-empty.
|
||||
Ts(Vec<super::ts::PesPacket>),
|
||||
Ps(Vec<super::ps::PsPacket>),
|
||||
/// Underlying reader returned an error. Terminal.
|
||||
Err(std::io::Error),
|
||||
}
|
||||
|
||||
/// Spawned demux thread. Drop joins.
|
||||
///
|
||||
/// In zero-copy mode the thread also owns an opaque
|
||||
/// `producer_shell: Option<Box<dyn Send>>` — the join handle of the
|
||||
/// upstream producer (sector or byte prefetcher). Dropping the
|
||||
/// `DemuxThread` runs the shell's `Drop`, which joins the producer.
|
||||
/// `Box<dyn Send>` rather than a concrete type so the same demux
|
||||
/// worker can be wired behind either prefetcher kind.
|
||||
pub struct DemuxThread {
|
||||
handle: Option<JoinHandle<()>>,
|
||||
#[allow(dead_code)]
|
||||
producer_shell: Option<Box<dyn Send>>,
|
||||
}
|
||||
|
||||
impl DemuxThread {
|
||||
/// Spawn the demux thread. Returns the thread handle and a
|
||||
/// receiver for [`DemuxBatch`] items.
|
||||
///
|
||||
/// `reader` is the fully-composed read+decrypt stack (e.g.
|
||||
/// [`PrefetchedSectorSource`](crate::sector::PrefetchedSectorSource)
|
||||
/// wrapping
|
||||
/// [`DecryptingSectorSource`](crate::sector::DecryptingSectorSource)).
|
||||
/// `extents` is what the thread walks; it issues one
|
||||
/// `read_sectors` per batch of `batch_sectors` sectors (aligned
|
||||
/// to 3-sector AACS units when possible).
|
||||
pub fn spawn<S: SectorSource + Send + 'static>(
|
||||
mut reader: S,
|
||||
extents: Vec<crate::disc::Extent>,
|
||||
batch_sectors: u16,
|
||||
halt: Option<Halt>,
|
||||
ts: Option<super::ts::TsDemuxer>,
|
||||
ps: Option<super::ps::PsDemuxer>,
|
||||
) -> (Self, Receiver<DemuxBatch>) {
|
||||
let (tx, rx) = bounded::<DemuxBatch>(DEMUX_CHANNEL_DEPTH);
|
||||
let mut ts = ts;
|
||||
let mut ps = ps;
|
||||
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("freemkv-demux".into())
|
||||
.spawn(move || {
|
||||
let mut buf = vec![0u8; batch_sectors as usize * 2048];
|
||||
let mut ext_idx = 0usize;
|
||||
let mut offset: u32 = 0;
|
||||
let prof = std::env::var_os("FREEMKV_PROFILE").is_some();
|
||||
let mut prof_started = std::time::Instant::now();
|
||||
let mut prof_last_dump = prof_started;
|
||||
let mut prof_read_ns: u128 = 0;
|
||||
let mut prof_feed_ns: u128 = 0;
|
||||
let mut prof_send_ns: u128 = 0;
|
||||
let mut prof_bytes: u64 = 0;
|
||||
while ext_idx < extents.len() {
|
||||
if halt.as_ref().map(|h| h.is_cancelled()).unwrap_or(false) {
|
||||
return;
|
||||
}
|
||||
let ext = &extents[ext_idx];
|
||||
let remaining = ext.sector_count.saturating_sub(offset);
|
||||
if remaining == 0 {
|
||||
ext_idx += 1;
|
||||
offset = 0;
|
||||
continue;
|
||||
}
|
||||
let mut sectors = remaining.min(batch_sectors as u32) as u16;
|
||||
if sectors >= 3 {
|
||||
sectors -= sectors % 3;
|
||||
}
|
||||
let bytes = sectors as usize * 2048;
|
||||
if buf.len() < bytes {
|
||||
buf.resize(bytes, 0);
|
||||
}
|
||||
let lba = ext.start_lba + offset;
|
||||
let t0 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let n = match reader.read_sectors(lba, sectors, &mut buf[..bytes], false) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
let _ = tx.send(DemuxBatch::Err(e.into()));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let t1 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
offset += sectors as u32;
|
||||
|
||||
// Demux this batch immediately so the channel
|
||||
// carries already-parsed PesPackets, not raw
|
||||
// sector bytes.
|
||||
if let Some(ref mut d) = ts {
|
||||
let pkts = d.feed(&buf[..n]);
|
||||
let t2 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if !pkts.is_empty() && tx.send(DemuxBatch::Ts(pkts)).is_err() {
|
||||
return; // consumer dropped
|
||||
}
|
||||
let t3 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if prof {
|
||||
prof_read_ns += t1.unwrap().duration_since(t0.unwrap()).as_nanos();
|
||||
prof_feed_ns += t2.unwrap().duration_since(t1.unwrap()).as_nanos();
|
||||
prof_send_ns += t3.unwrap().duration_since(t2.unwrap()).as_nanos();
|
||||
prof_bytes += n as u64;
|
||||
let now = t3.unwrap();
|
||||
if now.duration_since(prof_last_dump)
|
||||
>= std::time::Duration::from_secs(5)
|
||||
{
|
||||
let el = now.duration_since(prof_started).as_millis().max(1);
|
||||
let mbps = prof_bytes as u128 * 1000 / 1_000_000 / el;
|
||||
eprintln!(
|
||||
"[demux] elapsed={}ms in={}MB/s read={}% feed={}% send={}%",
|
||||
el,
|
||||
mbps,
|
||||
prof_read_ns / 10_000 / el,
|
||||
prof_feed_ns / 10_000 / el,
|
||||
prof_send_ns / 10_000 / el,
|
||||
);
|
||||
prof_last_dump = now;
|
||||
prof_started = now;
|
||||
prof_read_ns = 0;
|
||||
prof_feed_ns = 0;
|
||||
prof_send_ns = 0;
|
||||
prof_bytes = 0;
|
||||
}
|
||||
}
|
||||
} else if let Some(ref mut d) = ps {
|
||||
let pkts = d.feed(&buf[..n]);
|
||||
if !pkts.is_empty() && tx.send(DemuxBatch::Ps(pkts)).is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
// EOF — emit any flushed packets too.
|
||||
if let Some(ref mut d) = ts {
|
||||
let tail = d.flush();
|
||||
if !tail.is_empty() {
|
||||
let _ = tx.send(DemuxBatch::Ts(tail));
|
||||
}
|
||||
} else if let Some(ref mut d) = ps {
|
||||
let tail = d.flush();
|
||||
if !tail.is_empty() {
|
||||
let _ = tx.send(DemuxBatch::Ps(tail));
|
||||
}
|
||||
}
|
||||
// Sender drops here -> consumer sees RecvError → EOF.
|
||||
})
|
||||
.expect("freemkv-demux thread spawn failed");
|
||||
|
||||
(
|
||||
Self {
|
||||
handle: Some(handle),
|
||||
producer_shell: None,
|
||||
},
|
||||
rx,
|
||||
)
|
||||
}
|
||||
|
||||
/// Zero-copy variant. Instead of taking a `SectorSource` and
|
||||
/// memcpy-ing through its `read_sectors` API, this constructor
|
||||
/// consumes the prefetch channels directly: filled buffers come
|
||||
/// in via `prefetch_rx`, the demux thread feeds them, then
|
||||
/// returns them to `recycle_tx` for the producer to re-fill.
|
||||
/// Eliminates the 16 MiB memcpy per batch that the SectorSource
|
||||
/// adapter incurred (and, with the producer-side recycling pool,
|
||||
/// also eliminates the per-batch heap alloc / cross-thread free
|
||||
/// that was costing 40 %+ of demux-thread time before).
|
||||
///
|
||||
/// `producer_shell` is an opaque handle whose only purpose is to
|
||||
/// outlive the demux thread and join the upstream producer when
|
||||
/// dropped. Both
|
||||
/// [`crate::sector::PrefetchedSectorSource::into_channels`] and
|
||||
/// [`crate::io::byte_prefetcher::BytePrefetcher::into_channels`]
|
||||
/// hand back a shell that fits — pass either.
|
||||
pub fn spawn_zero_copy<S: Send + 'static>(
|
||||
prefetch_rx: Receiver<std::io::Result<Vec<u8>>>,
|
||||
recycle_tx: Sender<Vec<u8>>,
|
||||
producer_shell: S,
|
||||
halt: Option<Halt>,
|
||||
ts: Option<super::ts::TsDemuxer>,
|
||||
ps: Option<super::ps::PsDemuxer>,
|
||||
) -> (Self, Receiver<DemuxBatch>) {
|
||||
let (tx, rx) = bounded::<DemuxBatch>(DEMUX_CHANNEL_DEPTH);
|
||||
let mut ts = ts;
|
||||
let mut ps = ps;
|
||||
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("freemkv-demux".into())
|
||||
.spawn(move || {
|
||||
let prof = std::env::var_os("FREEMKV_PROFILE").is_some();
|
||||
let mut prof_started = std::time::Instant::now();
|
||||
let mut prof_last_dump = prof_started;
|
||||
let mut prof_read_ns: u128 = 0;
|
||||
let mut prof_feed_ns: u128 = 0;
|
||||
let mut prof_bytes: u64 = 0;
|
||||
loop {
|
||||
if halt.as_ref().map(|h| h.is_cancelled()).unwrap_or(false) {
|
||||
return;
|
||||
}
|
||||
let t0 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let buf = match prefetch_rx.recv() {
|
||||
Ok(Ok(b)) => b,
|
||||
Ok(Err(e)) => {
|
||||
let _ = tx.send(DemuxBatch::Err(e));
|
||||
return;
|
||||
}
|
||||
Err(_) => break, // producer done → EOF
|
||||
};
|
||||
let t1 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let n = buf.len();
|
||||
if let Some(ref mut d) = ts {
|
||||
let pkts = d.feed(&buf);
|
||||
let t2 = if prof {
|
||||
Some(std::time::Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Recycle the buffer back to the producer
|
||||
// before pushing the demuxed packets. If the
|
||||
// recycle channel is closed the producer has
|
||||
// exited; we drop the buffer and continue.
|
||||
let _ = recycle_tx.send(buf);
|
||||
if !pkts.is_empty() && tx.send(DemuxBatch::Ts(pkts)).is_err() {
|
||||
return;
|
||||
}
|
||||
if prof {
|
||||
prof_read_ns += t1.unwrap().duration_since(t0.unwrap()).as_nanos();
|
||||
prof_feed_ns += t2.unwrap().duration_since(t1.unwrap()).as_nanos();
|
||||
prof_bytes += n as u64;
|
||||
let now = std::time::Instant::now();
|
||||
if now.duration_since(prof_last_dump)
|
||||
>= std::time::Duration::from_secs(5)
|
||||
{
|
||||
let el = now.duration_since(prof_started).as_millis().max(1);
|
||||
let mbps = prof_bytes as u128 * 1000 / 1_000_000 / el;
|
||||
eprintln!(
|
||||
"[demux] elapsed={}ms in={}MB/s read={}% feed={}%",
|
||||
el,
|
||||
mbps,
|
||||
prof_read_ns / 10_000 / el,
|
||||
prof_feed_ns / 10_000 / el,
|
||||
);
|
||||
prof_last_dump = now;
|
||||
prof_started = now;
|
||||
prof_read_ns = 0;
|
||||
prof_feed_ns = 0;
|
||||
prof_bytes = 0;
|
||||
}
|
||||
}
|
||||
} else if let Some(ref mut d) = ps {
|
||||
let pkts = d.feed(&buf);
|
||||
let _ = recycle_tx.send(buf);
|
||||
if !pkts.is_empty() && tx.send(DemuxBatch::Ps(pkts)).is_err() {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
let _ = recycle_tx.send(buf);
|
||||
}
|
||||
}
|
||||
// Flush tail packets at EOF.
|
||||
if let Some(ref mut d) = ts {
|
||||
let tail = d.flush();
|
||||
if !tail.is_empty() {
|
||||
let _ = tx.send(DemuxBatch::Ts(tail));
|
||||
}
|
||||
} else if let Some(ref mut d) = ps {
|
||||
let tail = d.flush();
|
||||
if !tail.is_empty() {
|
||||
let _ = tx.send(DemuxBatch::Ps(tail));
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("freemkv-demux thread spawn failed");
|
||||
|
||||
(
|
||||
Self {
|
||||
handle: Some(handle),
|
||||
producer_shell: Some(Box::new(producer_shell)),
|
||||
},
|
||||
rx,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DemuxThread {
|
||||
fn drop(&mut self) {
|
||||
if let Some(h) = self.handle.take() {
|
||||
let _ = h.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
+345
-35
@@ -1,17 +1,18 @@
|
||||
//! DiscStream — read any disc (physical drive or ISO file) → PES frames.
|
||||
//!
|
||||
//! One stream type for all disc sources. The source is a SectorReader —
|
||||
//! Drive (hardware) or IsoSectorReader (file). DiscStream doesn't care.
|
||||
//! One stream type for all disc sources. The source is a SectorSource —
|
||||
//! Drive (hardware) or FileSectorSource (file). DiscStream doesn't care.
|
||||
//!
|
||||
//! Read-only. For disc→ISO (raw sector copy), use `Disc::copy()`.
|
||||
|
||||
use crate::disc::{Disc, DiscTitle, Extent};
|
||||
use crate::drive::extract_scsi_context;
|
||||
use crate::event::{BatchSizeReason, Event, EventKind};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::halt::Halt;
|
||||
use crate::sector::{DecryptingSectorSource, SectorSource};
|
||||
use std::io;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
/// Ramp back up to the preferred batch size after this many sectors
|
||||
/// of clean reading at the current (reduced) size. 100 MiB = 51,200 sectors.
|
||||
@@ -96,12 +97,22 @@ impl AdaptiveBatch {
|
||||
|
||||
/// Disc stream. Reads sectors from any source → PES frames.
|
||||
///
|
||||
/// Sources: physical drive, ISO file, or any SectorReader.
|
||||
/// Sources: physical drive, ISO file, or any SectorSource.
|
||||
/// Decrypt, demux, and codec parsing happen internally.
|
||||
pub struct DiscStream {
|
||||
reader: Box<dyn SectorReader>,
|
||||
/// Underlying sector source wrapped in the 0.18
|
||||
/// [`DecryptingSectorSource`] decorator. Every `read_sectors`
|
||||
/// call yields plaintext, so `fill_extents` no longer needs an
|
||||
/// inline `decrypt::decrypt_sectors` step. `DecryptKeys::None`
|
||||
/// (raw / unencrypted disc) makes the decorator a pass-through.
|
||||
reader: DecryptingSectorSource<Box<dyn SectorSource>>,
|
||||
title: DiscTitle,
|
||||
disc: Option<Disc>,
|
||||
/// Mirror of the keys handed in at construction. The decorator
|
||||
/// owns the cryptographic state; this field is kept for
|
||||
/// metadata-side callers (`info()` and friends) that want to
|
||||
/// know whether the disc was encrypted, without reaching through
|
||||
/// the wrapper.
|
||||
decrypt_keys: crate::decrypt::DecryptKeys,
|
||||
|
||||
// Extents to read
|
||||
@@ -120,11 +131,13 @@ pub struct DiscStream {
|
||||
adaptive: AdaptiveBatch,
|
||||
pub errors: u64,
|
||||
pub skip_errors: bool,
|
||||
/// When set and the flag is raised, fill_extents returns Err(Halted) at the
|
||||
/// next retry boundary. Unlike skip_errors, this propagates the error up so
|
||||
/// the rip terminates cleanly. Share the Arc with Drive::halt_flag() to get
|
||||
/// unified Stop behavior across drive reads and sector processing.
|
||||
halt: Option<Arc<AtomicBool>>,
|
||||
/// When set and the token is cancelled, fill_extents returns Err(Halted)
|
||||
/// at the next retry boundary. Unlike skip_errors, this propagates the
|
||||
/// error up so the rip terminates cleanly. Construct with
|
||||
/// [`DiscStream::with_halt`] (preferred) or set post-hoc via the
|
||||
/// deprecated [`DiscStream::set_halt`] bridge — both populate this same
|
||||
/// field and either entry point yields one source of truth.
|
||||
halt: Option<Halt>,
|
||||
event_fn: Option<Box<dyn Fn(Event) + Send>>,
|
||||
eof: bool,
|
||||
|
||||
@@ -136,7 +149,11 @@ pub struct DiscStream {
|
||||
// a percent without a separate API call.
|
||||
bytes_total_extents: u64,
|
||||
|
||||
// PES output
|
||||
// PES output — single-threaded inline demux + codec parse. The
|
||||
// pipeline-mode mux (3-stage threaded) lives in
|
||||
// [`super::pipelined_stream::PipelinedPesStream`]; this type is
|
||||
// the legacy in-thread path for live-disc reads where adaptive
|
||||
// batch retry on bad sectors lives in `fill_extents`.
|
||||
ts_demuxer: Option<super::ts::TsDemuxer>,
|
||||
ps_demuxer: Option<super::ps::PsDemuxer>,
|
||||
parsers: Vec<(u16, Box<dyn super::codec::CodecParser>)>,
|
||||
@@ -147,11 +164,11 @@ pub struct DiscStream {
|
||||
impl DiscStream {
|
||||
/// Create a disc stream from any sector reader.
|
||||
///
|
||||
/// Works with physical drives and ISO files — both implement SectorReader.
|
||||
/// Works with physical drives and ISO files — both implement SectorSource.
|
||||
/// The caller opens the source, scans for titles/keys, and passes them in.
|
||||
/// The stream handles demuxing, decryption, and codec parsing internally.
|
||||
pub fn new(
|
||||
reader: Box<dyn SectorReader>,
|
||||
reader: Box<dyn SectorSource>,
|
||||
title: DiscTitle,
|
||||
decrypt_keys: crate::decrypt::DecryptKeys,
|
||||
batch_sectors: u16,
|
||||
@@ -160,6 +177,13 @@ impl DiscStream {
|
||||
let extents = title.extents.clone();
|
||||
let bytes_total_extents: u64 = extents.iter().map(|e| e.sector_count as u64 * 2048).sum();
|
||||
|
||||
// Debug log reader type at construction — critical for diagnosing mux reading from drive instead of ISO
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"DiscStream constructed with reader type: {}",
|
||||
std::any::type_name::<dyn SectorSource>()
|
||||
);
|
||||
|
||||
let mut pids = Vec::new();
|
||||
let mut parsers = Vec::new();
|
||||
let mut pid_to_track = Vec::new();
|
||||
@@ -171,7 +195,8 @@ impl DiscStream {
|
||||
};
|
||||
pids.push(pid);
|
||||
pid_to_track.push((pid, idx));
|
||||
parsers.push((pid, super::codec::parser_for_codec(codec, None)));
|
||||
let is_dvd_ps = matches!(content_format, crate::disc::ContentFormat::MpegPs);
|
||||
parsers.push((pid, super::codec::parser_for_codec(codec, None, is_dvd_ps)));
|
||||
}
|
||||
|
||||
let mut ts_demuxer = None;
|
||||
@@ -189,7 +214,11 @@ impl DiscStream {
|
||||
}
|
||||
|
||||
Self {
|
||||
reader,
|
||||
// Wrap the input reader in DecryptingSectorSource so the
|
||||
// internal fill_extents path sees plaintext bytes. For
|
||||
// DecryptKeys::None (unencrypted / raw / test fixtures)
|
||||
// the decorator is a pass-through.
|
||||
reader: DecryptingSectorSource::new(reader, decrypt_keys.clone()),
|
||||
title,
|
||||
disc: None,
|
||||
decrypt_keys,
|
||||
@@ -219,19 +248,39 @@ impl DiscStream {
|
||||
self.event_fn = Some(Box::new(f));
|
||||
}
|
||||
|
||||
/// Share a halt flag — typically from `Drive::halt_flag()`. When raised,
|
||||
/// the next read-retry boundary inside fill_extents returns Err(Halted)
|
||||
/// instead of continuing. Required for Stop to work during dense bad-sector
|
||||
/// regions (where read() loops internally waiting for enough clean data to
|
||||
/// emit a frame and would otherwise never check an external stop signal).
|
||||
/// Constructor-time builder: attach a [`Halt`] token so that when
|
||||
/// any clone is cancelled, the next read-retry boundary inside
|
||||
/// `fill_extents` returns `Err(Halted)`. Required for Stop to work
|
||||
/// during dense bad-sector regions (where the outer PES read() loop
|
||||
/// can spend minutes inside fill_extents before emitting a frame).
|
||||
///
|
||||
/// Preferred over the post-hoc [`DiscStream::set_halt`] bridge —
|
||||
/// pass the same `Halt` clone you hand to sweep / patch / mux so
|
||||
/// every phase observes a single Stop signal.
|
||||
pub fn with_halt(mut self, halt: Halt) -> Self {
|
||||
self.halt = Some(halt);
|
||||
self
|
||||
}
|
||||
|
||||
/// Bridge for callers that haven't migrated to the
|
||||
/// [`DiscStream::with_halt`] constructor-time path yet. Wraps the
|
||||
/// supplied `Arc<AtomicBool>` as a [`Halt`] (`Halt::from_arc`) and
|
||||
/// stores it in the same internal slot, so a halt installed via
|
||||
/// either entry point goes through one halt-check inside
|
||||
/// `fill_extents`. Calling `set_halt` after `with_halt` (or vice
|
||||
/// versa) replaces the previous token with the new one.
|
||||
#[deprecated(
|
||||
since = "0.18.0",
|
||||
note = "use `DiscStream::with_halt(Halt)` at construction instead"
|
||||
)]
|
||||
pub fn set_halt(&mut self, flag: Arc<AtomicBool>) {
|
||||
self.halt = Some(flag);
|
||||
self.halt = Some(Halt::from_arc(flag));
|
||||
}
|
||||
|
||||
fn is_halted(&self) -> bool {
|
||||
self.halt
|
||||
.as_ref()
|
||||
.map(|h| h.load(Ordering::Relaxed))
|
||||
.map(|h| h.is_cancelled())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
@@ -241,9 +290,12 @@ impl DiscStream {
|
||||
}
|
||||
}
|
||||
|
||||
/// Skip decryption — return raw encrypted bytes.
|
||||
/// Skip decryption — return raw encrypted bytes. Updates both
|
||||
/// the metadata-side key field and the wrapped reader's keys so
|
||||
/// subsequent `read_sectors` calls become a pass-through.
|
||||
pub fn set_raw(&mut self) {
|
||||
self.decrypt_keys = crate::decrypt::DecryptKeys::None;
|
||||
self.reader.set_keys(crate::decrypt::DecryptKeys::None);
|
||||
}
|
||||
|
||||
/// Get the scanned Disc (for listing all titles).
|
||||
@@ -275,10 +327,19 @@ impl DiscStream {
|
||||
// this loop can spend minutes shrinking and skipping sectors; without
|
||||
// the check, Stop wouldn't take effect until the outer PES read() loop
|
||||
// finally emits a frame, which may never happen.
|
||||
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
loop {
|
||||
if self.is_halted() {
|
||||
return Err(crate::error::Error::Halted.into());
|
||||
}
|
||||
|
||||
// Debug: log slow reads during mux — helps diagnose stalls
|
||||
if cfg!(debug_assertions) && start_time.elapsed().as_secs() > 5 {
|
||||
tracing::debug!(target: "mux", "fill_extents waiting at LBA {} ({}s elapsed, sectors={})", lba, start_time.elapsed().as_secs(), remaining);
|
||||
}
|
||||
|
||||
let mut sectors = remaining.min(self.adaptive.current() as u32) as u16;
|
||||
// Align to 3-sector AACS units when possible. Partial units at
|
||||
// extent boundaries are safely handled by decrypt_sectors().
|
||||
@@ -347,6 +408,71 @@ impl DiscStream {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-stage profiling state — populated only when `FREEMKV_PROFILE`
|
||||
/// is set. Dumps a percentage breakdown to stderr every
|
||||
/// [`PROFILE_INTERVAL`]. Zero overhead in normal runs (Option check
|
||||
/// is the only added cost).
|
||||
struct StageProf {
|
||||
started: std::time::Instant,
|
||||
last_dump: std::time::Instant,
|
||||
fill_ns: u128,
|
||||
feed_ns: u128,
|
||||
consume_ns: u128,
|
||||
bytes_in: u64,
|
||||
}
|
||||
|
||||
const PROFILE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
|
||||
thread_local! {
|
||||
static STAGE_PROF: std::cell::RefCell<Option<StageProf>> = const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
fn prof_active() -> bool {
|
||||
std::env::var_os("FREEMKV_PROFILE").is_some()
|
||||
}
|
||||
|
||||
fn prof_tick(stage: &str, ns: u128, bytes: u64) {
|
||||
STAGE_PROF.with(|cell| {
|
||||
let mut slot = cell.borrow_mut();
|
||||
if slot.is_none() {
|
||||
if !prof_active() {
|
||||
return;
|
||||
}
|
||||
let now = std::time::Instant::now();
|
||||
*slot = Some(StageProf {
|
||||
started: now,
|
||||
last_dump: now,
|
||||
fill_ns: 0,
|
||||
feed_ns: 0,
|
||||
consume_ns: 0,
|
||||
bytes_in: 0,
|
||||
});
|
||||
}
|
||||
let p = slot.as_mut().unwrap();
|
||||
match stage {
|
||||
"fill" => p.fill_ns += ns,
|
||||
"feed" => p.feed_ns += ns,
|
||||
"consume" => p.consume_ns += ns,
|
||||
_ => {}
|
||||
}
|
||||
p.bytes_in += bytes;
|
||||
let now = std::time::Instant::now();
|
||||
if now.duration_since(p.last_dump) < PROFILE_INTERVAL {
|
||||
return;
|
||||
}
|
||||
let elapsed_ms = now.duration_since(p.started).as_millis().max(1);
|
||||
let fill_pct = p.fill_ns / 10_000 / elapsed_ms;
|
||||
let feed_pct = p.feed_ns / 10_000 / elapsed_ms;
|
||||
let consume_pct = p.consume_ns / 10_000 / elapsed_ms;
|
||||
let mbps = p.bytes_in as u128 * 1000 / 1_000_000 / elapsed_ms;
|
||||
eprintln!(
|
||||
"[profile] elapsed={}ms in={}MB/s fill={}% feed={}% consume={}%",
|
||||
elapsed_ms, mbps, fill_pct, feed_pct, consume_pct,
|
||||
);
|
||||
p.last_dump = now;
|
||||
});
|
||||
}
|
||||
|
||||
impl crate::pes::Stream for DiscStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
if let Some(frame) = self.pending_frames.pop_front() {
|
||||
@@ -358,6 +484,7 @@ impl crate::pes::Stream for DiscStream {
|
||||
}
|
||||
|
||||
loop {
|
||||
let t0 = std::time::Instant::now();
|
||||
if !self.fill_extents()? {
|
||||
self.eof = true;
|
||||
// Flush demuxer — last PES packet may still be in the assembler
|
||||
@@ -415,33 +542,68 @@ impl crate::pes::Stream for DiscStream {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Drain any access unit a codec parser buffered past the last
|
||||
// PES (DTS-HD's final core+extension unit, assembled across
|
||||
// PES boundaries).
|
||||
let pid_to_track = &self.pid_to_track;
|
||||
let pending = &mut self.pending_frames;
|
||||
for (pid, parser) in self.parsers.iter_mut() {
|
||||
let Some(&(_, track)) = pid_to_track.iter().find(|(p, _)| p == pid) else {
|
||||
continue;
|
||||
};
|
||||
for frame in parser.flush() {
|
||||
pending.push_back(crate::pes::PesFrame::from_codec_frame(track, frame));
|
||||
}
|
||||
}
|
||||
return Ok(self.pending_frames.pop_front());
|
||||
}
|
||||
|
||||
let bytes = self.buf_valid;
|
||||
if let Err(e) =
|
||||
crate::decrypt::decrypt_sectors(&mut self.read_buf[..bytes], &self.decrypt_keys, 0)
|
||||
{
|
||||
return Err(e.into());
|
||||
}
|
||||
let t1 = std::time::Instant::now();
|
||||
prof_tick("fill", t1.duration_since(t0).as_nanos(), bytes as u64);
|
||||
// Plaintext: the wrapped reader (DecryptingSectorSource)
|
||||
// applied AACS / CSS in-place during fill_extents'
|
||||
// read_sectors call. The pre-0.18 inline decrypt step
|
||||
// lived here.
|
||||
|
||||
if let Some(ref mut demuxer) = self.ts_demuxer {
|
||||
let packets = demuxer.feed(&self.read_buf[..bytes]);
|
||||
for pes in &packets {
|
||||
if let Some((_, track)) =
|
||||
self.pid_to_track.iter().find(|(pid, _)| *pid == pes.pid)
|
||||
let t2 = std::time::Instant::now();
|
||||
prof_tick("feed", t2.duration_since(t1).as_nanos(), 0);
|
||||
let skip_parse = std::env::var_os("FREEMKV_SKIP_PARSE").is_some();
|
||||
for pes in packets {
|
||||
if let Some((_, track)) = self
|
||||
.pid_to_track
|
||||
.iter()
|
||||
.find(|(pid, _)| *pid == pes.pid)
|
||||
.copied()
|
||||
{
|
||||
if let Some((_, parser)) =
|
||||
if skip_parse {
|
||||
// Profiling escape hatch — bypass the codec
|
||||
// parser and pass the raw PES bytes straight
|
||||
// through as a single PesFrame. Lets us
|
||||
// attribute consumer-thread time to
|
||||
// "demux + framing" vs "codec parse".
|
||||
self.pending_frames.push_back(crate::pes::PesFrame {
|
||||
track,
|
||||
pts: pes.pts.map(super::codec::pts_to_ns).unwrap_or(0),
|
||||
keyframe: false,
|
||||
data: pes.data,
|
||||
duration_ns: None,
|
||||
});
|
||||
} else if let Some((_, parser)) =
|
||||
self.parsers.iter_mut().find(|(pid, _)| *pid == pes.pid)
|
||||
{
|
||||
for frame in parser.parse(pes) {
|
||||
for frame in parser.parse(&pes) {
|
||||
self.pending_frames.push_back(
|
||||
crate::pes::PesFrame::from_codec_frame(*track, frame),
|
||||
crate::pes::PesFrame::from_codec_frame(track, frame),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let t3 = std::time::Instant::now();
|
||||
prof_tick("consume", t3.duration_since(t2).as_nanos(), 0);
|
||||
} else if let Some(ref mut demuxer) = self.ps_demuxer {
|
||||
let packets = demuxer.feed(&self.read_buf[..bytes]);
|
||||
for ps in &packets {
|
||||
@@ -515,6 +677,13 @@ impl crate::pes::Stream for DiscStream {
|
||||
}
|
||||
|
||||
fn headers_ready(&self) -> bool {
|
||||
// FREEMKV_SKIP_PARSE bypasses codec parsers entirely for
|
||||
// bottleneck profiling, so codec_private is never populated.
|
||||
// Pretend headers are ready immediately in that mode so the
|
||||
// CLI loop doesn't hang waiting for them.
|
||||
if std::env::var_os("FREEMKV_SKIP_PARSE").is_some() {
|
||||
return true;
|
||||
}
|
||||
for (idx, s) in self.title.streams.iter().enumerate() {
|
||||
if let crate::disc::Stream::Video(v) = s {
|
||||
if !v.secondary && self.codec_private(idx).is_none() {
|
||||
@@ -524,4 +693,145 @@ impl crate::pes::Stream for DiscStream {
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn errors(&self) -> u64 {
|
||||
self.errors
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
//! `DiscStream` is the only read-only `Stream` impl in tree (every
|
||||
//! other concrete impl in `mux/*` is bidirectional or write-only).
|
||||
//! These tests lock down a static `Send` assertion plus a
|
||||
//! `Box<dyn Stream>` round trip exercising every method through the
|
||||
//! trait object, so future Send-breaking edits to `DiscStream`'s
|
||||
//! interior types fail at compile time.
|
||||
use super::*;
|
||||
use crate::disc::{ContentFormat, DiscTitle};
|
||||
use crate::pes::Stream;
|
||||
|
||||
/// Static-assert `DiscStream: Send`. The `Stream` trait has `Send` as a
|
||||
/// supertrait — if a future field on `DiscStream` is non-`Send` (e.g.
|
||||
/// a `Box<dyn Read>` instead of `Box<dyn SectorSource>`), this fails
|
||||
/// at compile time, before the runtime trait-object test below.
|
||||
fn _assert_disc_stream_is_send() {
|
||||
fn requires_send<T: Send>() {}
|
||||
requires_send::<DiscStream>();
|
||||
}
|
||||
|
||||
/// Trivial `SectorSource` that yields zeroed sectors. Empty title means
|
||||
/// the demuxer produces no PES frames, so `read()` walks the extents to
|
||||
/// EOF and returns `Ok(None)`. That's enough to exercise the trait-object
|
||||
/// dispatch — the goal here is the bridge, not the demuxer.
|
||||
struct ZeroReader {
|
||||
capacity: u32,
|
||||
}
|
||||
|
||||
impl crate::sector::SectorSource for ZeroReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
_lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> crate::error::Result<usize> {
|
||||
let bytes = count as usize * 2048;
|
||||
buf[..bytes].fill(0);
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
|
||||
fn synthetic_title(sector_count: u32) -> DiscTitle {
|
||||
DiscTitle {
|
||||
extents: vec![crate::disc::Extent {
|
||||
start_lba: 0,
|
||||
sector_count,
|
||||
}],
|
||||
..DiscTitle::empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Smallest credible witness that `DiscStream` flows through `dyn Stream`:
|
||||
/// build a `Box<dyn Stream>`, drive `read()` to EOF, exercise `info()` /
|
||||
/// `headers_ready()` / `codec_private()` through the trait object.
|
||||
#[test]
|
||||
fn stream_via_dyn_object() {
|
||||
let reader = ZeroReader { capacity: 8 };
|
||||
let title = synthetic_title(8);
|
||||
let stream = DiscStream::new(
|
||||
Box::new(reader),
|
||||
title,
|
||||
crate::decrypt::DecryptKeys::None,
|
||||
8,
|
||||
ContentFormat::BdTs,
|
||||
);
|
||||
|
||||
let mut src: Box<dyn Stream> = Box::new(stream);
|
||||
|
||||
// Empty-title fixture has no streams configured, so headers are
|
||||
// trivially ready and codec_private() yields nothing on track 0.
|
||||
assert!(src.headers_ready());
|
||||
assert!(src.codec_private(0).is_none());
|
||||
let _ = src.info();
|
||||
|
||||
// Drive read() to EOF through the trait object — empty-title fixture
|
||||
// produces no frames, but the call still routes through the blanket
|
||||
// dispatch into Stream::read.
|
||||
let mut frames = 0usize;
|
||||
while src.read().expect("read").is_some() {
|
||||
frames += 1;
|
||||
if frames > 1024 {
|
||||
panic!("unexpected unbounded frame stream from empty title");
|
||||
}
|
||||
}
|
||||
assert_eq!(frames, 0);
|
||||
}
|
||||
|
||||
/// `is_halted()` must observe a cancellation signal regardless of
|
||||
/// which entry point installed the token. The deprecated
|
||||
/// `set_halt(Arc<AtomicBool>)` and the new `with_halt(Halt)` are
|
||||
/// two views over one slot — flipping either bit must cause the
|
||||
/// next `fill_extents` retry boundary to bail.
|
||||
#[test]
|
||||
fn halt_via_with_halt_observed_by_is_halted() {
|
||||
let halt = Halt::new();
|
||||
let stream = DiscStream::new(
|
||||
Box::new(ZeroReader { capacity: 8 }),
|
||||
synthetic_title(8),
|
||||
crate::decrypt::DecryptKeys::None,
|
||||
8,
|
||||
crate::disc::ContentFormat::BdTs,
|
||||
)
|
||||
.with_halt(halt.clone());
|
||||
assert!(!stream.is_halted());
|
||||
halt.cancel();
|
||||
assert!(
|
||||
stream.is_halted(),
|
||||
"with_halt token cancellation must be observed by is_halted()"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_via_set_halt_bridge_observed_by_is_halted() {
|
||||
let arc = Arc::new(AtomicBool::new(false));
|
||||
let mut stream = DiscStream::new(
|
||||
Box::new(ZeroReader { capacity: 8 }),
|
||||
synthetic_title(8),
|
||||
crate::decrypt::DecryptKeys::None,
|
||||
8,
|
||||
crate::disc::ContentFormat::BdTs,
|
||||
);
|
||||
stream.set_halt(arc.clone());
|
||||
assert!(!stream.is_halted());
|
||||
arc.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(
|
||||
stream.is_halted(),
|
||||
"set_halt(Arc<AtomicBool>) bridge must observe Arc-side flips"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+59
-13
@@ -264,6 +264,13 @@ pub fn read_element_header(r: &mut impl Read) -> io::Result<(u32, u64, usize)> {
|
||||
|
||||
/// Read an unsigned integer value of `len` bytes.
|
||||
pub fn read_uint_val(r: &mut impl Read, len: usize) -> io::Result<u64> {
|
||||
// An EBML unsigned integer is at most 8 bytes. A malformed element
|
||||
// claiming `len > 8` would index past this stack buffer and panic
|
||||
// (DoS on untrusted input) — reject it at the source so every caller
|
||||
// is safe, not just the ones that pre-check.
|
||||
if len > 8 {
|
||||
return Err(crate::error::Error::MkvInvalid.into());
|
||||
}
|
||||
let mut buf = [0u8; 8];
|
||||
r.read_exact(&mut buf[..len])?;
|
||||
let mut val = 0u64;
|
||||
@@ -273,23 +280,33 @@ pub fn read_uint_val(r: &mut impl Read, len: usize) -> io::Result<u64> {
|
||||
Ok(val)
|
||||
}
|
||||
|
||||
/// Read a float value (4 or 8 bytes).
|
||||
/// Read a float value. EBML floats are exactly 0, 4, or 8 bytes.
|
||||
///
|
||||
/// The previous `else` branch read a fixed 8 bytes for ANY non-4 length,
|
||||
/// so a malformed element with `len > 8` left `len - 8` unconsumed bytes
|
||||
/// (mis-read as the next EBML header → desync of the rest of the parent
|
||||
/// element) and `len < 4` over-read. Consume exactly `len` bytes and
|
||||
/// reject anything that isn't a valid float width.
|
||||
pub fn read_float_val(r: &mut impl Read, len: usize) -> io::Result<f64> {
|
||||
if len == 4 {
|
||||
let mut buf = [0u8; 4];
|
||||
r.read_exact(&mut buf)?;
|
||||
Ok(f32::from_be_bytes(buf) as f64)
|
||||
} else {
|
||||
let mut buf = [0u8; 8];
|
||||
r.read_exact(&mut buf)?;
|
||||
Ok(f64::from_be_bytes(buf))
|
||||
match len {
|
||||
0 => Ok(0.0),
|
||||
4 => {
|
||||
let mut buf = [0u8; 4];
|
||||
r.read_exact(&mut buf)?;
|
||||
Ok(f32::from_be_bytes(buf) as f64)
|
||||
}
|
||||
8 => {
|
||||
let mut buf = [0u8; 8];
|
||||
r.read_exact(&mut buf)?;
|
||||
Ok(f64::from_be_bytes(buf))
|
||||
}
|
||||
_ => Err(crate::error::Error::MkvInvalid.into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a UTF-8 string value of `len` bytes.
|
||||
pub fn read_string_val(r: &mut impl Read, len: usize) -> io::Result<String> {
|
||||
let mut buf = vec![0u8; len];
|
||||
r.read_exact(&mut buf)?;
|
||||
let mut buf = read_exact_bounded(r, len)?;
|
||||
// Strip trailing nulls
|
||||
while buf.last() == Some(&0) {
|
||||
buf.pop();
|
||||
@@ -299,8 +316,22 @@ pub fn read_string_val(r: &mut impl Read, len: usize) -> io::Result<String> {
|
||||
|
||||
/// Read binary data of `len` bytes.
|
||||
pub fn read_binary_val(r: &mut impl Read, len: usize) -> io::Result<Vec<u8>> {
|
||||
let mut buf = vec![0u8; len];
|
||||
r.read_exact(&mut buf)?;
|
||||
read_exact_bounded(r, len)
|
||||
}
|
||||
|
||||
/// Read exactly `len` bytes WITHOUT trusting `len` to size the allocation.
|
||||
///
|
||||
/// `vec![0u8; len]` on an attacker-controlled EBML size would allocate
|
||||
/// gigabytes before the read fails. Instead we cap the reader to `len`
|
||||
/// and grow the buffer as bytes actually arrive: a malformed element that
|
||||
/// claims a huge length but supplies few bytes allocates only what it
|
||||
/// delivers, then errors on the short read.
|
||||
fn read_exact_bounded(r: &mut impl Read, len: usize) -> io::Result<Vec<u8>> {
|
||||
let mut buf = Vec::new();
|
||||
let got = r.take(len as u64).read_to_end(&mut buf)?;
|
||||
if got != len {
|
||||
return Err(io::ErrorKind::UnexpectedEof.into());
|
||||
}
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
@@ -321,6 +352,12 @@ pub const EBML_DOC_TYPE_READ_VERSION: u32 = 0x4285;
|
||||
// Segment
|
||||
pub const SEGMENT: u32 = 0x1853_8067;
|
||||
|
||||
// SeekHead
|
||||
pub const SEEK_HEAD: u32 = 0x114D_9B74;
|
||||
pub const SEEK: u32 = 0x4DBB;
|
||||
pub const SEEK_ID: u32 = 0x53AB;
|
||||
pub const SEEK_POSITION: u32 = 0x53AC;
|
||||
|
||||
// Segment Info
|
||||
pub const INFO: u32 = 0x1549_A966;
|
||||
pub const TIMESTAMP_SCALE: u32 = 0x2A_D7B1;
|
||||
@@ -356,6 +393,12 @@ pub const MATRIX_COEFFICIENTS: u32 = 0x55B1;
|
||||
pub const PRIMARIES: u32 = 0x55BB;
|
||||
pub const RANGE: u32 = 0x55B9;
|
||||
|
||||
// Dolby Vision — BlockAdditionMapping carries the DOVIDecoderConfigurationRecord
|
||||
// (dvcC) so players / mediainfo recognise the track as Dolby Vision.
|
||||
pub const BLOCK_ADDITION_MAPPING: u32 = 0x41E4;
|
||||
pub const BLOCK_ADD_ID_TYPE: u32 = 0x41E7;
|
||||
pub const BLOCK_ADD_ID_EXTRA_DATA: u32 = 0x41ED;
|
||||
|
||||
// Audio
|
||||
pub const AUDIO: u32 = 0xE1;
|
||||
pub const SAMPLING_FREQUENCY: u32 = 0xB5;
|
||||
@@ -366,6 +409,9 @@ pub const BIT_DEPTH: u32 = 0x6264;
|
||||
pub const CLUSTER: u32 = 0x1F43_B675;
|
||||
pub const CLUSTER_TIMESTAMP: u32 = 0xE7;
|
||||
pub const SIMPLE_BLOCK: u32 = 0xA3;
|
||||
pub const BLOCK_GROUP: u32 = 0xA0;
|
||||
pub const BLOCK: u32 = 0xA1;
|
||||
pub const BLOCK_DURATION: u32 = 0x9B;
|
||||
|
||||
// Cues
|
||||
pub const CUES: u32 = 0x1C53_BB6B;
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
//! Fragmented MP4 muxer — **stub** for Phase 3.
|
||||
//!
|
||||
//! Goal: ISO/IEC 14496-12 fragmented MP4 (`ftyp` + `moov` init segment,
|
||||
//! then a sequence of `moof+mdat` media fragments) targeting a
|
||||
//! [`SequentialSink`](crate::io::sink::SequentialSink). DASH-friendly,
|
||||
//! no Cues backpatch.
|
||||
//!
|
||||
//! Status (v0.21.0 Phase 3): **STUB**. We ship the init segment
|
||||
//! (`ftyp` + a minimal HEVC `moov` skeleton with one video track) so
|
||||
//! the muxer's shape and call site are validated, but media fragments
|
||||
//! are NOT yet emitted — calls to [`Fmp4Mux::write_video`] currently
|
||||
//! accumulate frames into an internal buffer and discard them on
|
||||
//! [`Fmp4Mux::finish`].
|
||||
//!
|
||||
//! ## What's TODO (tracked in Phase 4 / v0.22.0 scope)
|
||||
//!
|
||||
//! - `moof` box: `mfhd` (sequence_number) + `traf` (`tfhd` + `tfdt`
|
||||
//! + `trun` with sample sizes, durations, flags, composition offsets).
|
||||
//! - `mdat` box: concatenated sample data.
|
||||
//! - Fragment cadence: one fragment per GOP or every N seconds,
|
||||
//! whichever comes first.
|
||||
//! - HEVC `hvcC` box inside `moov.trak.mdia.minf.stbl.stsd` so the
|
||||
//! init segment is self-describing.
|
||||
//! - Sample-flags computation (sync vs. delta, depends_on, etc.).
|
||||
//! - Edit lists / fragment_duration for accurate seeking.
|
||||
//!
|
||||
//! Reference: ISO/IEC 14496-12 §8 (Movie Fragments).
|
||||
|
||||
use std::io::{self, Write};
|
||||
|
||||
// Box type literals — four-character codes per ISO/IEC 14496-12 §4.2.
|
||||
|
||||
const FTYP: [u8; 4] = *b"ftyp";
|
||||
const MOOV: [u8; 4] = *b"moov";
|
||||
const MVHD: [u8; 4] = *b"mvhd";
|
||||
const TRAK: [u8; 4] = *b"trak";
|
||||
const TKHD: [u8; 4] = *b"tkhd";
|
||||
const MDIA: [u8; 4] = *b"mdia";
|
||||
const MDHD: [u8; 4] = *b"mdhd";
|
||||
const HDLR: [u8; 4] = *b"hdlr";
|
||||
const MINF: [u8; 4] = *b"minf";
|
||||
const VMHD: [u8; 4] = *b"vmhd";
|
||||
const DINF: [u8; 4] = *b"dinf";
|
||||
const DREF: [u8; 4] = *b"dref";
|
||||
const URL_: [u8; 4] = *b"url ";
|
||||
const STBL: [u8; 4] = *b"stbl";
|
||||
const STSD: [u8; 4] = *b"stsd";
|
||||
const STTS: [u8; 4] = *b"stts";
|
||||
const STSC: [u8; 4] = *b"stsc";
|
||||
const STSZ: [u8; 4] = *b"stsz";
|
||||
const STCO: [u8; 4] = *b"stco";
|
||||
const MVEX: [u8; 4] = *b"mvex";
|
||||
const TREX: [u8; 4] = *b"trex";
|
||||
|
||||
/// Default movie timescale — 90 kHz lines up with MPEG-TS PTS and the
|
||||
/// HEVC SPS `vui_time_scale` for film content, simplifying the math
|
||||
/// when fragment emission lands.
|
||||
const MOVIE_TIMESCALE: u32 = 90_000;
|
||||
/// Video track ID. fMP4 init segments conventionally use track_ID=1
|
||||
/// for the primary video track; a single-track DASH representation has
|
||||
/// no reason to deviate.
|
||||
const VIDEO_TRACK_ID: u32 = 1;
|
||||
|
||||
/// Fragmented MP4 muxer — stub.
|
||||
///
|
||||
/// See the module-level doc comment for what is and isn't shipped in
|
||||
/// this stub.
|
||||
pub struct Fmp4Mux<W: Write> {
|
||||
writer: W,
|
||||
header_written: bool,
|
||||
/// Pending frames — held for the future fragment-emit path. The
|
||||
/// stub drops these on `finish` but keeping them around lets the
|
||||
/// post-stub work re-attach without changing the public API.
|
||||
pending: Vec<PendingSample>,
|
||||
/// hvcC bytes, if provided. Embedded in the `moov.…stsd.hvc1.hvcC`
|
||||
/// box once that path lands.
|
||||
#[allow(dead_code)]
|
||||
codec_private: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
struct PendingSample {
|
||||
#[allow(dead_code)]
|
||||
pts_ns: i64,
|
||||
#[allow(dead_code)]
|
||||
keyframe: bool,
|
||||
#[allow(dead_code)]
|
||||
data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl<W: Write> Fmp4Mux<W> {
|
||||
pub fn new(writer: W) -> Self {
|
||||
Self {
|
||||
writer,
|
||||
header_written: false,
|
||||
pending: Vec::new(),
|
||||
codec_private: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Provide the `HEVCDecoderConfigurationRecord` for the video track.
|
||||
/// The stub stores it but doesn't yet embed it in `moov` — that's
|
||||
/// part of the post-stub work.
|
||||
pub fn set_video_codec_private(&mut self, hvcc: Vec<u8>) {
|
||||
self.codec_private = Some(hvcc);
|
||||
}
|
||||
|
||||
/// Write one video PES frame.
|
||||
///
|
||||
/// **Stub behaviour:** the first call emits the init segment
|
||||
/// (`ftyp` + `moov`) so any consumer that just wants the shape can
|
||||
/// receive it. Subsequent calls accumulate frames in memory for
|
||||
/// the future fragmenting path; **no media bytes are written yet**.
|
||||
pub fn write_video(&mut self, pts_ns: i64, keyframe: bool, data: &[u8]) -> io::Result<()> {
|
||||
if !self.header_written {
|
||||
self.write_init_segment()?;
|
||||
self.header_written = true;
|
||||
}
|
||||
// TODO(0.22.0): emit one `moof+mdat` per GOP. For now stash the
|
||||
// frame so the future patch can hot-wire emission without API
|
||||
// churn.
|
||||
self.pending.push(PendingSample {
|
||||
pts_ns,
|
||||
keyframe,
|
||||
data: data.to_vec(),
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush. The stub additionally drops accumulated `pending` frames.
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
// TODO(0.22.0): emit final fragment from pending; today the
|
||||
// stub just clears the buffer to release memory.
|
||||
self.pending.clear();
|
||||
self.writer.flush()
|
||||
}
|
||||
|
||||
fn write_init_segment(&mut self) -> io::Result<()> {
|
||||
let ftyp = build_ftyp();
|
||||
let moov = build_moov();
|
||||
self.writer.write_all(&ftyp)?;
|
||||
self.writer.write_all(&moov)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the `ftyp` box. `major_brand = "iso6"`, `minor_version = 1`,
|
||||
/// compatible brands `iso6 dash msdh hvc1` — the same conservative set
|
||||
/// shaka-packager uses for HEVC-in-fMP4 outputs.
|
||||
fn build_ftyp() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(b"iso6");
|
||||
body.extend_from_slice(&1u32.to_be_bytes());
|
||||
body.extend_from_slice(b"iso6");
|
||||
body.extend_from_slice(b"dash");
|
||||
body.extend_from_slice(b"msdh");
|
||||
body.extend_from_slice(b"hvc1");
|
||||
wrap_box(&FTYP, &body)
|
||||
}
|
||||
|
||||
/// Build the `moov` box — minimal skeleton. Single video trak, no
|
||||
/// hvcC inside stsd yet (TODO: full hvc1 sample entry).
|
||||
fn build_moov() -> Vec<u8> {
|
||||
let mvhd = build_mvhd();
|
||||
let trak = build_video_trak();
|
||||
let mvex = build_mvex();
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&mvhd);
|
||||
body.extend_from_slice(&trak);
|
||||
body.extend_from_slice(&mvex);
|
||||
wrap_box(&MOOV, &body)
|
||||
}
|
||||
|
||||
fn build_mvhd() -> Vec<u8> {
|
||||
// Version 0, 100 bytes total body. Fields per ISO/IEC 14496-12 §8.2.2.
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0, 0, 0, 0]); // version + flags
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // creation_time
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // modification_time
|
||||
body.extend_from_slice(&MOVIE_TIMESCALE.to_be_bytes());
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // duration = 0 (fragmented)
|
||||
body.extend_from_slice(&0x0001_0000u32.to_be_bytes()); // rate 1.0
|
||||
body.extend_from_slice(&0x0100u16.to_be_bytes()); // volume 1.0
|
||||
body.extend_from_slice(&[0u8; 2]); // reserved
|
||||
body.extend_from_slice(&[0u8; 8]); // reserved
|
||||
// 3x3 identity transformation matrix in 16.16 fixed point.
|
||||
for v in [0x1_0000u32, 0, 0, 0, 0x1_0000, 0, 0, 0, 0x4000_0000] {
|
||||
body.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
body.extend_from_slice(&[0u8; 24]); // pre_defined[6]
|
||||
body.extend_from_slice(&2u32.to_be_bytes()); // next_track_ID (1 reserved for video)
|
||||
wrap_box(&MVHD, &body)
|
||||
}
|
||||
|
||||
fn build_video_trak() -> Vec<u8> {
|
||||
let tkhd = build_tkhd();
|
||||
let mdia = build_mdia();
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&tkhd);
|
||||
body.extend_from_slice(&mdia);
|
||||
wrap_box(&TRAK, &body)
|
||||
}
|
||||
|
||||
fn build_tkhd() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
// version=0 | flags=0x000007 (track_enabled | in_movie | in_preview)
|
||||
body.extend_from_slice(&[0, 0, 0, 7]);
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // creation_time
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // modification_time
|
||||
body.extend_from_slice(&VIDEO_TRACK_ID.to_be_bytes());
|
||||
body.extend_from_slice(&[0u8; 4]); // reserved
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // duration
|
||||
body.extend_from_slice(&[0u8; 8]); // reserved
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // layer
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // alternate_group
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // volume (video=0)
|
||||
body.extend_from_slice(&[0u8; 2]); // reserved
|
||||
// 3x3 identity matrix.
|
||||
for v in [0x1_0000u32, 0, 0, 0, 0x1_0000, 0, 0, 0, 0x4000_0000] {
|
||||
body.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
// width / height in 16.16 fixed point — placeholder 1920x1080.
|
||||
body.extend_from_slice(&(1920u32 << 16).to_be_bytes());
|
||||
body.extend_from_slice(&(1080u32 << 16).to_be_bytes());
|
||||
wrap_box(&TKHD, &body)
|
||||
}
|
||||
|
||||
fn build_mdia() -> Vec<u8> {
|
||||
let mdhd = build_mdhd();
|
||||
let hdlr = build_hdlr_vide();
|
||||
let minf = build_minf();
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&mdhd);
|
||||
body.extend_from_slice(&hdlr);
|
||||
body.extend_from_slice(&minf);
|
||||
wrap_box(&MDIA, &body)
|
||||
}
|
||||
|
||||
fn build_mdhd() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0, 0, 0, 0]); // version + flags
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // creation_time
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // modification_time
|
||||
body.extend_from_slice(&MOVIE_TIMESCALE.to_be_bytes());
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // duration
|
||||
// language: 'und' in 5-bit-per-char ISO 639-2 packed (bit 15 = 0).
|
||||
body.extend_from_slice(&[0x55, 0xC4]);
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // pre_defined
|
||||
wrap_box(&MDHD, &body)
|
||||
}
|
||||
|
||||
fn build_hdlr_vide() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0, 0, 0, 0]); // version + flags
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // pre_defined
|
||||
body.extend_from_slice(b"vide");
|
||||
body.extend_from_slice(&[0u8; 12]); // reserved
|
||||
body.extend_from_slice(b"VideoHandler\0");
|
||||
wrap_box(&HDLR, &body)
|
||||
}
|
||||
|
||||
fn build_minf() -> Vec<u8> {
|
||||
let vmhd = build_vmhd();
|
||||
let dinf = build_dinf();
|
||||
let stbl = build_stbl();
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&vmhd);
|
||||
body.extend_from_slice(&dinf);
|
||||
body.extend_from_slice(&stbl);
|
||||
wrap_box(&MINF, &body)
|
||||
}
|
||||
|
||||
fn build_vmhd() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0, 0, 0, 1]); // version + flags=1
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // graphicsmode
|
||||
body.extend_from_slice(&[0u8; 6]); // opcolor
|
||||
wrap_box(&VMHD, &body)
|
||||
}
|
||||
|
||||
fn build_dinf() -> Vec<u8> {
|
||||
let mut dref_body = Vec::new();
|
||||
dref_body.extend_from_slice(&[0, 0, 0, 0]);
|
||||
dref_body.extend_from_slice(&1u32.to_be_bytes()); // entry_count
|
||||
// url with flags=1 (self-contained) and zero name.
|
||||
let url_body = [0u8, 0, 0, 1];
|
||||
dref_body.extend_from_slice(&wrap_box(&URL_, &url_body));
|
||||
let dref = wrap_box(&DREF, &dref_body);
|
||||
wrap_box(&DINF, &dref)
|
||||
}
|
||||
|
||||
fn build_stbl() -> Vec<u8> {
|
||||
// Stub stsd: empty sample description (zero entries). Replace with
|
||||
// hvc1+hvcC once the fragmenting path lands so the init segment is
|
||||
// actually decodable.
|
||||
let mut stsd_body = Vec::new();
|
||||
stsd_body.extend_from_slice(&[0, 0, 0, 0]);
|
||||
stsd_body.extend_from_slice(&0u32.to_be_bytes()); // entry_count
|
||||
let stsd = wrap_box(&STSD, &stsd_body);
|
||||
|
||||
// Empty stts/stsc/stsz/stco — fragmented init has no samples here.
|
||||
let stts = wrap_box(&STTS, &[0, 0, 0, 0, 0, 0, 0, 0]); // version+flags, count=0
|
||||
let stsc = wrap_box(&STSC, &[0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
let stsz = wrap_box(
|
||||
&STSZ,
|
||||
&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // version+flags, sample_size=0, count=0
|
||||
);
|
||||
let stco = wrap_box(&STCO, &[0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&stsd);
|
||||
body.extend_from_slice(&stts);
|
||||
body.extend_from_slice(&stsc);
|
||||
body.extend_from_slice(&stsz);
|
||||
body.extend_from_slice(&stco);
|
||||
wrap_box(&STBL, &body)
|
||||
}
|
||||
|
||||
fn build_mvex() -> Vec<u8> {
|
||||
// trex: track_ID=1, default_sample_description_index=1, others=0.
|
||||
let mut trex_body = Vec::new();
|
||||
trex_body.extend_from_slice(&[0, 0, 0, 0]); // version + flags
|
||||
trex_body.extend_from_slice(&VIDEO_TRACK_ID.to_be_bytes());
|
||||
trex_body.extend_from_slice(&1u32.to_be_bytes()); // default_sample_description_index
|
||||
trex_body.extend_from_slice(&0u32.to_be_bytes()); // default_sample_duration
|
||||
trex_body.extend_from_slice(&0u32.to_be_bytes()); // default_sample_size
|
||||
trex_body.extend_from_slice(&0u32.to_be_bytes()); // default_sample_flags
|
||||
let trex = wrap_box(&TREX, &trex_body);
|
||||
wrap_box(&MVEX, &trex)
|
||||
}
|
||||
|
||||
/// Wrap a box body in `[size:u32-BE][type:4]`. Suitable for any body
|
||||
/// that fits in u32; oversized boxes (size > 4 GiB) need the 64-bit
|
||||
/// large-size extension which we don't generate in the stub.
|
||||
fn wrap_box(box_type: &[u8; 4], body: &[u8]) -> Vec<u8> {
|
||||
let size = (body.len() + 8) as u32;
|
||||
let mut out = Vec::with_capacity(body.len() + 8);
|
||||
out.extend_from_slice(&size.to_be_bytes());
|
||||
out.extend_from_slice(box_type);
|
||||
out.extend_from_slice(body);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Decode the first box's size + type from `buf`.
|
||||
fn read_box_header(buf: &[u8]) -> (u32, [u8; 4]) {
|
||||
let size = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]);
|
||||
let bt = [buf[4], buf[5], buf[6], buf[7]];
|
||||
(size, bt)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn init_segment_starts_with_ftyp_then_moov() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = Fmp4Mux::new(&mut sink);
|
||||
// Trigger init emission via a single (stubbed) write.
|
||||
mux.write_video(0, true, &[0x00, 0x00, 0x00, 0x01, 0x40])
|
||||
.unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
let (ftyp_size, ftyp_type) = read_box_header(&sink);
|
||||
assert_eq!(&ftyp_type, b"ftyp");
|
||||
assert!(ftyp_size >= 24, "ftyp too small: {ftyp_size}");
|
||||
|
||||
let (moov_size, moov_type) = read_box_header(&sink[ftyp_size as usize..]);
|
||||
assert_eq!(&moov_type, b"moov");
|
||||
assert!(moov_size > 100, "moov skeleton too small: {moov_size}");
|
||||
|
||||
// Stub guarantee: no media bytes after the init segment.
|
||||
let total = ftyp_size as usize + moov_size as usize;
|
||||
assert_eq!(sink.len(), total, "stub leaked media bytes past moov");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn moov_contains_trak_mvex() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = Fmp4Mux::new(&mut sink);
|
||||
mux.write_video(0, true, &[]).unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(sink);
|
||||
|
||||
// Re-emit into a fresh buffer for parsing.
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
let mut mux2 = Fmp4Mux::new(&mut buf);
|
||||
mux2.write_video(0, true, &[]).unwrap();
|
||||
mux2.finish().unwrap();
|
||||
drop(mux2);
|
||||
|
||||
// Find moov payload start.
|
||||
let (ftyp_size, _) = read_box_header(&buf);
|
||||
let moov_start = ftyp_size as usize;
|
||||
let (moov_size, _) = read_box_header(&buf[moov_start..]);
|
||||
let moov_payload = &buf[moov_start + 8..moov_start + moov_size as usize];
|
||||
|
||||
// Scan for the trak and mvex four-CC anywhere in the moov payload.
|
||||
let has_trak = moov_payload.windows(4).any(|w| w == b"trak");
|
||||
let has_mvex = moov_payload.windows(4).any(|w| w == b"mvex");
|
||||
assert!(has_trak, "moov missing trak");
|
||||
assert!(has_mvex, "moov missing mvex");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
//! HEVC (H.265) elementary stream muxer — Annex B byte stream.
|
||||
//!
|
||||
//! Consumes [`PesFrame`](crate::pes::PesFrame)s for a single video track
|
||||
//! and writes them as a raw `.hevc` / `.h265` Annex B byte stream:
|
||||
//! `00 00 00 01 | NAL_unit | 00 00 00 01 | NAL_unit | …` with no
|
||||
//! container framing.
|
||||
//!
|
||||
//! On the first frame the muxer emits the codec_private's VPS, SPS, PPS
|
||||
//! (parsed from a `HEVCDecoderConfigurationRecord` in
|
||||
//! `length-prefixed-in-hvcC` form), then converts each PES frame's
|
||||
//! length-prefixed NAL units to Annex B and writes them.
|
||||
//!
|
||||
//! Sequential-only — no Cues, no backpatch. Target sink is any
|
||||
//! [`SequentialSink`](crate::io::sink::SequentialSink): file, socket,
|
||||
//! pipe, anything `Write + Send`.
|
||||
|
||||
use std::io::{self, Write};
|
||||
|
||||
/// Annex B 4-byte start code.
|
||||
pub(crate) const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01];
|
||||
|
||||
/// HEVC NAL unit type bits live in `(byte0 >> 1) & 0x3F` in Annex B.
|
||||
/// We don't filter NAL types here — the muxer is format-only — but we
|
||||
/// keep the constant as documentation of the field layout.
|
||||
#[allow(dead_code)]
|
||||
const HEVC_NAL_TYPE_MASK: u8 = 0x3F;
|
||||
|
||||
/// Streaming HEVC Annex B muxer.
|
||||
///
|
||||
/// One instance per output stream. Tracks whether parameter sets have
|
||||
/// already been emitted so they're written exactly once at the head of
|
||||
/// the stream, mirroring the convention used by `ffmpeg -c:v copy -f
|
||||
/// hevc`.
|
||||
pub struct HevcMux<W: Write> {
|
||||
writer: W,
|
||||
/// `HEVCDecoderConfigurationRecord` payload (hvcC). Parsed lazily
|
||||
/// on the first `write_frame` so callers can set it after
|
||||
/// construction but before the first frame.
|
||||
codec_private: Option<Vec<u8>>,
|
||||
/// Set once VPS/SPS/PPS have been written to the stream. Subsequent
|
||||
/// frames write only their own NAL units.
|
||||
params_written: bool,
|
||||
}
|
||||
|
||||
impl<W: Write> HevcMux<W> {
|
||||
/// Construct over `writer`. The muxer does not impose any extra
|
||||
/// buffering of its own — the sink owns its write buffering policy
|
||||
/// (see [`LocalFileSink`](crate::io::sink::LocalFileSink) and
|
||||
/// [`SocketSink`](crate::io::sink::SocketSink)).
|
||||
pub fn new(writer: W) -> Self {
|
||||
Self {
|
||||
writer,
|
||||
codec_private: None,
|
||||
params_written: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Provide the `HEVCDecoderConfigurationRecord` (hvcC) so the muxer
|
||||
/// can prepend VPS/SPS/PPS Annex B NALs at stream start. Optional —
|
||||
/// if the PES frames already carry inline parameter sets (some
|
||||
/// upstream demuxers do this), skipping this call is fine.
|
||||
pub fn set_codec_private(&mut self, data: Vec<u8>) {
|
||||
self.codec_private = Some(data);
|
||||
}
|
||||
|
||||
/// Write one PES frame (= one access unit) as Annex B NAL units.
|
||||
///
|
||||
/// Input may be either:
|
||||
/// - Length-prefixed: `[u32-BE len][NAL bytes]` repeated. This is
|
||||
/// the form emitted by libfreemkv's HEVC parser (the MKV-native
|
||||
/// layout). Converted to Annex B.
|
||||
/// - Already Annex B: bytes containing `00 00 00 01` start codes
|
||||
/// anywhere in the buffer. Passed through unchanged.
|
||||
///
|
||||
/// `_pts_ns` is accepted for symmetry with other muxers but ignored
|
||||
/// — Annex B has no timing layer.
|
||||
pub fn write_frame(&mut self, _pts_ns: i64, data: &[u8]) -> io::Result<()> {
|
||||
if !self.params_written {
|
||||
if let Some(cp) = &self.codec_private {
|
||||
if let Some(params) = hvcc_to_annex_b(cp) {
|
||||
self.writer.write_all(¶ms)?;
|
||||
}
|
||||
}
|
||||
self.params_written = true;
|
||||
}
|
||||
let annex_b = length_prefixed_to_annex_b(data);
|
||||
self.writer.write_all(&annex_b)
|
||||
}
|
||||
|
||||
/// Flush the underlying writer. No trailer NAL is needed — an Annex
|
||||
/// B stream ends whenever the file/socket ends.
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
self.writer.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a `HEVCDecoderConfigurationRecord` (hvcC) into Annex B NAL
|
||||
/// units. Returns `Some(bytes)` if at least one NAL was extracted, else
|
||||
/// `None`.
|
||||
///
|
||||
/// Layout (per ISO/IEC 14496-15 §8.3.3.1.2):
|
||||
/// - 22-byte fixed header
|
||||
/// - byte 22 = `numOfArrays`
|
||||
/// - each array: `array_completeness:1 | reserved:1 | NAL_unit_type:6`,
|
||||
/// `numNalus:u16-BE`, then `numNalus` × `(nalUnitLength:u16-BE +
|
||||
/// NAL bytes)`.
|
||||
///
|
||||
/// We don't filter on NAL type — VPS (32), SPS (33), PPS (34), and any
|
||||
/// SEI arrays included in hvcC all get the same Annex B treatment.
|
||||
fn hvcc_to_annex_b(hvcc: &[u8]) -> Option<Vec<u8>> {
|
||||
if hvcc.len() < 23 {
|
||||
return None;
|
||||
}
|
||||
let num_arrays = hvcc[22] as usize;
|
||||
let mut out = Vec::new();
|
||||
let mut offset = 23;
|
||||
for _ in 0..num_arrays {
|
||||
if offset + 3 > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
offset += 1; // array_completeness + nal_type byte
|
||||
let num_nalus = u16::from_be_bytes([hvcc[offset], hvcc[offset + 1]]) as usize;
|
||||
offset += 2;
|
||||
for _ in 0..num_nalus {
|
||||
if offset + 2 > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
let nal_len = u16::from_be_bytes([hvcc[offset], hvcc[offset + 1]]) as usize;
|
||||
offset += 2;
|
||||
if offset + nal_len > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
out.extend_from_slice(&START_CODE);
|
||||
out.extend_from_slice(&hvcc[offset..offset + nal_len]);
|
||||
offset += nal_len;
|
||||
}
|
||||
}
|
||||
if out.is_empty() { None } else { Some(out) }
|
||||
}
|
||||
|
||||
/// Convert length-prefixed NAL units (`[u32-BE len][NAL]` repeated) to
|
||||
/// Annex B (`00 00 00 01 [NAL]` repeated).
|
||||
///
|
||||
/// If the input doesn't parse as length-prefixed (no valid lengths
|
||||
/// extracted), it's returned unchanged on the assumption that it's
|
||||
/// already Annex B — some upstream paths (raw HEVC ES from disc) pass
|
||||
/// Annex B straight through the PES layer.
|
||||
pub(crate) fn length_prefixed_to_annex_b(data: &[u8]) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(data.len() + (data.len() / 32));
|
||||
let mut offset = 0;
|
||||
while offset + 4 <= data.len() {
|
||||
let len = u32::from_be_bytes([
|
||||
data[offset],
|
||||
data[offset + 1],
|
||||
data[offset + 2],
|
||||
data[offset + 3],
|
||||
]) as usize;
|
||||
offset += 4;
|
||||
if offset + len > data.len() {
|
||||
// Mid-NAL truncation — fall through to the pass-through path
|
||||
// rather than emitting a half-NAL.
|
||||
return data.to_vec();
|
||||
}
|
||||
out.extend_from_slice(&START_CODE);
|
||||
out.extend_from_slice(&data[offset..offset + len]);
|
||||
offset += len;
|
||||
}
|
||||
if out.is_empty() && !data.is_empty() {
|
||||
// No length prefixes found — input is likely already Annex B.
|
||||
return data.to_vec();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn length_prefixed_converts_to_annex_b() {
|
||||
// Two NALs: [3-byte payload AA BB CC] and [2-byte payload DD EE].
|
||||
let mut buf = Vec::new();
|
||||
buf.extend_from_slice(&3u32.to_be_bytes());
|
||||
buf.extend_from_slice(&[0xAA, 0xBB, 0xCC]);
|
||||
buf.extend_from_slice(&2u32.to_be_bytes());
|
||||
buf.extend_from_slice(&[0xDD, 0xEE]);
|
||||
|
||||
let got = length_prefixed_to_annex_b(&buf);
|
||||
let want = [
|
||||
0x00, 0x00, 0x00, 0x01, 0xAA, 0xBB, 0xCC, // first NAL
|
||||
0x00, 0x00, 0x00, 0x01, 0xDD, 0xEE, // second NAL
|
||||
];
|
||||
assert_eq!(&got[..], &want[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn already_annex_b_passes_through_when_no_lengths_match() {
|
||||
// A buffer < 4 bytes can't parse a length prefix at all →
|
||||
// pass-through path triggers.
|
||||
let raw = [0xAA, 0xBB, 0xCC];
|
||||
let got = length_prefixed_to_annex_b(&raw);
|
||||
assert_eq!(&got[..], &raw[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mid_nal_truncation_returns_original() {
|
||||
// `[u32-BE 100][only 3 bytes]` — length prefix claims 100 bytes
|
||||
// but the input only has 3 after the prefix. We treat that as
|
||||
// malformed and pass the original buffer through so receivers
|
||||
// can attempt their own recovery.
|
||||
let mut raw = Vec::new();
|
||||
raw.extend_from_slice(&100u32.to_be_bytes());
|
||||
raw.extend_from_slice(&[0xAA, 0xBB, 0xCC]);
|
||||
let got = length_prefixed_to_annex_b(&raw);
|
||||
assert_eq!(&got[..], &raw[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hvcc_extracts_vps_sps_pps() {
|
||||
// Build a minimal-but-valid hvcC: 22-byte header, then 3 arrays
|
||||
// (VPS / SPS / PPS), each with 1 NAL of a 4-byte payload that
|
||||
// we can spot in the output.
|
||||
let mut hvcc = vec![0u8; 22];
|
||||
hvcc.push(3); // numOfArrays
|
||||
for (nal_type, payload) in [
|
||||
(32u8, [0x40, 0x01, 0x0C, 0x01]),
|
||||
(33, [0x42, 0x01, 0x01, 0x01]),
|
||||
(34, [0x44, 0x01, 0xC1, 0x72]),
|
||||
] {
|
||||
hvcc.push(nal_type & 0x3F);
|
||||
hvcc.extend_from_slice(&1u16.to_be_bytes()); // numNalus
|
||||
hvcc.extend_from_slice(&(payload.len() as u16).to_be_bytes());
|
||||
hvcc.extend_from_slice(&payload);
|
||||
}
|
||||
|
||||
let annex_b = hvcc_to_annex_b(&hvcc).expect("at least one NAL");
|
||||
// Three NALs × (4-byte start + 4-byte payload) = 24 bytes.
|
||||
assert_eq!(annex_b.len(), 24);
|
||||
assert_eq!(&annex_b[..4], &START_CODE);
|
||||
assert_eq!(&annex_b[8..12], &START_CODE);
|
||||
assert_eq!(&annex_b[16..20], &START_CODE);
|
||||
assert_eq!(annex_b[4], 0x40); // VPS first byte
|
||||
assert_eq!(annex_b[12], 0x42); // SPS first byte
|
||||
assert_eq!(annex_b[20], 0x44); // PPS first byte
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mux_writes_params_then_frames() {
|
||||
// Build hvcC with one SPS to verify params-once semantics.
|
||||
let mut hvcc = vec![0u8; 22];
|
||||
hvcc.push(1);
|
||||
hvcc.push(33);
|
||||
hvcc.extend_from_slice(&1u16.to_be_bytes());
|
||||
hvcc.extend_from_slice(&3u16.to_be_bytes());
|
||||
hvcc.extend_from_slice(&[0x42, 0x01, 0x01]);
|
||||
|
||||
let mut frame_data = Vec::new();
|
||||
frame_data.extend_from_slice(&2u32.to_be_bytes());
|
||||
frame_data.extend_from_slice(&[0xAA, 0xBB]);
|
||||
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = HevcMux::new(&mut sink);
|
||||
mux.set_codec_private(hvcc);
|
||||
mux.write_frame(0, &frame_data).unwrap();
|
||||
// Second frame — no SPS re-emission.
|
||||
mux.write_frame(40_000_000, &frame_data).unwrap();
|
||||
mux.finish().unwrap();
|
||||
|
||||
// SPS NAL (7 bytes) + 2× frame NAL (6 bytes) = 19 bytes.
|
||||
assert_eq!(sink.len(), 7 + 6 + 6);
|
||||
// Start codes at offsets 0 (SPS), 7 (frame1), 13 (frame2).
|
||||
assert_eq!(&sink[0..4], &START_CODE);
|
||||
assert_eq!(&sink[7..11], &START_CODE);
|
||||
assert_eq!(&sink[13..17], &START_CODE);
|
||||
assert_eq!(sink[4], 0x42);
|
||||
assert_eq!(sink[11], 0xAA);
|
||||
assert_eq!(sink[17], 0xAA);
|
||||
}
|
||||
}
|
||||
-101
@@ -1,101 +0,0 @@
|
||||
//! ISO sector reader — file-backed SectorReader for Blu-ray ISO images.
|
||||
//!
|
||||
//! An ISO is a flat image of 2048-byte sectors. Sector N starts at byte offset N * 2048.
|
||||
//! Used by DiscStream::open_iso() and Disc::scan_image().
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorReader;
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::Path;
|
||||
|
||||
const SECTOR_SIZE: u64 = 2048;
|
||||
|
||||
/// File-backed sector reader for ISO images.
|
||||
pub struct IsoSectorReader {
|
||||
file: File,
|
||||
capacity: u32,
|
||||
}
|
||||
|
||||
impl IsoSectorReader {
|
||||
pub fn open(path: &str) -> std::io::Result<Self> {
|
||||
let file = File::open(Path::new(path))?;
|
||||
let size = file.metadata()?.len();
|
||||
let sectors = size / SECTOR_SIZE;
|
||||
if sectors > u32::MAX as u64 {
|
||||
return Err(crate::error::Error::IsoTooLarge {
|
||||
path: path.to_string(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let capacity = sectors as u32;
|
||||
Ok(Self { file, capacity })
|
||||
}
|
||||
|
||||
pub fn capacity(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for IsoSectorReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
let bytes = count as usize * SECTOR_SIZE as usize;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(lba as u64 * SECTOR_SIZE))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.read_exact(&mut buf[..bytes])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn iso_reader_read_sectors() {
|
||||
let mut data = vec![0u8; 4 * SECTOR_SIZE as usize];
|
||||
for i in 0..4u8 {
|
||||
let offset = i as usize * SECTOR_SIZE as usize;
|
||||
data[offset] = i + 1;
|
||||
data[offset + 2047] = i + 100;
|
||||
}
|
||||
|
||||
let dir = std::env::temp_dir().join("freemkv_test_iso_read");
|
||||
std::fs::write(&dir, &data).unwrap();
|
||||
|
||||
let mut reader = IsoSectorReader::open(dir.to_str().unwrap()).unwrap();
|
||||
assert_eq!(reader.capacity(), 4);
|
||||
|
||||
let mut buf = [0u8; 2048];
|
||||
reader.read_sectors(0, 1, &mut buf, true).unwrap();
|
||||
assert_eq!(buf[0], 1);
|
||||
assert_eq!(buf[2047], 100);
|
||||
|
||||
reader.read_sectors(2, 1, &mut buf, true).unwrap();
|
||||
assert_eq!(buf[0], 3);
|
||||
assert_eq!(buf[2047], 102);
|
||||
|
||||
std::fs::remove_file(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn iso_reader_capacity() {
|
||||
let data = vec![0u8; 10 * SECTOR_SIZE as usize];
|
||||
let dir = std::env::temp_dir().join("freemkv_test_iso_cap");
|
||||
std::fs::write(&dir, &data).unwrap();
|
||||
|
||||
let reader = IsoSectorReader::open(dir.to_str().unwrap()).unwrap();
|
||||
assert_eq!(reader.capacity(), 10);
|
||||
|
||||
std::fs::remove_file(&dir).ok();
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user