mirror of
https://github.com/freemkv/libfreemkv.git
synced 2026-10-07 18:13:10 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ee66db8b95 | ||
|
|
60df09f965 | ||
|
|
46dc8720be | ||
|
|
353c8ce1fb | ||
|
|
14070a2dec | ||
|
|
1c8672dcaf | ||
|
|
b9c0a7c208 | ||
|
|
3c6351975b | ||
|
|
fda771a06a | ||
|
|
2f1697d587 | ||
|
|
fc95d55d57 | ||
|
|
571c613c39 | ||
|
|
a12f3b6c78 | ||
|
|
a6073d6866 | ||
|
|
b4b05df97a | ||
|
|
52b9731453 | ||
|
|
fd8abc3684 | ||
|
|
2307f0d79a | ||
|
|
3d2f115691 | ||
|
|
9db9b9bdf6 | ||
|
|
3f38ee6fd5 | ||
|
|
350df85439 | ||
|
|
62a013d906 | ||
|
|
7db0a154e9 | ||
|
|
2a11844cad | ||
|
|
051e5b5f25 | ||
|
|
3e2740d815 | ||
|
|
ef5a1b9ff7 | ||
|
|
6c55f0cf20 | ||
|
|
95c3c0486b | ||
|
|
73b1bb0509 | ||
|
|
86de7fe931 | ||
|
|
0993d60e8c | ||
|
|
b080fb6e1b | ||
|
|
d87defef9e | ||
|
|
8d8f2fa463 | ||
|
|
db64d6cea5 | ||
|
|
7f1827565a | ||
|
|
787b4ff54c | ||
|
|
75be0159e6 | ||
|
|
8c9672819e | ||
|
|
c9f9b1dd08 | ||
|
|
d94e54e52b | ||
|
|
7ace1aa3c6 | ||
|
|
20c1035730 | ||
|
|
8db1d01833 | ||
|
|
3bdf490fce | ||
|
|
cec217741b | ||
|
|
e4f27a0daf | ||
|
|
5b1009df71 | ||
|
|
1fb9463eaf | ||
|
|
23414d289d | ||
|
|
828aed1ecb | ||
|
|
10709fafbf | ||
|
|
c4fda5c873 | ||
|
|
ab18f384c7 | ||
|
|
18d1f16582 | ||
|
|
1aea0d81a5 | ||
|
|
c197b5236f | ||
|
|
d868d4e720 | ||
|
|
241a592323 | ||
|
|
a4a12859db | ||
|
|
7d3f413030 | ||
|
|
b993320249 | ||
|
|
cb96550ba5 | ||
|
|
9d2007274c | ||
|
|
e58ce237e7 | ||
|
|
fc82d98bf1 | ||
|
|
d40733d0b1 | ||
|
|
ec35b2bce1 | ||
|
|
4b6d3dda64 | ||
|
|
de3e62e6cc | ||
|
|
3cbbe148d1 | ||
|
|
d2cf068a60 | ||
|
|
dc18dd9e8e | ||
|
|
9e552916de | ||
|
|
cfc41086ba | ||
|
|
c2f6d04198 | ||
|
|
a1aca559cf | ||
|
|
e49f21a45a | ||
|
|
06f6d8932f | ||
|
|
15802da1db | ||
|
|
ea5ee9dc6b | ||
|
|
b7a8937786 | ||
|
|
36012e026c | ||
|
|
872d78d89d | ||
|
|
cfd71eed3b | ||
|
|
b531763cae | ||
|
|
2d9a82743a | ||
|
|
23f5ae3665 | ||
|
|
ad06f3062e | ||
|
|
2fcb7d3b77 | ||
|
|
9e2f5f0109 | ||
|
|
b5ac505c9d | ||
|
|
a7280eef85 | ||
|
|
ab4c66e9d4 | ||
|
|
2fbdd28023 | ||
|
|
aef72b920f | ||
|
|
1a64ba2790 | ||
|
|
81d3adbbb3 | ||
|
|
3ce5929fa5 | ||
|
|
77fb07f954 | ||
|
|
54b79b0395 | ||
|
|
7710fccddd | ||
|
|
d2ff433fac | ||
|
|
30b2b86cad | ||
|
|
a192e696ac | ||
|
|
dc2dcdba55 | ||
|
|
fc6d9d42fd | ||
|
|
03f6e076ab | ||
|
|
e5e8ee43ec | ||
|
|
dd9be0d21d | ||
|
|
835429c35d | ||
|
|
804c4cb194 | ||
|
|
fa3d6dabdd | ||
|
|
bd3664d3c5 | ||
|
|
a5861157de | ||
|
|
831ca31911 | ||
|
|
d127783522 | ||
|
|
80bf9eba60 | ||
|
|
be8d555551 | ||
|
|
2cab62dc22 | ||
|
|
b12c1e622d | ||
|
|
4d81affb45 | ||
|
|
c73a3dbcb6 | ||
|
|
4f606ae9a3 | ||
|
|
25acd09504 | ||
|
|
159e967760 | ||
|
|
6dc62bcd84 | ||
|
|
9250f5bb30 | ||
|
|
e960c2f1be | ||
|
|
f74979bdb4 | ||
|
|
dc2cac1b5f | ||
|
|
337e77951c | ||
|
|
5941c059c6 | ||
|
|
e8bb6225ac | ||
|
|
9c80ef8245 | ||
|
|
dc87962e50 | ||
|
|
4221cd6a86 | ||
|
|
dda4e7482b | ||
|
|
f80551f278 | ||
|
|
41a6d89cd1 | ||
|
|
f79c2a0aa9 | ||
|
|
d181362460 | ||
|
|
8000bae177 | ||
|
|
2a55bab3ed | ||
|
|
222a596c55 | ||
|
|
c1b4f3cbb3 | ||
|
|
06c30aa466 | ||
|
|
061f68594a | ||
|
|
5b6ea8f5c4 | ||
|
|
eeba94b21d | ||
|
|
cfc12774f2 | ||
|
|
fd543c058b | ||
|
|
cd8ce708ac | ||
|
|
2eee777b8f | ||
|
|
a5962be86c | ||
|
|
7306f661b9 | ||
|
|
77f7aced83 | ||
|
|
97ae452e40 | ||
|
|
eec0594a30 | ||
|
|
dab6ea4359 | ||
|
|
9a5ed57044 | ||
|
|
7d58ba7b08 | ||
|
|
4f1dbfd042 | ||
|
|
5b702a76a7 | ||
|
|
6be5198886 | ||
|
|
e2aa9abd6d | ||
|
|
3b7bee9ed4 | ||
|
|
b7405e2d27 | ||
|
|
e04d79c593 | ||
|
|
b34af1fa74 | ||
|
|
a7317f8885 | ||
|
|
474273afc0 | ||
|
|
0a2bab5789 | ||
|
|
fa4d7ef871 | ||
|
|
36d1af1b7f | ||
|
|
b96f6206fe | ||
|
|
bb32fb993b | ||
|
|
c1eb74dfa5 | ||
|
|
46838c63ca | ||
|
|
1e60220ff6 | ||
|
|
080f03e8ed | ||
|
|
35de6101d4 | ||
|
|
9012101573 | ||
|
|
8bc1de6c9b | ||
|
|
d94a4d3444 | ||
|
|
1bd7e1de2a | ||
|
|
da62ee7cf2 | ||
|
|
575c76156f | ||
|
|
b518860d9c | ||
|
|
c8eb42b490 | ||
|
|
48570ac065 | ||
|
|
08e46640fd | ||
|
|
a9195824ff | ||
|
|
401fe23988 | ||
|
|
f89bce5851 | ||
|
|
dfccb85e15 | ||
|
|
ebedffb762 | ||
|
|
fdf63ccb7f | ||
|
|
c735d284da | ||
|
|
94ab7bc73c | ||
|
|
9f209fe066 | ||
|
|
32a1a6e095 | ||
|
|
1d3b8f5fb6 | ||
|
|
97b0ae7be2 | ||
|
|
7756f1feca | ||
|
|
1565da610a | ||
|
|
e134616422 | ||
|
|
8d54a3c64e | ||
|
|
c0478e1273 | ||
|
|
7d29168fec | ||
|
|
8e14c9b850 | ||
|
|
e1c8343f77 | ||
|
|
a956c6ad94 | ||
|
|
1805d92ca4 | ||
|
|
823f0ad430 | ||
|
|
477bdf1835 | ||
|
|
4d83b69c20 | ||
|
|
7dbbfc6726 | ||
|
|
e635c9556f | ||
|
|
d7b5c30f5d | ||
|
|
739a276a39 | ||
|
|
5f1028a62a | ||
|
|
ea15d212de | ||
|
|
00673c8ec3 | ||
|
|
bfefb4cb5b | ||
|
|
1b95193517 | ||
|
|
7dcac44136 | ||
|
|
4da559e39f | ||
|
|
eeca250f69 | ||
|
|
c51b3181f2 | ||
|
|
2a31a47434 | ||
|
|
b08662f95c | ||
|
|
4594196b6e | ||
|
|
9d6974df3e | ||
|
|
3394a5b3fe | ||
|
|
e2c7e20329 | ||
|
|
ff22385a61 | ||
|
|
d39d3933f5 | ||
|
|
31dc3d35c3 | ||
|
|
f7165a2020 | ||
|
|
d836f8b1e7 | ||
|
|
8919de5b03 | ||
|
|
0dc1f12108 | ||
|
|
6abc25a7e8 | ||
|
|
4d22cf632f | ||
|
|
611ad77586 | ||
|
|
60590193e3 | ||
|
|
5f24843d3c | ||
|
|
ecf9a4d01e | ||
|
|
8ec767339e | ||
|
|
335e8b68cc | ||
|
|
1137b4ea1f | ||
|
|
7788c560d1 | ||
|
|
84e0c2ca69 | ||
|
|
f4a881b25d | ||
|
|
1edda7fedb | ||
|
|
3dd71a0650 | ||
|
|
579c94bd23 | ||
|
|
3b2a68fe7a | ||
|
|
5b56f112a0 | ||
|
|
f27e4c3088 | ||
|
|
b3576f2ae6 | ||
|
|
39f2d0e992 | ||
|
|
50867516b8 | ||
|
|
193e7c1680 | ||
|
|
b6463729e8 | ||
|
|
dffee56102 | ||
|
|
d8f27cdee0 | ||
|
|
fc13268dc6 | ||
|
|
d9b55f02d0 | ||
|
|
35d66d2059 | ||
|
|
d703ce439b | ||
|
|
6112a26d15 | ||
|
|
2a33364253 | ||
|
|
fa3872ddcc | ||
|
|
c427389f36 | ||
|
|
6980f562df | ||
|
|
e3b2c9d850 | ||
|
|
a383200ef1 | ||
|
|
51d15f551e | ||
|
|
523af461a5 | ||
|
|
695b65149e | ||
|
|
d3f9560689 | ||
|
|
1fa5a7d27f | ||
|
|
55219070f1 | ||
|
|
a34c419521 | ||
|
|
b179846f5d | ||
|
|
e110e80e6e | ||
|
|
597fa34099 | ||
|
|
52eeb949e3 | ||
|
|
6ca62b8411 | ||
|
|
04195c27d8 | ||
|
|
5a8f8e54e1 | ||
|
|
1a2c830cf6 | ||
|
|
637c2bfe36 | ||
|
|
0d28357b4d | ||
|
|
5495332f07 | ||
|
|
e22fc6fd47 | ||
|
|
5b98c13e47 | ||
|
|
8f8f1a62a2 | ||
|
|
e3cfd27942 | ||
|
|
aa12bdad62 | ||
|
|
ef3895cdc5 | ||
|
|
2dcf969ac8 | ||
|
|
005f887bf9 | ||
|
|
d7243a6044 | ||
|
|
f1926c38dc | ||
|
|
4709a73c80 | ||
|
|
55bd1ee868 | ||
|
|
5285e8b6ec | ||
|
|
c5af61e46b | ||
|
|
ed73c1ab68 | ||
|
|
5ce2df0888 | ||
|
|
ea591a36a5 | ||
|
|
368e10486e | ||
|
|
a99c4f8487 | ||
|
|
4df470e572 | ||
|
|
a0b13941fe | ||
|
|
275c9da009 | ||
|
|
da183722fa | ||
|
|
4a5424eabe | ||
|
|
293d89c586 | ||
|
|
4840f5134d | ||
|
|
11c8211605 | ||
|
|
1038beaf06 | ||
|
|
f1df57196c | ||
|
|
8ba9dc1c0b | ||
|
|
49b131f39a | ||
|
|
8238ec4ce7 | ||
|
|
4fd5d27df8 | ||
|
|
1fbe272832 | ||
|
|
6bd635725e | ||
|
|
09b77b4dea | ||
|
|
688058b3e8 | ||
|
|
c96bac7977 | ||
|
|
c32acff3e4 | ||
|
|
b58e2d9873 | ||
|
|
a9e802c1c2 | ||
|
|
a876ce846b | ||
|
|
058fd8396f | ||
|
|
5ee28c08b9 | ||
|
|
764230b1eb | ||
|
|
7fb2e07aed | ||
|
|
376aadb335 | ||
|
|
586495b2ee | ||
|
|
72f2224efe | ||
|
|
0bca7a11bd | ||
|
|
72ab714c1e | ||
|
|
d7fb1b35ed | ||
|
|
06be4defd2 | ||
|
|
a832bad697 | ||
|
|
26e2d847d5 | ||
|
|
4442fa2df6 | ||
|
|
23dc55661c | ||
|
|
fbdb50c79f | ||
|
|
7cc74f0087 | ||
|
|
3aa1e528c5 | ||
|
|
7c6b0f82ab | ||
|
|
4226a53e73 | ||
|
|
92f34a289e | ||
|
|
fdbe469d50 | ||
|
|
4ec75a03f2 | ||
|
|
99236ffd55 | ||
|
|
9e3327a569 | ||
|
|
0081955686 | ||
|
|
3244fdd683 | ||
|
|
ef5487e5a5 | ||
|
|
e7dffa63d2 | ||
|
|
e5989c8270 | ||
|
|
e1c938b82a | ||
|
|
ed801a708b | ||
|
|
d94451954c | ||
|
|
385c9f094c | ||
|
|
9f2a13739d | ||
|
|
d9ce69bc9d | ||
|
|
7cd2c937ed | ||
|
|
107524da5e | ||
|
|
055a3c5276 | ||
|
|
50b04790ce | ||
|
|
a00b2c884b | ||
|
|
44e2c64d7e | ||
|
|
d6535b8f57 | ||
|
|
90ab00ed45 | ||
|
|
6bfd7dfd13 | ||
|
|
b9a7f601d4 | ||
|
|
cf0a61f8b4 | ||
|
|
f98f07b2d3 | ||
|
|
b53454fa09 | ||
|
|
f28b6ee6d9 | ||
|
|
d9d778fa64 | ||
|
|
bbfb887a35 | ||
|
|
9884346c24 | ||
|
|
766b7c6636 | ||
|
|
925c30686b | ||
|
|
f52e4c5d22 | ||
|
|
8d67790d2e | ||
|
|
246a439990 | ||
|
|
bd19041648 | ||
|
|
198268b725 | ||
|
|
31424c8203 | ||
|
|
c747ecc589 | ||
|
|
283a561c12 | ||
|
|
2667d68675 | ||
|
|
57f2c22e30 | ||
|
|
5f3545d244 | ||
|
|
6e17ef0859 | ||
|
|
6ec97af104 | ||
|
|
3a6c1aa5a3 | ||
|
|
1ba3264747 | ||
|
|
ae2909fe8d | ||
|
|
6a3d19a453 | ||
|
|
a35596d2d1 | ||
|
|
318f654fed | ||
|
|
5e82ef65de | ||
|
|
8534607329 | ||
|
|
97e1a4cad3 | ||
|
|
3f6bf33de5 | ||
|
|
359301e8fc |
@@ -13,7 +13,12 @@ jobs:
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
with:
|
||||
components: clippy, rustfmt
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo fmt --check
|
||||
# libfreemkv is a library — Cargo.lock is gitignored. --locked
|
||||
# would always fail on a fresh runner because there's no committed
|
||||
# lockfile to lock against. The binary crates (freemkv, autorip,
|
||||
# bdemu) track Cargo.lock and DO use --locked.
|
||||
- run: cargo clippy -- -D warnings
|
||||
|
||||
test:
|
||||
@@ -21,6 +26,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo test --tests
|
||||
|
||||
check-macos:
|
||||
@@ -28,6 +34,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: cargo check
|
||||
|
||||
check-windows:
|
||||
@@ -35,4 +42,10 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
- run: cargo check
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
# Build the tests (not just `cargo check`): catches errors in test
|
||||
# code and forces full codegen of the Windows-only SPTI transport
|
||||
# (src/scsi/windows.rs), which never compiles on the Linux/macOS dev
|
||||
# hosts. We don't `cargo test` here — the suite needs no drive but the
|
||||
# extra build is the value; running tests is covered by the Linux job.
|
||||
- run: cargo build --tests
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
name: leak-guard
|
||||
|
||||
# Self-contained public-repo leak gate. Public CI cannot reach the private
|
||||
# tooling, so this encodes only the generic net: internal-infra references,
|
||||
# tracked CLAUDE.md/.claude paths, and AI-attribution in commit messages.
|
||||
# No project-specific reverse-engineering vocabulary lives here.
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
leak-guard:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Compute commit range
|
||||
id: range
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
base="${{ github.event.pull_request.base.sha }}"
|
||||
head="${{ github.event.pull_request.head.sha }}"
|
||||
echo "range=$base..$head" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
before="${{ github.event.before }}"
|
||||
after="${{ github.sha }}"
|
||||
# New branch / first push: github.event.before is all-zeros.
|
||||
if [ -z "$before" ] || [ "$before" = "0000000000000000000000000000000000000000" ]; then
|
||||
echo "range=$after" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "range=$before..$after" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
fi
|
||||
- name: Run leak-guard
|
||||
run: bash ci/leak-guard.sh "${{ steps.range.outputs.range }}"
|
||||
@@ -22,27 +22,51 @@ jobs:
|
||||
fi
|
||||
echo "Version match: $CARGO_VER"
|
||||
|
||||
# Tests run as a PARALLEL TRIPWIRE: they fail the run if they fail, but the
|
||||
# publish/release jobs do NOT `needs:` this job. The tag decision was already
|
||||
# gated by the local precommit (same Rust 1.86, same commit). Binary consumers
|
||||
# (freemkv/autorip/bdemu) git-tag-pin libfreemkv and therefore start building
|
||||
# the instant this tag exists — so this test job and the crates.io publish
|
||||
# below must NOT sit on their critical path.
|
||||
test:
|
||||
needs: verify
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
# libfreemkv is a library — Cargo.lock isn't tracked, so --locked
|
||||
# would always fail (no lockfile to lock against on a fresh runner).
|
||||
- run: cargo test
|
||||
|
||||
# crates.io publish is an INDEPENDENT job: it serves EXTERNAL consumers only.
|
||||
# The freemkv binaries no longer depend on it (they git-tag-pin libfreemkv via
|
||||
# a committed [patch.crates-io]), so this publish runs in parallel with their
|
||||
# release builds rather than gating them. It `needs: [verify, test]` so a
|
||||
# failing test suite still blocks publication to crates.io — external
|
||||
# consumers who `cargo add libfreemkv` must never receive a release whose
|
||||
# tests were failing. (The two upstream jobs run in parallel, so this gate
|
||||
# does not serialize publish behind test beyond their own completion.)
|
||||
publish:
|
||||
needs: test
|
||||
needs: [verify, test]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@1.86.0
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
# --no-verify: CI already compiled this exact commit (in the `test` job
|
||||
# and on every push via ci.yml). cargo publish's default re-verify does a
|
||||
# full cold release build of the packaged tarball, which here is pure
|
||||
# redundant work (~a cold lib build). Skip it.
|
||||
- name: Publish to crates.io
|
||||
run: cargo publish
|
||||
run: cargo publish --no-verify
|
||||
env:
|
||||
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
||||
|
||||
release:
|
||||
needs: test
|
||||
# Only needs `verify`; the GitHub Release can be cut as soon as the version
|
||||
# check passes, in parallel with test + publish.
|
||||
needs: verify
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
+10
@@ -4,3 +4,13 @@ Cargo.lock
|
||||
*.swo
|
||||
.DS_Store
|
||||
.cargo/
|
||||
|
||||
# session scratch — never track (may contain RE breadcrumbs)
|
||||
scratch/
|
||||
|
||||
# stray local build artifact
|
||||
/rust_out
|
||||
|
||||
# internal agent context — never publish (path AND dir; leak-guard blocks both)
|
||||
CLAUDE.md
|
||||
.claude/
|
||||
|
||||
+427
-1630
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,83 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation.
|
||||
|
||||
We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to a positive environment for our community include:
|
||||
|
||||
* Demonstrating empathy and kindness toward other people
|
||||
* Being respectful of differing opinions, viewpoints, and experiences
|
||||
* Giving and gracefully accepting constructive feedback
|
||||
* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience
|
||||
* Focusing on what is best not just for us as individuals, but for the overall community
|
||||
|
||||
Examples of unacceptable behavior include:
|
||||
|
||||
* The use of sexualized language or imagery, and sexual attention or advances of any kind
|
||||
* Trolling, insulting or derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or email address, without their explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a professional setting
|
||||
|
||||
## Enforcement Responsibilities
|
||||
|
||||
Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful.
|
||||
|
||||
Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at owner@example.invalid. All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
All community leaders are obligated to respect the privacy and security of the reporter of any incident.
|
||||
|
||||
## Enforcement Guidelines
|
||||
|
||||
Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct:
|
||||
|
||||
### 1. Correction
|
||||
|
||||
**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community.
|
||||
|
||||
**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested.
|
||||
|
||||
### 2. Warning
|
||||
|
||||
**Community Impact**: A violation through a single incident or series of actions.
|
||||
|
||||
**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban.
|
||||
|
||||
### 3. Temporary Ban
|
||||
|
||||
**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior.
|
||||
|
||||
**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban.
|
||||
|
||||
### 4. Permanent Ban
|
||||
|
||||
**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals.
|
||||
|
||||
**Consequence**: A permanent ban from any sort of public interaction within the community.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
|
||||
|
||||
Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC].
|
||||
|
||||
For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at [https://www.contributor-covenant.org/translations][translations].
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
|
||||
[Mozilla CoC]: https://github.com/mozilla/diversity
|
||||
[FAQ]: https://www.contributor-covenant.org/faq
|
||||
[translations]: https://www.contributor-covenant.org/translations
|
||||
+24
-3
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "libfreemkv"
|
||||
version = "0.17.0"
|
||||
version = "1.1.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.86"
|
||||
license = "AGPL-3.0-only"
|
||||
@@ -8,6 +8,12 @@ description = "Open source raw disc access library for optical drives"
|
||||
repository = "https://github.com/freemkv/libfreemkv"
|
||||
keywords = ["bluray", "uhd", "optical", "scsi", "disc"]
|
||||
categories = ["hardware-support", "multimedia"]
|
||||
# Keep internal AI-instruction / private notes out of the published crate.
|
||||
exclude = ["CLAUDE.md"]
|
||||
|
||||
[profile.release]
|
||||
lto = "thin"
|
||||
codegen-units = 1
|
||||
|
||||
[dependencies]
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
@@ -16,7 +22,6 @@ sha1 = "0.10"
|
||||
sha2 = "0.10"
|
||||
aes = "0.8"
|
||||
cbc = "0.1"
|
||||
flate2 = "1"
|
||||
num-bigint = "0.4"
|
||||
num-traits = "0.2"
|
||||
num-integer = "0.1"
|
||||
@@ -25,9 +30,25 @@ cmac = "0.7"
|
||||
zip = { version = "2", default-features = false, features = ["deflate"] }
|
||||
base64 = "0.22.1"
|
||||
# Trace-level instrumentation for Disc::copy + SgIoTransport::execute. Permitted
|
||||
# under project docs ("Acceptable strings: debug/trace logging"). Consumers (autorip)
|
||||
# under CLAUDE.md ("Acceptable strings: debug/trace logging"). Consumers (autorip)
|
||||
# wire a tracing subscriber and pipe events into the JSONL debug log.
|
||||
tracing = "0.1"
|
||||
# Bounded MPSC channel with kernel-wakeup send_timeout. Used by `io::pipeline`
|
||||
# so the halt-aware send/finish loops can BLOCK on consumer drain instead of
|
||||
# polling — the 50 ms poll cadence of the previous mpsc-based impl capped mux
|
||||
# throughput at ~1 MB/s (0.21.7).
|
||||
crossbeam-channel = "0.5"
|
||||
# Persistent work-stealing thread pool for parallel AACS unit
|
||||
# decryption. Per-call std::thread::scope spawned fresh OS threads
|
||||
# and that overhead dominated for typical batch sizes (60 units).
|
||||
# rayon's global pool initialises once on first use.
|
||||
rayon = "1"
|
||||
# SIMD-accelerated bytestring search. Drives the HEVC/H.264 start-code
|
||||
# scan in `mux::codec::h264::find_start_code` — naive byte-by-byte
|
||||
# walk is ~500 MB/s single-thread on x86_64; memchr's vectorised
|
||||
# `memmem::find` for the 3-byte `00 00 01` needle hits ~5 GB/s on
|
||||
# AVX2-capable hosts.
|
||||
memchr = "2"
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
# libfreemkv — Feature List
|
||||
|
||||
## v0.10.10 (current)
|
||||
|
||||
### Done
|
||||
- [x] Drive access: open, identify, unlock, firmware upload, speed calibration, eject
|
||||
- [x] 206 bundled drive profiles (MediaTek MT1959 A + B variants)
|
||||
- [x] SCSI transport: Linux SG_IO, macOS IOKit, Windows SPTI
|
||||
- [x] UDF 2.50 filesystem parser (metadata partitions, Blu-ray profile)
|
||||
- [x] MPLS playlist parser (play items, STN table, secondary streams)
|
||||
- [x] CLPI clip info parser (EP map, sector extents)
|
||||
- [x] AACS 1.0 decryption (4 VUK paths: KEYDB, media key, processing key, device key)
|
||||
- [x] AACS 2.0 SCSI handshake (P-256/SHA-256 ECDH, bus decryption, read data key)
|
||||
- [x] DVD IFO parser (VMG, VTS, PGC chains, cell addresses)
|
||||
- [x] DVD CSS decryption (bus auth, disc key via player keys, title key, sector descramble)
|
||||
- [x] CSS Stevenson plaintext attack for ISO key recovery
|
||||
- [x] MPEG-2 PS demuxer (DVD Program Stream with PES extraction)
|
||||
- [x] MPEG-2 video codec parser (sequence header, quantizer matrices, keyframe detection)
|
||||
- [x] KEYDB.cfg download, verify, save (raw TCP, zero HTTP deps)
|
||||
- [x] Content reading with adaptive batch sizing, error recovery, 12+ MB/s
|
||||
- [x] Stream labels: 5 BD-J format parsers (Paramount, Criterion, Pixelogic, CTRM, Deluxe)
|
||||
- [x] MKV muxer: 14 codec parsers (H.264, HEVC, MPEG-2, AC-3, DTS, TrueHD, PGS, DVD Sub, VC-1, LPCM, +4 more)
|
||||
- [x] SectorReader trait: decouples disc scanning from SCSI
|
||||
- [x] 7 stream types: Disc, ISO, MKV, M2TS, Network, Stdio, Null
|
||||
- [x] PES pipeline with unified Stream trait (any source → any dest)
|
||||
- [x] FMKV metadata header for M2TS and network streams
|
||||
- [x] Numeric error codes only (no English text in library)
|
||||
- [x] Event system for progress callbacks
|
||||
|
||||
### Planned
|
||||
- [ ] Windows testing on real hardware
|
||||
- [ ] Pioneer Renesas platform support (48 drives, need GET_CONFIG 010C)
|
||||
- [ ] TranscodeStream (ffmpeg integration)
|
||||
- [ ] ISO write with BD-compliant UDF structure
|
||||
@@ -1,6 +1,5 @@
|
||||
# libfreemkv — local dev helper.
|
||||
# Mirrors the cross-crate scripts in (internal)/scripts/test-all.sh
|
||||
# but scoped to this single crate.
|
||||
# Mirrors the workspace-wide CI checks but scoped to this single crate.
|
||||
|
||||
.PHONY: test build check ci clean
|
||||
|
||||
|
||||
@@ -4,9 +4,11 @@
|
||||
|
||||
# libfreemkv
|
||||
|
||||
Rust library for 4K UHD / Blu-ray / DVD optical drives. Drive access, disc scanning, stream labels, AACS decryption, CSS decryption, KEYDB updates, and content reading in one crate. Bundled drive profiles — no external files needed.
|
||||
Rust library for 4K UHD / Blu-ray / DVD optical drives. Drive access, disc scanning, stream labels, AACS decryption, CSS decryption, KEYDB updates, and content reading in one crate. Firmware-clean core: drive-unlock support is plugged in via the `Unlocker` trait, with concrete unlockers shipped as separate crates.
|
||||
|
||||
**12+ MB/s** sustained read speeds on BD. Full init: unlock, firmware upload, speed calibration — all from pure Rust.
|
||||
DVDs (CSS) decrypt out of the box. Blu-ray and UHD (AACS) require a `keydb.cfg` (default `~/.config/freemkv/keydb.cfg`) supplying disc-specific volume unique keys; no AACS key material is compiled in.
|
||||
|
||||
**12+ MB/s** sustained read speeds on BD. Drive prep routes through the pluggable unlock seam — register an unlocker and `init()` drives it; with none registered the library rips via the host-certificate AACS handshake.
|
||||
|
||||
Multi-lingual by design — the library outputs structured data and numeric error codes, never English text. Build any UI or localization on top.
|
||||
|
||||
@@ -18,7 +20,7 @@ Part of the [freemkv](https://github.com/freemkv) project.
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
libfreemkv = "0.17"
|
||||
libfreemkv = "1.0.0-rc.1"
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
@@ -27,10 +29,10 @@ libfreemkv = "0.17"
|
||||
use libfreemkv::{Drive, Disc, ScanOptions};
|
||||
use std::path::Path;
|
||||
|
||||
// Open drive — profiles are bundled, auto-identified
|
||||
// Open drive — identified via INQUIRY
|
||||
let mut drive = Drive::open(Path::new("/dev/sg4"))?;
|
||||
drive.wait_ready()?; // wait for disc
|
||||
drive.init()?; // unlock + firmware upload
|
||||
drive.init()?; // route through the unlock seam (if an unlocker is registered)
|
||||
drive.probe_disc()?; // probe disc surface for optimal speeds
|
||||
|
||||
// Scan disc — UDF, playlists, streams, AACS (all automatic)
|
||||
@@ -53,32 +55,52 @@ output.finish()?;
|
||||
|
||||
### Multi-pass recovery rip
|
||||
|
||||
For damaged discs, the library offers a two-stage rip model: fast sweep with zero-fill and a ddrescue-format mapfile, then targeted retry of bad ranges. See [`docs/rip-recovery.md`](docs/rip-recovery.md) for the full architecture.
|
||||
For damaged discs the library exposes two flat verbs — `Disc::sweep` for the
|
||||
forward Pass 1 and `Disc::patch` for retrying bad ranges. The library never
|
||||
loops; the multipass policy is the caller's job. See
|
||||
[`docs/rip-recovery.md`](docs/rip-recovery.md).
|
||||
|
||||
```rust
|
||||
use libfreemkv::disc::{CopyOptions, PatchOptions};
|
||||
use libfreemkv::{SweepOptions, PatchOptions};
|
||||
use libfreemkv::disc::{mapfile, mapfile_path_for};
|
||||
use std::path::Path;
|
||||
|
||||
// Pass 1: disc → ISO. Fast 64 KB reads, skip-forward on failure,
|
||||
// zero-fill bad blocks, write a sidecar .mapfile.
|
||||
let mut result = disc.copy(
|
||||
&mut drive,
|
||||
Path::new("disc.iso"),
|
||||
&CopyOptions { skip_on_error: true, skip_forward: true, ..Default::default() },
|
||||
)?;
|
||||
let iso = Path::new("disc.iso");
|
||||
|
||||
// Pass 2..N: retry bad ranges with full drive recovery.
|
||||
// Idempotent — call as many times as you want.
|
||||
while result.bytes_unreadable + result.bytes_pending > 0 {
|
||||
let pr = disc.patch(&mut drive, Path::new("disc.iso"), &PatchOptions::default())?;
|
||||
if pr.bytes_recovered_this_pass == 0 { break; }
|
||||
// Pass 1: disc → ISO. Skip-on-error, zero-fill, write the sidecar mapfile.
|
||||
disc.sweep(&mut drive, iso, &SweepOptions {
|
||||
decrypt: true,
|
||||
resume: false,
|
||||
batch_sectors: None,
|
||||
skip_on_error: true,
|
||||
progress: None,
|
||||
halt: None,
|
||||
})?;
|
||||
|
||||
// Pass 2..N: retry every non-finished range. Idempotent.
|
||||
loop {
|
||||
let map = mapfile::Mapfile::load(&mapfile_path_for(iso))?;
|
||||
let stats = map.stats();
|
||||
if stats.bytes_pending + stats.bytes_unreadable == 0 { break; }
|
||||
|
||||
let outcome = disc.patch(&mut drive, iso, &PatchOptions {
|
||||
decrypt: true,
|
||||
block_sectors: None,
|
||||
full_recovery: true,
|
||||
reverse: true,
|
||||
wedged_threshold: 50,
|
||||
progress: None,
|
||||
halt: None,
|
||||
})?;
|
||||
if outcome.bytes_recovered_this_pass == 0 { break; }
|
||||
}
|
||||
|
||||
// Then mux from the ISO via the normal stream pipeline (no drive involvement).
|
||||
// Mux from the ISO via the normal stream pipeline (no drive involvement).
|
||||
```
|
||||
|
||||
## What It Does
|
||||
|
||||
- **Drive access** — open, identify, unlock, firmware upload, speed calibration, eject
|
||||
- **Drive access** — open, identify, pluggable unlock seam, speed control, eject
|
||||
- **12+ MB/s reads** — auto-detects kernel transfer limits, sustained full speed
|
||||
- **Disc scanning** — UDF 2.50 filesystem, MPLS playlists, CLPI clip info
|
||||
- **Stream labels** — 5 BD-J format parsers (Paramount, Criterion, Pixelogic, CTRM, Deluxe)
|
||||
@@ -92,24 +114,28 @@ while result.bytes_unreadable + result.bytes_pending > 0 {
|
||||
| Stream | Input | Output | Transport |
|
||||
|--------|-------|--------|-----------|
|
||||
| DiscStream | Yes | -- | Optical drive via SCSI |
|
||||
| IsoStream | Yes | -- | Blu-ray ISO image file (read via stream pipeline; written via `Disc::copy()`) |
|
||||
| IsoStream | Yes | -- | Blu-ray ISO image file (read via stream pipeline; written via `Disc::sweep()`) |
|
||||
| MkvStream | Yes | Yes | Matroska container |
|
||||
| M2tsStream | Yes | Yes | BD transport stream with FMKV metadata header |
|
||||
| NetworkStream | Yes (listen) | Yes (connect) | TCP with FMKV metadata header |
|
||||
| StdioStream | Yes (stdin) | Yes (stdout) | Raw byte pipe |
|
||||
| NullStream | -- | Yes | Discard sink (byte counter for benchmarks) |
|
||||
|
||||
Streams implement `pes::Stream` (frame-level). `input()` / `output()` resolve URL strings to PES stream instances. All URLs use the `scheme://path` format — bare paths are rejected.
|
||||
Streams implement a single unified `pes::Stream` trait (re-exported as `PesStream`) exposing `read()` and `write()` on one type. `input()` / `output()` resolve URL strings to PES stream instances. All URLs use the `scheme://path` format — bare paths are rejected.
|
||||
|
||||
AACS decryption requires a KEYDB.cfg file. If available at `~/.config/aacs/KEYDB.cfg` or passed via `ScanOptions`, the library handles everything — handshake, key derivation, and per-sector decryption — without the application needing to know anything about encryption.
|
||||
### Keys
|
||||
|
||||
DVDs (CSS) decrypt out of the box, with no external key file needed.
|
||||
|
||||
Blu-rays and UHD (AACS) require a `keydb.cfg` at `~/.config/freemkv/keydb.cfg` (or passed via `ScanOptions`). No AACS key material is compiled into the binary.
|
||||
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
Drive — open, identify, init, unlock, single-shot read
|
||||
Drive — open, identify, init, single-shot read
|
||||
├── ScsiTransport — SG_IO (Linux), IOKit (macOS), SPTI (Windows)
|
||||
├── DriveProfile — per-drive unlock parameters (bundled)
|
||||
└── PlatformDriver — MediaTek (supported), Renesas (planned)
|
||||
└── Unlocker seam — pluggable trait + registry; concrete unlockers
|
||||
live in the separate freemkv-unlock repo
|
||||
|
||||
Disc — scan titles, streams, AACS/CSS state
|
||||
├── UDF reader — Blu-ray UDF 2.50 with metadata partitions
|
||||
@@ -118,11 +144,11 @@ Disc — scan titles, streams, AACS/CSS state
|
||||
├── IFO parser — DVD title sets, PGC chains, cell addresses
|
||||
├── Labels — 5 BD-J format parsers (detect + parse)
|
||||
├── AACS — key resolution + content decryption
|
||||
├── CSS — DVD CSS cipher (table-driven, no keys needed)
|
||||
├── CSS — DVD CSS (bus auth → player-key disc crack → known-plaintext title-key attack)
|
||||
└── KEYDB — download + verify + save
|
||||
|
||||
Streams — unified PES pipeline
|
||||
├── pes::Stream — read()/write() PES frames
|
||||
├── PesStream — pes::Stream: one trait, read()/write() PES frames
|
||||
├── DiscStream — sectors → decrypt → TS demux → PES
|
||||
├── IsoStream — ISO file → decrypt → TS demux → PES
|
||||
├── MkvStream — MKV mux/demux
|
||||
@@ -148,6 +174,7 @@ All errors are structured with numeric codes. No user-facing English text — ap
|
||||
| E6xxx | Disc format errors |
|
||||
| E7xxx | AACS errors |
|
||||
| E8xxx | KEYDB update errors |
|
||||
| E9xxx | Stream / mux errors (URL, PES, ISO, pipeline, demux) |
|
||||
|
||||
## Platform Support
|
||||
|
||||
@@ -159,7 +186,7 @@ All errors are structured with numeric codes. No user-facing English text — ap
|
||||
|
||||
## Contributing
|
||||
|
||||
Run `freemkv info disc:// --share` with the [freemkv CLI](https://github.com/freemkv/freemkv) to contribute your drive's profile.
|
||||
Run `freemkv info disc:// --share` with the [freemkv CLI](https://github.com/freemkv/freemkv) to capture your drive's identity for contribution. Drive-unlock profiles are maintained in the [freemkv-unlock](https://github.com/freemkv/freemkv-unlock) repository.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
fn main() {
|
||||
emit_git_suffix();
|
||||
|
||||
let target = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default();
|
||||
if target == "macos" {
|
||||
println!("cargo:rustc-link-lib=framework=IOKit");
|
||||
@@ -8,8 +10,22 @@ fn main() {
|
||||
let obj = format!("{out_dir}/macos_shim.o");
|
||||
let lib = format!("{out_dir}/libmacos_scsi.a");
|
||||
|
||||
// Build the shim for the TARGET arch, not the host's. A bare `cc` on an
|
||||
// Apple-Silicon CI runner defaults to arm64, so cross-building to
|
||||
// x86_64-apple-darwin would link a host-arch object against x86_64 Rust
|
||||
// code → "Undefined symbols for architecture x86_64". (Still raw `cc`,
|
||||
// not the `cc` crate, which breaks IOKit exclusive access.)
|
||||
let target_arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default();
|
||||
let clang_arch: &str = if target_arch == "aarch64" {
|
||||
"arm64"
|
||||
} else {
|
||||
&target_arch // x86_64 → x86_64
|
||||
};
|
||||
|
||||
std::process::Command::new("cc")
|
||||
.args([
|
||||
"-arch",
|
||||
clang_arch,
|
||||
"-c",
|
||||
"src/scsi/macos_shim.c",
|
||||
"-o",
|
||||
@@ -34,3 +50,48 @@ fn main() {
|
||||
println!("cargo:rerun-if-changed=src/scsi/macos_shim.c");
|
||||
}
|
||||
}
|
||||
|
||||
/// Bake the git short hash into the build as `GIT_SUFFIX` so any muxed MKV or
|
||||
/// FVI index is traceable to the exact source revision (e.g. ` (g835cc99)`).
|
||||
/// Empty when git or the repo is unavailable (e.g. a crates.io tarball build),
|
||||
/// leaving just the package version. Always emitted so `env!("GIT_SUFFIX")`
|
||||
/// resolves on every target.
|
||||
fn emit_git_suffix() {
|
||||
// Version label for the muxing-app / FVI generator tag. `FREEMKV_BUILD_LABEL`
|
||||
// overrides the Cargo package version when set (non-empty) — used to stamp a
|
||||
// pre-release/test build without bumping Cargo.toml and disturbing the
|
||||
// tag-pinned [patch] version matching. Unset → the package version.
|
||||
let version = std::env::var("FREEMKV_BUILD_LABEL")
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.or_else(|| std::env::var("CARGO_PKG_VERSION").ok())
|
||||
.unwrap_or_default();
|
||||
println!("cargo:rustc-env=FREEMKV_VERSION={version}");
|
||||
println!("cargo:rerun-if-env-changed=FREEMKV_BUILD_LABEL");
|
||||
|
||||
let suffix = git_short_hash()
|
||||
.map(|h| format!(" (g{h})"))
|
||||
.unwrap_or_default();
|
||||
println!("cargo:rustc-env=GIT_SUFFIX={suffix}");
|
||||
|
||||
// Re-run when HEAD (or the branch it points at) moves so the stamp stays
|
||||
// current without a clean rebuild.
|
||||
println!("cargo:rerun-if-changed=.git/HEAD");
|
||||
if let Ok(head) = std::fs::read_to_string(".git/HEAD") {
|
||||
if let Some(ref_path) = head.strip_prefix("ref: ") {
|
||||
println!("cargo:rerun-if-changed=.git/{}", ref_path.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn git_short_hash() -> Option<String> {
|
||||
let out = std::process::Command::new("git")
|
||||
.args(["rev-parse", "--short=7", "HEAD"])
|
||||
.output()
|
||||
.ok()?;
|
||||
if !out.status.success() {
|
||||
return None;
|
||||
}
|
||||
let h = String::from_utf8(out.stdout).ok()?.trim().to_string();
|
||||
if h.is_empty() { None } else { Some(h) }
|
||||
}
|
||||
|
||||
Executable
+120
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# leak-guard.sh — self-contained public-repo leak gate.
|
||||
#
|
||||
# This is the LAST line of defense in CI. It is intentionally self-contained:
|
||||
# public CI cannot reach the private tooling, so this script encodes ONLY the
|
||||
# generic net — internal infrastructure references, agent-context files, and
|
||||
# AI-attribution in commit messages. It deliberately contains NO project-
|
||||
# specific reverse-engineering vocabulary (those words would themselves be a
|
||||
# leak). The richer private scanner stays private.
|
||||
#
|
||||
# Fails (exit 1) if any of the following appear in the repo:
|
||||
# 1. a tracked CLAUDE.md or .claude/ path (agent context — never public),
|
||||
# 2. tracked file content matching the internal-infra net,
|
||||
# 3. a commit message (in the given range) with AI attribution.
|
||||
#
|
||||
# Usage:
|
||||
# leak-guard.sh [<commit-range>]
|
||||
# <commit-range> optional git rev-list range to scan commit messages
|
||||
# (e.g. "abc..def"). If omitted, commit-message scan is
|
||||
# skipped (path + content checks always run).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Absolute path to this script, resolved before any cd, so we can exclude it
|
||||
# from the content scan (it necessarily contains the detection patterns).
|
||||
SELF_ABS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"
|
||||
|
||||
REPO="$(git rev-parse --show-toplevel)"
|
||||
cd "$REPO"
|
||||
|
||||
fail=0
|
||||
note() { printf ' ✗ %s\n' "$1"; fail=1; }
|
||||
|
||||
# Internal-infra net — GENERIC ONLY. This script ships in the public repo, so
|
||||
# the patterns themselves must not name any org-specific identifier (doing so
|
||||
# would itself leak the infra they guard). We catch the leak *class*:
|
||||
# - RFC1918 private IPv4 ranges (10/8, 172.16/12, 192.168/16),
|
||||
# - private/internal/non-routable TLDs (.internal/.local/.lan/.corp/.invalid),
|
||||
# - docker.internal.
|
||||
# The full org-specific net (literal hostnames, service names, repo paths,
|
||||
# vendor tooling, …) lives ONLY in the private scanner and never ships here.
|
||||
INFRA_RE='\b10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|\b172\.(1[6-9]|2[0-9]|3[01])\.[0-9]{1,3}\.[0-9]{1,3}|\b192\.168\.[0-9]{1,3}\.[0-9]{1,3}|\.internal\b|\.local\b|\.lan\b|\.corp\b|\.invalid\b|docker\.internal'
|
||||
# Home-path net — GENERIC ONLY. Catches an absolute developer home path
|
||||
# committed into a tracked file (a macOS /Users/<user>/… or Linux /home/<user>/…
|
||||
# path). This names NO specific user — it matches the leak *class* (any home
|
||||
# path), so the pattern itself reveals nothing org- or person-specific. A real
|
||||
# leak (e.g. /Users/alice/Developer/x slipping into a public RELEASE.md) trips
|
||||
# this regardless of whose machine it came from. The username segment is a
|
||||
# literal-username class ([A-Za-z0-9._-]) so dynamic/templated paths that build
|
||||
# the user at runtime — shell `/home/$USER/`, doc `/home/<rip>/`, Rust
|
||||
# `/home/{user}/` — do NOT false-positive; only a baked-in literal home leaks.
|
||||
HOMEPATH_RE='/Users/[A-Za-z0-9._-]+/|/home/[A-Za-z0-9._-]+/'
|
||||
# AI-attribution net (case-insensitive). "claude" matches only as a standalone
|
||||
# word — NOT preceded by a dot/slash/alnum and NOT followed by .md — so legit
|
||||
# mentions of CLAUDE.md / .claude/ in a commit message don't false-positive.
|
||||
ATTR_RE='co-authored-by|generated with|🤖|(?<![.\/A-Za-z0-9])claude(?!\.md)'
|
||||
|
||||
echo "── leak-guard: tracked agent-context paths ──"
|
||||
while IFS= read -r f; do
|
||||
case "$f" in
|
||||
CLAUDE.md|*/CLAUDE.md|.claude|.claude/*|*/.claude|*/.claude/*)
|
||||
note "tracked agent-context file: $f (CLAUDE.md/.claude must never be tracked in a public repo)" ;;
|
||||
esac
|
||||
done < <(git ls-files)
|
||||
|
||||
# Match a PCRE against a file, emitting "LINE: MATCH". The pattern is passed as
|
||||
# an argument (not interpolated into a //) so metacharacters like the "/" in a
|
||||
# path-style token can't break the regex. Reads raw bytes so non-UTF-8 blobs
|
||||
# don't abort the scan.
|
||||
pcre_matches() {
|
||||
perl -e '
|
||||
my ($file, $re) = @ARGV;
|
||||
open(my $fh, "<:raw", $file) or exit 0;
|
||||
my $rx; eval { $rx = qr/$re/i }; exit 0 if $@;
|
||||
while (my $l = <$fh>) { if ($l =~ /$rx/) { print "$.: $&\n"; } }
|
||||
' "$1" "$2" 2>/dev/null
|
||||
}
|
||||
|
||||
# This script's own source necessarily contains the detection patterns (e.g.
|
||||
# the regex tokens in INFRA_RE), so scanning it would always self-flag. Skip it.
|
||||
SELF="$(git ls-files --full-name -- "$SELF_ABS" 2>/dev/null | head -1)"
|
||||
|
||||
echo "── leak-guard: internal-infra references in tracked files ──"
|
||||
while IFS= read -r f; do
|
||||
case "$f" in *.png|*.jpg|*.jpeg|*.ico|*.gif|*.bin|*.crate|*.gz|*.zip|*.pdf) continue ;; esac
|
||||
[ -n "$SELF" ] && [ "$f" = "$SELF" ] && continue
|
||||
[ -f "$f" ] || continue
|
||||
while IFS= read -r hit; do
|
||||
[ -z "$hit" ] && continue
|
||||
note "internal-infra reference: $f:$hit"
|
||||
done < <(pcre_matches "$f" "$INFRA_RE")
|
||||
while IFS= read -r hit; do
|
||||
[ -z "$hit" ] && continue
|
||||
note "[HOME-PATH] absolute home path: $f:$hit (no local home path may be committed to a public repo)"
|
||||
done < <(pcre_matches "$f" "$HOMEPATH_RE")
|
||||
done < <(git ls-files)
|
||||
|
||||
RANGE="${1:-}"
|
||||
if [ -n "$RANGE" ]; then
|
||||
echo "── leak-guard: AI-attribution in commit messages ($RANGE) ──"
|
||||
while IFS= read -r sha; do
|
||||
[ -z "$sha" ] && continue
|
||||
msg="$(git log -1 --format='%B' "$sha" 2>/dev/null || true)"
|
||||
# Pass the pattern as an argument (not interpolated into a //) so the
|
||||
# lookbehind char class and "/" don't break the regex.
|
||||
hit="$(printf '%s' "$msg" | perl -e '
|
||||
my $re = $ARGV[0]; my $rx = qr/$re/i;
|
||||
while (my $l = <STDIN>) { if ($l =~ /($rx)/) { print "$1\n"; last; } }
|
||||
' "$ATTR_RE" | head -1 || true)"
|
||||
[ -n "$hit" ] && note "commit ${sha:0:12}: message contains \"$hit\" (owner rule: zero AI attribution, ever)"
|
||||
done < <(git rev-list "$RANGE" 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fail" -ne 0 ]; then
|
||||
echo "✗ leak-guard: blocking finding(s) above — DO NOT MERGE/PUBLISH"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ leak-guard: clean"
|
||||
@@ -0,0 +1,302 @@
|
||||
# FVI — Freemkv Video Index Format
|
||||
|
||||
**Specification version:** 1.0 (DRAFT)\
|
||||
**File extension:** `.fvi`\
|
||||
**Media type:** `application/vnd.freemkv.fvi+jsonl`\
|
||||
**Status:** Draft for review. This document is the normative reference for the FVI
|
||||
format; implementations and downstream tools cite it by section.
|
||||
|
||||
---
|
||||
|
||||
## 1. Scope and purpose
|
||||
|
||||
FVI is an open, codec-agnostic, byte-exact **index of the coded pictures** in a
|
||||
video bitstream, together with **provenance** back to the source medium.
|
||||
|
||||
An FVI document answers, for every picture in a stream, three questions:
|
||||
|
||||
1. **Where is it?** — the byte-exact offset of its first byte in the *source*
|
||||
(the disc/ISO/file), so a reader can extract or seek to any picture without
|
||||
re-parsing the whole bitstream.
|
||||
2. **What is it?** — coding type, random-access capability, GOP boundary, and
|
||||
(where the codec defines them) field/pulldown attributes.
|
||||
3. **When is it?** — decode and presentation timestamps on a declared timescale.
|
||||
|
||||
FVI is **not** a container, a codec, or a copy of the bitstream. It indexes; it
|
||||
never stores coded samples. It is the serialized form of an indexer's per-picture
|
||||
truth — carried from the demuxer, **never reconstructed** (§9).
|
||||
|
||||
## 2. Conformance
|
||||
|
||||
The key words **MUST**, **MUST NOT**, **REQUIRED**, **SHALL**, **SHALL NOT**,
|
||||
**SHOULD**, **SHOULD NOT**, **MAY**, and **OPTIONAL** are to be interpreted as
|
||||
described in BCP 14 (RFC 2119, RFC 8174) when, and only when, they appear in all
|
||||
capitals.
|
||||
|
||||
A **conformant writer** MUST emit a document that satisfies §4–§10. A
|
||||
**conformant reader** MUST accept any such document and MUST ignore unknown
|
||||
object members (§11) so that forward-compatible extensions do not break it.
|
||||
|
||||
## 3. Terminology
|
||||
|
||||
- **Picture** — one coded video frame (or pair of fields coded as a frame). The
|
||||
unit FVI indexes.
|
||||
- **Access unit (AU)** — the set of bitstream bytes that decode to exactly one
|
||||
picture (ISO/IEC 14496-10 §3; ISO/IEC 23008-2 §3).
|
||||
- **Coded order** — the order pictures appear in the bitstream. FVI records are
|
||||
emitted in coded order.
|
||||
- **GOP / coded video sequence** — a self-contained run beginning at a
|
||||
random-access point.
|
||||
- **Provenance** — the mapping from an AU back to the exact bytes of the physical
|
||||
source it was read from (§9).
|
||||
- **Source position (`src`)** — `{ file, sector, byte }`, the provenance anchor of
|
||||
an AU.
|
||||
|
||||
## 4. Encoding
|
||||
|
||||
An FVI document is a sequence of **UTF-8** text lines separated by a single LF
|
||||
(`U+000A`). Each non-empty line is exactly one JSON value (RFC 8259), forming a
|
||||
**JSON Lines / NDJSON** stream. A writer MUST NOT emit a UTF-8 BOM. A writer MUST
|
||||
NOT pretty-print: each JSON value occupies exactly one line.
|
||||
|
||||
The first line MUST be the **Header** object (§6). Each subsequent line is one
|
||||
**Picture record** (§7), in coded order.
|
||||
|
||||
Rationale: line-delimited JSON is streamable (a writer appends as it indexes; a
|
||||
reader processes without loading the whole file), line-addressable (picture *n*
|
||||
is near line *n+1*), append-safe, and parseable by every language without a
|
||||
custom grammar — while remaining a precisely specified format, not an ad-hoc dump.
|
||||
|
||||
A document MAY be concatenated for multiple elementary streams: each stream is its
|
||||
own header line followed by its records. Readers MUST treat a Header line as the
|
||||
start of a new stream section.
|
||||
|
||||
## 5. Document structure
|
||||
|
||||
```
|
||||
<header> line 1 (exactly one Header object)
|
||||
<record> line 2 .. N (one Picture record per picture, coded order)
|
||||
[<header> <record>…] (OPTIONAL further stream sections)
|
||||
```
|
||||
|
||||
## 6. Header object
|
||||
|
||||
| Member | JSON type | Req | Semantics / reference |
|
||||
|---|---|---|---|
|
||||
| `format` | string | MUST | Constant `"freemkv/video-index"`. Signature: a document begins with these bytes. |
|
||||
| `fvi_version` | integer | MUST | Document format version. This spec defines `1`. |
|
||||
| `generator` | string | SHOULD | Producing tool + version, e.g. `"freemkv/1.0.0-rc.6"`. |
|
||||
| `stream` | object | MUST | The indexed elementary stream (§6.1). |
|
||||
| `source` | object | MUST | Provenance root (§6.2). |
|
||||
| `timescale` | integer | MUST | Ticks per second for all `pts`/`dts` (§10). E.g. `90000`. |
|
||||
| `picture_count` | integer | MAY | Total pictures, if known at header time; OMITTED when streaming. |
|
||||
|
||||
### 6.1 `stream` object
|
||||
|
||||
| Member | JSON type | Req | Semantics / reference |
|
||||
|---|---|---|---|
|
||||
| `codec` | string | MUST | Registered codec id (Appendix B), e.g. `"mpeg2video"`, `"hevc"`. |
|
||||
| `width`,`height` | integer | MUST | Coded luma dimensions in pixels. |
|
||||
| `dar` | `[int,int]` | SHOULD | Display aspect ratio as `[num,den]`. |
|
||||
| `frame_rate` | `[int,int]` | SHOULD | Nominal rate as exact rational `[num,den]` (e.g. `[24000,1001]`). |
|
||||
| `scan` | string | MUST | `"progressive"`<br>`"interlaced"`<br>`"mbaff"` |
|
||||
| `colour` | object | SHOULD | CICP per ITU-T H.273: `primaries`, `transfer`, `matrix` (integer CICP codes or registered names)<br>`range`: `"limited"` \| `"full"`<br>HDR: `mastering_display`, `max_cll`, `max_fall` per ITU-T H.273 / SMPTE ST 2086. |
|
||||
| `language` | string | MAY | BCP 47 tag, if known. |
|
||||
|
||||
### 6.2 `source` object
|
||||
|
||||
| Member | JSON type | Req | Semantics / reference |
|
||||
|---|---|---|---|
|
||||
| `medium` | string | MUST | `"disc"`<br>`"iso"`<br>`"file"`<br>`"stream"` |
|
||||
| `path` | string | MAY | Source path/label. |
|
||||
| `title` | integer | MAY | Title/program number. |
|
||||
| `playlist` | string | MAY | Playlist/PGC identifier. |
|
||||
| `volume_id` | string | MAY | Disc volume identifier, if read. |
|
||||
| `sector_size` | integer | SHOULD | Bytes per `src.sector` unit (e.g. `2048`). Lets readers convert `src` to an absolute byte offset. |
|
||||
|
||||
## 7. Picture record
|
||||
|
||||
One JSON object per coded picture, in coded order.
|
||||
|
||||
| Member | JSON type | Req | Semantics / reference |
|
||||
|---|---|---|---|
|
||||
| `n` | integer | MUST | Coded-order index, 0-based, contiguous. |
|
||||
| `src` | object | MUST | Provenance: `{ "file": int?, "sector": uint, "byte": uint }` — the offset of this AU's **first byte** in the source (§9). MUST be carried from demux, never reconstructed. |
|
||||
| `type` | string | MUST | Coding type:<br>`"I"`<br>`"P"`<br>`"B"`<br>_ISO/IEC 13818-2 §6.3.9; H.264/H.265 slice types collapsed to frame type._ |
|
||||
| `key` | boolean | MUST | `true` iff this picture is an intra (I) picture / parser-flagged decode-restart point (IDR / IRAP / I-picture).<br>_MPEG-2 open-GOP clean-RAP precision (`closed_gop`) is not currently distinguished — see note below._ |
|
||||
| `gop` | boolean | SHOULD | `true` iff this picture begins a GOP / coded video sequence.<br>_Omitted when the implementation does not carry a distinct GOP-boundary signal._ |
|
||||
| `pts` | integer\|null | SHOULD | Presentation timestamp in `timescale` ticks; `null` if unknown. |
|
||||
| `dts` | integer\|null | MAY | Decode timestamp in `timescale` ticks. |
|
||||
| `size` | integer | MAY | AU length in bytes; enables byte-range extraction with `src`. |
|
||||
| `recovered` | boolean | MAY | `true` iff any byte of this AU came from a retried/marginal read (§9.1).<br>_Default `false`._ |
|
||||
| codec ext | object | MAY | Codec-specific members under the codec's namespace (§8). |
|
||||
|
||||
The `type` and `key` members are **codec-agnostic** and MUST be populated for
|
||||
every codec. `type` is the I/P/B coding type the parser decoded (collapsing
|
||||
H.264/H.265 slice types to a frame type); where no per-picture coding is carried
|
||||
(audio / synthetic frames), `type` is `"I"` for a key picture else `"P"`. `key`
|
||||
is the picture's random-access flag as the codec parser sets it (IDR / IRAP /
|
||||
I-picture). A writer MUST NOT emit a degraded record (`type:"?"` or `src:null`)
|
||||
merely because a codec lacks per-picture coding info — those fallbacks are
|
||||
reserved for a field that is genuinely unavailable (e.g. provenance absent on a
|
||||
synthetic source).
|
||||
|
||||
> **Limitation (honest random-access).** `key` is set from the picture's
|
||||
> intra / decode-restart flag. The per-picture coding model this index carries
|
||||
> does **not** distinguish MPEG-2 open-GOP clean random-access points
|
||||
> (`closed_gop`) from any other I-picture, so `key` is the parser-flagged
|
||||
> decode-restart point, not a verified clean-RAP claim. A future revision MAY
|
||||
> tighten `key` for codecs/profiles that carry that signal; readers MUST NOT
|
||||
> assume present `key` precision beyond "intra / decode-restart point".
|
||||
|
||||
### 7.1 Interlace / pulldown fields
|
||||
|
||||
Codec-agnostic interlace/pulldown attributes, derived through the indexer's
|
||||
per-picture coding accessors (MPEG-2: ISO/IEC 13818-2 §6.3.10). Emitted as
|
||||
top-level members of the record, and ONLY when the codec actually measured the
|
||||
signal — an OPTIONAL member that is omitted (not defaulted) when unknown:
|
||||
|
||||
| Member | JSON type | Req | Semantics / reference |
|
||||
|---|---|---|---|
|
||||
| `field_order` | string | MAY | Display field order:<br>`"tff"` — top field first<br>`"bff"` — bottom field first<br>`"progressive"` — no field order applies<br>_Omitted when the codec did not signal it._ |
|
||||
| `progressive` | boolean | MAY | `true` iff the picture is progressive.<br>_Omitted when the codec did not signal it._ |
|
||||
| `nb_fields` | integer | MAY | Number of displayed field periods this picture occupies (the soft-telecine / 2:3 pulldown basis):<br>`1` for a single field picture<br>`2` for a normal frame<br>`3`/`4`/`6` for `repeat_first_field` pulldown per §6.3.10 |
|
||||
|
||||
Codecs that carry only a coding type (e.g. H.264 / HEVC / VC-1 through this
|
||||
pipeline) omit `field_order` and `progressive` rather than guessing a default.
|
||||
|
||||
## 8. Codec model and extensibility
|
||||
|
||||
Core record members (§7) are codec-agnostic and present for every codec.
|
||||
Codec-specific data is either (a) promoted to top-level members for a small,
|
||||
registered set per codec profile (e.g. MPEG-2 §7.1), or (b) placed under an
|
||||
`ext` object keyed by codec id for richer/optional data:
|
||||
|
||||
```json
|
||||
{
|
||||
"n": 42,
|
||||
"type": "P",
|
||||
"key": false,
|
||||
"src": {
|
||||
"sector": 17,
|
||||
"byte": 924
|
||||
},
|
||||
"ext": {
|
||||
"hevc": {
|
||||
"temporal_id": 0,
|
||||
"nal_type": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
New codecs and members are added through Appendix B (codec registry) without a
|
||||
breaking version bump, provided readers continue to ignore unknown members (§11).
|
||||
|
||||
## 9. Provenance and recovery semantics
|
||||
|
||||
`src` is **byte-exact** to the source as read. `src.sector` counts in
|
||||
`source.sector_size`-byte units; `src.byte` is the offset within that sector of
|
||||
the AU's first byte. For multi-file sources, `src.file` indexes a writer-declared
|
||||
file list. Provenance MUST be the value observed at demux time; an implementation
|
||||
MUST NOT recompute `src` by re-parsing — the point of FVI is to *carry* the truth.
|
||||
|
||||
### 9.1 Recovery
|
||||
|
||||
Because FVI is provenance-native, it can record reliability. A record with
|
||||
`"recovered":true` indicates the AU's source bytes required retry/marginal-read
|
||||
recovery. This lets downstream tools surface or quarantine pictures whose bytes
|
||||
are not byte-identical to a clean read — a capability legacy index formats lack.
|
||||
|
||||
## 10. Time model
|
||||
|
||||
All `pts`/`dts` are integers in units of `1/timescale` seconds. `pts` is
|
||||
presentation (display) time; `dts` is decode time. Records are in **coded**
|
||||
(decode) order, so `pts` is not necessarily monotonic across records (B-pictures
|
||||
reorder); `dts` is non-decreasing. Readers needing display order sort by `pts`.
|
||||
|
||||
## 11. Versioning and forward compatibility
|
||||
|
||||
- `fvi_version` is the document version; this spec defines `1`.
|
||||
- **Additive** changes (new OPTIONAL members, new registered codecs) do NOT bump
|
||||
`fvi_version`. Readers MUST ignore members they do not recognize.
|
||||
- A change that alters the meaning of an existing member or makes a new member
|
||||
REQUIRED bumps `fvi_version`.
|
||||
- A reader encountering a higher `fvi_version` than it implements SHOULD process
|
||||
the members it understands and MUST NOT reject the document solely for the
|
||||
version being higher, unless a member it relies on is absent.
|
||||
|
||||
## 12. Conformance requirements (summary)
|
||||
|
||||
A conformant **writer** MUST: emit a Header first; emit records in coded order
|
||||
with contiguous `n`; populate `src` from demux; use named/registered codec ids;
|
||||
encode one JSON value per UTF-8 LF-terminated line.
|
||||
|
||||
A conformant **reader** MUST: accept any §4–§10 document; ignore unknown members;
|
||||
not assume `picture_count`, `pts`, or `size` are present unless required above.
|
||||
|
||||
---
|
||||
|
||||
## Appendix A — JSON Schema (informative)
|
||||
|
||||
Header:
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["format", "fvi_version", "stream", "source", "timescale"],
|
||||
"properties": {
|
||||
"format": { "const": "freemkv/video-index" },
|
||||
"fvi_version": { "type": "integer", "minimum": 1 },
|
||||
"timescale": { "type": "integer", "minimum": 1 },
|
||||
"stream": { "type": "object", "required": ["codec", "width", "height", "scan"] },
|
||||
"source": { "type": "object", "required": ["medium"] }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Record:
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"required": ["n", "src", "type", "key"],
|
||||
"properties": {
|
||||
"n": { "type": "integer", "minimum": 0 },
|
||||
"type": { "enum": ["I", "P", "B"] },
|
||||
"key": { "type": "boolean" },
|
||||
"src": {
|
||||
"type": "object",
|
||||
"required": ["sector", "byte"],
|
||||
"properties": {
|
||||
"file": { "type": "integer" },
|
||||
"sector": { "type": "integer", "minimum": 0 },
|
||||
"byte": { "type": "integer", "minimum": 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Appendix B — Registered codec identifiers
|
||||
|
||||
| `codec` | Bitstream | Field profile |
|
||||
|---|---|---|
|
||||
| `mpeg2video` | ISO/IEC 13818-2 | §7.1 (field_order/progressive/nb_fields) |
|
||||
| `mpeg1video` | ISO/IEC 11172-2 | §7.1 |
|
||||
| `h264` | ISO/IEC 14496-10 | core + `ext.h264` |
|
||||
| `hevc` | ISO/IEC 23008-2 | core + `ext.hevc` |
|
||||
| `vc1` | SMPTE 421M | core |
|
||||
|
||||
## Appendix C — Normative references
|
||||
|
||||
- RFC 2119, RFC 8174 — Requirement keywords (BCP 14).
|
||||
- RFC 8259 — JSON.
|
||||
- ISO/IEC 13818-2 — MPEG-2 video (picture coding, §6.3.9–6.3.10).
|
||||
- ISO/IEC 14496-10 — H.264/AVC. ISO/IEC 23008-2 — H.265/HEVC.
|
||||
- ITU-T H.273 — Coding-independent code points (colour primaries/transfer/matrix).
|
||||
- SMPTE ST 2086 — Mastering display colour volume (HDR).
|
||||
- BCP 47 — Language tags.
|
||||
- RFC 9559 — Matroska (alignment of colour/field-order semantics).
|
||||
+50
-264
@@ -2,191 +2,45 @@
|
||||
|
||||
## Overview
|
||||
|
||||
AACS (Advanced Access Content System) is the encryption layer used by Blu-ray and UHD 4K discs to protect content. libfreemkv implements AACS decryption to enable transparent disc access.
|
||||
AACS (Advanced Access Content System) is the encryption layer used by Blu-ray
|
||||
and UHD 4K discs to protect content. libfreemkv implements AACS decryption so
|
||||
disc access is transparent to the application.
|
||||
|
||||
There are two major versions:
|
||||
|
||||
- **AACS 1.0** -- Used by standard Blu-ray discs. Relies on a custom 160-bit elliptic curve for bus authentication and AES-128 for content encryption. Processing keys and device keys can derive the media key from the disc's Media Key Block (MKB).
|
||||
|
||||
- **AACS 2.0** -- Used by UHD 4K Blu-ray discs. Adds a per-sector bus encryption layer (read_data_key) on top of the standard content encryption. Uses P-256/SHA-256 for its native handshake, though drives accept AACS 1.0 host certificates for backward compatibility.
|
||||
|
||||
Both versions use AES-128-CBC for content decryption with a fixed initialization vector. The fundamental key hierarchy is the same: a Volume Unique Key (VUK) decrypts per-title unit keys, which in turn decrypt the content stream.
|
||||
|
||||
|
||||
## Architecture
|
||||
|
||||
AACS support is split across two modules:
|
||||
|
||||
### `aacs.rs` -- Keys and Decryption
|
||||
|
||||
Handles everything related to key resolution and content decryption:
|
||||
|
||||
- KEYDB.cfg parsing (device keys, processing keys, host certificates, per-disc entries)
|
||||
- Disc hash computation (SHA-1 of `Unit_Key_RO.inf`)
|
||||
- VUK resolution chain (4 paths, described below)
|
||||
- MKB record parsing and media key derivation
|
||||
- Subset-difference tree traversal (AACS-G3 key derivation)
|
||||
- Unit_Key_RO.inf parsing and unit key decryption
|
||||
- Content Certificate parsing (AACS version detection)
|
||||
- Aligned unit decryption (AES-128-CBC)
|
||||
- Bus decryption (AACS 2.0 read_data_key layer)
|
||||
|
||||
### `aacs_handshake.rs` -- SCSI Authentication
|
||||
|
||||
Handles the drive-level SCSI authentication protocol:
|
||||
|
||||
- ECDH key agreement on the AACS 160-bit curve
|
||||
- ECDSA signing and verification
|
||||
- Bus key derivation
|
||||
- AGID management (allocate/invalidate)
|
||||
- Volume ID retrieval (encrypted with bus key, verified by AES-CMAC)
|
||||
- Read Data Key retrieval (for AACS 2.0 bus decryption)
|
||||
- AACS LA public key certificate verification
|
||||
|
||||
|
||||
## Key Resolution Chain
|
||||
|
||||
When a disc is scanned, `resolve_keys()` attempts four paths in priority order. The first path that succeeds is used.
|
||||
|
||||
### Path 1: KEYDB VUK Lookup (fastest)
|
||||
|
||||
```
|
||||
Unit_Key_RO.inf --> SHA-1 --> disc_hash --> KEYDB lookup --> VUK
|
||||
```
|
||||
|
||||
The disc hash is computed as the SHA-1 digest of the raw `Unit_Key_RO.inf` file from the disc's `/AACS/` directory. This hash is used as the lookup key in `KEYDB.cfg`. If a matching entry contains a VUK (`V` field), it is used directly.
|
||||
|
||||
This is the fast path and resolves the vast majority of discs in a well-maintained KEYDB.
|
||||
|
||||
### Path 2: KEYDB Media Key + Volume ID
|
||||
|
||||
```
|
||||
KEYDB media_key + Volume ID (from SCSI handshake) --> VUK derivation
|
||||
```
|
||||
|
||||
If the disc hash is not in the KEYDB but a KEYDB entry has a matching Volume ID (`I` field) and a media key (`M` field), the VUK is derived:
|
||||
|
||||
```
|
||||
VUK = AES-128-ECB-DECRYPT(media_key, volume_id) XOR volume_id
|
||||
```
|
||||
|
||||
Requires a successful SCSI handshake to obtain the Volume ID.
|
||||
|
||||
### Path 3: MKB + Processing Keys
|
||||
|
||||
```
|
||||
MKB (from disc) + processing_keys (from KEYDB) --> media_key --> VUK
|
||||
```
|
||||
|
||||
Processing keys are pre-computed keys that work against specific MKB versions. For each processing key, the library:
|
||||
|
||||
1. Parses the MKB to extract the Verify Media Key Record (`mk_dv`), subset-difference index, and conditional values (cvalues).
|
||||
2. Tries each processing key against each UV/cvalue pair: `mk = AES-DEC(pk, cvalue) XOR cvalue`.
|
||||
3. Validates the derived media key: `AES-ECB(mk, mk_dv)` must produce 12 leading zero bytes.
|
||||
4. Derives VUK from the validated media key and Volume ID.
|
||||
|
||||
### Path 4: MKB + Device Keys (Subset-Difference Tree)
|
||||
|
||||
```
|
||||
MKB + device_keys --> subset-difference tree traversal --> processing_key --> media_key --> VUK
|
||||
```
|
||||
|
||||
The most complex path. Each device key has an associated node number, UV value, and mask parameters that position it in the AACS subset-difference tree. The library:
|
||||
|
||||
1. Finds the subset-difference entry in the MKB that applies to the device key's node.
|
||||
2. Traverses the tree using AACS-G3 key derivation: `aesg3(key, inc) = AES-DEC(key, seed) XOR seed`, where `seed[15]` is incremented by `inc`. Each tree node produces a left child (inc=0), a processing key (inc=1), and a right child (inc=2).
|
||||
3. At each level, selects left or right based on the UV bit at the current position.
|
||||
4. The resulting processing key is validated against the MKB cvalue to derive the media key.
|
||||
5. VUK is derived from the media key and Volume ID.
|
||||
|
||||
|
||||
## Content Decryption
|
||||
|
||||
### Aligned Units
|
||||
|
||||
AACS encrypts content in aligned units of 6144 bytes (3 sectors of 2048 bytes each). The encryption flag is signaled by the copy_permission_indicator bits in byte 0 of the unit (`unit[0] & 0xC0 != 0`).
|
||||
|
||||
### Per-Unit Key Derivation
|
||||
|
||||
Each aligned unit has its own decryption key derived from the CPS unit key:
|
||||
|
||||
1. **Derive**: AES-128-ECB encrypt the first 16 bytes of the unit (plaintext TP_extra_header) with the unit key.
|
||||
2. **XOR**: XOR the encrypted result with the original 16 bytes to produce the per-unit decryption key.
|
||||
3. **Decrypt**: AES-128-CBC decrypt bytes 16 through 6143 using the per-unit key and the fixed AACS IV.
|
||||
4. **Clear flag**: Clear the encryption indicator bits (`unit[0] &= !0xC0`).
|
||||
|
||||
### Fixed IV
|
||||
|
||||
All AES-CBC operations in AACS use the same fixed initialization vector, defined in the AACS specification.
|
||||
|
||||
### Verification
|
||||
|
||||
After decryption, the library verifies correctness by checking for MPEG-TS sync bytes (0x47) at the expected 192-byte packet boundaries within the unit. Blu-ray transport stream packets are 192 bytes: 4-byte TP_extra_header followed by a 188-byte TS packet.
|
||||
|
||||
|
||||
## Bus Encryption
|
||||
|
||||
### AACS 1.0
|
||||
|
||||
Standard Blu-ray discs do not use bus encryption. Content is read directly from the disc and decrypted using the unit key.
|
||||
|
||||
### AACS 2.0
|
||||
|
||||
UHD 4K discs add a per-sector bus encryption layer. The drive encrypts data as it is read from the disc, and the host must decrypt it before applying AACS content decryption.
|
||||
|
||||
Bus encryption uses a **read_data_key** obtained during the SCSI handshake. For each 2048-byte sector within an aligned unit, bytes 16 through 2047 are AES-128-CBC encrypted with the read_data_key and the fixed AACS IV. The first 16 bytes of each sector remain plaintext.
|
||||
|
||||
The full decryption pipeline for AACS 2.0:
|
||||
|
||||
1. **Bus decrypt**: For each sector, AES-128-CBC decrypt bytes 16..2047 with the read_data_key.
|
||||
2. **Content decrypt**: Standard per-unit key derivation and AES-128-CBC decryption as described above.
|
||||
|
||||
|
||||
## SCSI Handshake
|
||||
|
||||
The AACS SCSI authentication handshake establishes a shared bus key between host and drive, then uses it to securely transfer the Volume ID and read data keys.
|
||||
|
||||
### Protocol Flow
|
||||
|
||||
1. **Invalidate AGIDs**: Send REPORT KEY with format 0x3F for AGIDs 0-3 to clear stale sessions.
|
||||
2. **Allocate AGID**: REPORT KEY format 0x00 returns a fresh Authentication Grant ID.
|
||||
3. **Send host credentials**: SEND KEY format 0x01 transmits the host nonce (20 random bytes) and host certificate (92 bytes).
|
||||
4. **Receive drive credentials**: REPORT KEY format 0x01 returns the drive nonce and drive certificate.
|
||||
5. **Receive drive key**: REPORT KEY format 0x02 returns the drive's ephemeral EC key point and ECDSA signature over `host_nonce || drive_key_point`.
|
||||
6. **Verify drive key**: The signature is verified against the drive's public key (extracted from its certificate). AACS 1.0 certificates are verified against the AACS LA public key.
|
||||
7. **Send host key**: The host generates an ephemeral key pair, signs `drive_nonce || host_key_point` with the host private key, and sends via SEND KEY format 0x02.
|
||||
8. **Compute bus key**: ECDH shared secret = `host_private_key * drive_key_point`. The bus key is the low 128 bits of the shared point's x-coordinate.
|
||||
|
||||
### Post-Authentication Reads
|
||||
|
||||
- **Volume ID**: REPORT DISC STRUCTURE format 0x80. Returns 16-byte VID encrypted with the bus key, plus an AES-CMAC MAC for integrity verification.
|
||||
- **Read Data Keys**: REPORT DISC STRUCTURE format 0x84. Returns the read_data_key and write_data_key, each AES-ECB encrypted with the bus key.
|
||||
|
||||
### Elliptic Curve
|
||||
|
||||
AACS 1.0 uses a custom 160-bit Weierstrass curve (`y^2 = x^3 + ax + b mod p`) with 20-byte field elements. The library implements full EC arithmetic: point addition, doubling, scalar multiplication, modular inverse, ECDSA sign/verify, and ECDH key agreement.
|
||||
|
||||
|
||||
## AACS 2.0 Status
|
||||
|
||||
AACS 2.0 discs are detected via the Content Certificate file (`Content000.cer` or `Content001.cer`). A certificate type byte of 0x01 indicates AACS 2.0.
|
||||
|
||||
AACS 2.0 drives are identified by their drive certificate type (0x11). These drives natively use P-256/SHA-256, but accept AACS 1.0 host certificates for backward compatibility.
|
||||
|
||||
Current implementation status:
|
||||
|
||||
- AACS 2.0 detection: **implemented** (Content Certificate parsing, drive cert type check)
|
||||
- AACS 1.0 handshake with AACS 2.0 drives: **implemented** (backward compatibility mode)
|
||||
- Full P-256 AACS 2.0 handshake: **not yet implemented** (prepared but rarely needed since drives accept AACS 1.0 host certs)
|
||||
- Bus decryption with read_data_key: **implemented**
|
||||
- Content decryption: **implemented** (same as AACS 1.0)
|
||||
|
||||
In practice, AACS 2.0 UHD discs work through the backward-compatible AACS 1.0 handshake path, with the addition of read_data_key bus decryption.
|
||||
|
||||
- **AACS 1.0** -- Used by standard Blu-ray discs.
|
||||
- **AACS 2.0 / 2.1** -- Used by UHD 4K Blu-ray discs. Adds a per-sector bus
|
||||
encryption layer on top of the standard content encryption. UHD drives accept
|
||||
AACS 1.0 host credentials for backward compatibility.
|
||||
|
||||
All versions use AES-128 for content decryption. The library reads the keys it
|
||||
needs from `keydb.cfg`, walks the disc's Media Key Block (MKB) to resolve the
|
||||
disc's key, and decrypts the content stream. AACS-encrypted discs therefore
|
||||
require a `keydb.cfg`; CSS-protected DVDs do not (see the CSS notes in the
|
||||
library docs).
|
||||
|
||||
## How it works (feature level)
|
||||
|
||||
When a disc is scanned, the library:
|
||||
|
||||
1. Reads the disc's AACS key-input files from the `/AACS/` directory.
|
||||
2. Resolves the disc's key from `keydb.cfg` — either directly from a per-disc
|
||||
entry, or by walking the MKB with the keys present in the keydb.
|
||||
3. Performs the drive-level SCSI authentication handshake needed to obtain the
|
||||
Volume ID and, for UHD, the bus-decryption key.
|
||||
4. Decrypts the content stream as titles are read.
|
||||
|
||||
A resolved key is verified against actual disc content before it is applied, so
|
||||
a stale or wrong key fails loudly rather than producing silent garbage. If no
|
||||
usable key is available for an AACS-encrypted disc, the library surfaces a
|
||||
specific error (the E70xx family) describing which part of the chain was
|
||||
missing, and a missing `keydb.cfg` surfaces as `Error::KeydbLoad` with the
|
||||
sentinel path `<no keydb in search paths>`.
|
||||
|
||||
## API Usage
|
||||
|
||||
AACS decryption is transparent to the application. The `Disc::scan()` method handles everything automatically:
|
||||
AACS decryption is transparent to the application. `Disc::scan()` handles
|
||||
everything automatically:
|
||||
|
||||
```rust
|
||||
use libfreemkv::{Drive, Disc};
|
||||
@@ -203,7 +57,6 @@ if disc.encrypted {
|
||||
if let Some(ref aacs) = disc.aacs {
|
||||
println!("AACS {}.0", aacs.version);
|
||||
println!("Key source: {}", aacs.key_source.name());
|
||||
println!("Disc hash: {}", aacs.disc_hash);
|
||||
if let Some(mkb_ver) = aacs.mkb_version {
|
||||
println!("MKB version: {}", mkb_ver);
|
||||
}
|
||||
@@ -215,108 +68,41 @@ if disc.encrypted {
|
||||
// Read content -- decryption is automatic
|
||||
let mut reader = disc.open_title(&mut session, 0).unwrap();
|
||||
while let Some(unit) = reader.read_unit().unwrap() {
|
||||
// unit is 6144 bytes of decrypted content
|
||||
// decrypted content
|
||||
}
|
||||
```
|
||||
|
||||
The application never touches keys, never calls decryption functions, and never manages handshakes. All of that is internal to `Disc::scan()` and `ContentReader::read_unit()`.
|
||||
The application never touches keys, never calls decryption functions, and never
|
||||
manages handshakes. All of that is internal to `Disc::scan()` and the content
|
||||
reader.
|
||||
|
||||
### KEYDB Location
|
||||
|
||||
`ScanOptions` controls where the KEYDB is loaded from. If no explicit path is set, the library checks:
|
||||
|
||||
1. `~/.config/aacs/KEYDB.cfg`
|
||||
2. `/etc/aacs/KEYDB.cfg`
|
||||
|
||||
To specify an explicit path:
|
||||
`ScanOptions` controls where the keydb is loaded from. If no explicit path is
|
||||
set, the library checks the standard config locations. To specify an explicit
|
||||
path:
|
||||
|
||||
```rust
|
||||
let opts = ScanOptions::with_keydb("/path/to/KEYDB.cfg");
|
||||
let opts = ScanOptions::with_keydb("/path/to/keydb.cfg");
|
||||
let disc = Disc::scan(&mut session, &opts).unwrap();
|
||||
```
|
||||
|
||||
### AacsState
|
||||
|
||||
After a successful scan, `disc.aacs` contains an `AacsState` with:
|
||||
After a successful scan, `disc.aacs` contains an `AacsState`:
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `version` | `u8` | AACS version (1 or 2) |
|
||||
| `bus_encryption` | `bool` | Whether bus encryption is active |
|
||||
| `mkb_version` | `Option<u32>` | MKB version from disc |
|
||||
| `disc_hash` | `String` | SHA-1 of Unit_Key_RO.inf (hex with 0x prefix) |
|
||||
| `key_source` | `KeySource` | How keys were resolved |
|
||||
| `vuk` | `[u8; 16]` | Volume Unique Key |
|
||||
| `unit_keys` | `Vec<(u32, [u8; 16])>` | Decrypted unit keys (CPS unit number, key) |
|
||||
| `read_data_key` | `Option<[u8; 16]>` | AACS 2.0 bus decryption key |
|
||||
| `volume_id` | `[u8; 16]` | Volume ID from SCSI handshake |
|
||||
| `disc_hash` | `String` | Identifier for the disc's key-input files |
|
||||
| `key_source` | `KeySource` | How the disc's key was resolved |
|
||||
|
||||
### KeySource
|
||||
## keydb.cfg
|
||||
|
||||
| Variant | Description |
|
||||
|---------|-------------|
|
||||
| `KeyDb` | VUK found directly in KEYDB by disc hash |
|
||||
| `KeyDbDerived` | Media key + Volume ID from KEYDB, VUK derived |
|
||||
| `ProcessingKey` | MKB + processing keys from KEYDB |
|
||||
| `DeviceKey` | MKB + device keys, subset-difference tree traversal |
|
||||
|
||||
|
||||
## KEYDB.cfg Format Reference
|
||||
|
||||
The KEYDB.cfg file contains all cryptographic material needed for AACS decryption. Lines starting with `;` or `#` are comments.
|
||||
|
||||
### Device Keys
|
||||
|
||||
```
|
||||
| DK | DEVICE_KEY 0x<key> | DEVICE_NODE 0x<node> | KEY_UV 0x<uv> | KEY_U_MASK_SHIFT 0x<shift>
|
||||
```
|
||||
|
||||
- `key`: 16-byte AES device key (hex)
|
||||
- `node`: Device node number in the subset-difference tree (hex)
|
||||
- `uv`: UV value for tree positioning (hex)
|
||||
- `shift`: U mask shift value (hex)
|
||||
|
||||
### Processing Keys
|
||||
|
||||
```
|
||||
| PK | 0x<key>
|
||||
```
|
||||
|
||||
- `key`: 16-byte pre-computed processing key (hex)
|
||||
|
||||
### Host Certificate
|
||||
|
||||
```
|
||||
| HC | HOST_PRIV_KEY 0x<privkey> | HOST_CERT 0x<cert>
|
||||
```
|
||||
|
||||
- `privkey`: 20-byte ECDSA private key (hex)
|
||||
- `cert`: 92-byte AACS host certificate (hex)
|
||||
|
||||
The host certificate is used for SCSI authentication. It contains the host's public key and is signed by the AACS Licensing Administrator.
|
||||
|
||||
### Disc Entries
|
||||
|
||||
```
|
||||
0x<disc_hash> = <title> | D | <date> | M | 0x<media_key> | I | 0x<disc_id> | V | 0x<vuk> | U | <unit_keys>
|
||||
```
|
||||
|
||||
- `disc_hash`: 20-byte SHA-1 of Unit_Key_RO.inf (hex)
|
||||
- `title`: Human-readable disc title
|
||||
- `D`: Date tag, followed by release/rip date
|
||||
- `M`: Media key tag, followed by 16-byte media key (hex)
|
||||
- `I`: Disc ID tag, followed by 16-byte Volume ID (hex)
|
||||
- `V`: VUK tag, followed by 16-byte Volume Unique Key (hex)
|
||||
- `U`: Unit keys tag, followed by space-separated `<unit_num>-0x<key>` pairs
|
||||
|
||||
All fields after the title are optional. A minimal entry needs only the disc hash and VUK:
|
||||
|
||||
```
|
||||
0x<disc_hash> = <title> | V | 0x<vuk>
|
||||
```
|
||||
|
||||
Inline comments are supported with `;`:
|
||||
|
||||
```
|
||||
0x<disc_hash> = <title> | V | 0x<vuk> ; MKBv77
|
||||
```
|
||||
`keydb.cfg` is the single source of AACS key material. It is a text file (lines
|
||||
starting with `;` or `#` are comments) holding the host credentials and per-disc
|
||||
entries the library uses to resolve a disc. autorip can auto-download and
|
||||
refresh it from a configured URL. The library does not ship any AACS keys
|
||||
compiled into the binary.
|
||||
|
||||
+45
-44
@@ -44,35 +44,30 @@ while let Ok(Some(frame)) = input.read() {
|
||||
output.finish()?;
|
||||
```
|
||||
|
||||
The `pes::Stream` trait:
|
||||
The `FrameSource` and `FrameSink` traits — direction is type-checked, so
|
||||
calling `read()` on a write-only sink (or `write()` on a read-only source)
|
||||
is a compile error rather than a runtime fault:
|
||||
|
||||
```rust
|
||||
pub trait Stream {
|
||||
fn read(&mut self) -> io::Result<Option<PesFrame>>;
|
||||
fn write(&mut self, frame: &PesFrame) -> io::Result<()>;
|
||||
fn finish(&mut self) -> io::Result<()>;
|
||||
pub trait FrameSource: Send {
|
||||
fn read(&mut self) -> Result<Option<PesFrame>, Error>;
|
||||
fn info(&self) -> &DiscTitle;
|
||||
fn codec_private(&self, track: usize) -> Option<Vec<u8>>;
|
||||
fn headers_ready(&self) -> bool;
|
||||
fn codec_private(&self, track: usize) -> Option<Vec<u8>> { None }
|
||||
fn headers_ready(&self) -> bool { true }
|
||||
}
|
||||
```
|
||||
|
||||
## IOStream (byte-level API)
|
||||
|
||||
For raw byte copies (disc→ISO, resume, benchmarks). Lower level than PES.
|
||||
|
||||
```rust
|
||||
let opts = InputOptions::default();
|
||||
let mut input = open_input("iso://Disc.iso", &opts)?;
|
||||
let mut output = open_output("mkv://Movie.mkv", input.info())?;
|
||||
io::copy(&mut *input, &mut *output)?;
|
||||
output.finish()?;
|
||||
pub trait FrameSink: Send {
|
||||
fn write(&mut self, frame: &PesFrame) -> Result<(), Error>;
|
||||
fn finish(self: Box<Self>) -> Result<(), Error>;
|
||||
fn info(&self) -> &DiscTitle;
|
||||
}
|
||||
```
|
||||
|
||||
## Streams
|
||||
|
||||
All streams implement `IOStream` (byte-level) and/or `pes::Stream` (frame-level).
|
||||
URL-based resolvers open any stream by string.
|
||||
All streams implement `FrameSource` (read) and/or `FrameSink` (write); the
|
||||
directional split prevents runtime "wrong-direction" errors. URL-based
|
||||
resolvers open any stream by string.
|
||||
|
||||
| Stream | Input | Output | URL | Transport |
|
||||
|--------|-------|--------|-----|-----------|
|
||||
@@ -87,21 +82,14 @@ URL-based resolvers open any stream by string.
|
||||
All URLs require a `scheme://path` format. Bare paths are rejected.
|
||||
|
||||
```rust
|
||||
// PES pipeline (frame-level)
|
||||
// PES pipeline (frame-level) — input() returns Box<dyn FrameSource>,
|
||||
// output() returns Box<dyn FrameSink>.
|
||||
let input = libfreemkv::input("disc:///dev/sg4", &opts)?; // DiscStream
|
||||
let input = libfreemkv::input("iso://Dune.iso", &opts)?; // IsoStream
|
||||
let output = libfreemkv::output("mkv://Dune.mkv", &title)?; // MkvOutputStream
|
||||
let output = libfreemkv::output("m2ts://Dune.m2ts", &title)?; // M2tsOutputStream
|
||||
let output = libfreemkv::output("network://10.0.0.1:9000", &title)?; // NetworkOutputStream
|
||||
let input = libfreemkv::input("iso://Movie.iso", &opts)?; // IsoStream
|
||||
let output = libfreemkv::output("mkv://Movie.mkv", &title)?; // MkvOutputStream
|
||||
let output = libfreemkv::output("m2ts://Movie.m2ts", &title)?; // M2tsOutputStream
|
||||
let output = libfreemkv::output("network://192.0.2.10:9000", &title)?; // NetworkOutputStream
|
||||
let output = libfreemkv::output("null://", &title)?; // NullOutputStream
|
||||
|
||||
// IOStream (byte-level)
|
||||
let input = open_input("disc://", &opts)?; // DiscStream
|
||||
let input = open_input("iso://Dune.iso", &opts)?; // IsoStream
|
||||
let output = open_output("iso://Copy.iso", &meta)?; // IsoStream (write)
|
||||
let output = open_output("mkv://Dune.mkv", &meta)?; // MkvStream
|
||||
let output = open_output("m2ts://Dune.m2ts", &meta)?; // M2tsStream
|
||||
let output = open_output("null://", &meta)?; // NullStream
|
||||
```
|
||||
|
||||
### FMKV Metadata Header
|
||||
@@ -175,20 +163,34 @@ libfreemkv/src/
|
||||
├── lib.rs Public exports
|
||||
├── error.rs Error codes (no English)
|
||||
├── event.rs Event types for callbacks
|
||||
├── halt.rs Halt cancellation token (Arc<AtomicBool> wrapper)
|
||||
├── io/ Pipeline + WritebackFile primitives
|
||||
│ ├── mod.rs Re-exports WritebackFile, Pipeline, Sink, Flow
|
||||
│ ├── pipeline.rs Generic Pipeline<I, R> + Sink trait
|
||||
│ ├── writeback_file.rs WritebackFile (was crate::io::Writer)
|
||||
│ └── writeback.rs sync_file_range pipeline
|
||||
├── drive/ Drive (open, init, single-shot read)
|
||||
│ ├── mod.rs Drive struct, init, read (single-shot), reset, eject
|
||||
│ ├── capture.rs Drive profile capture for contribution
|
||||
│ ├── capture.rs Raw drive SCSI capture (INQUIRY/GET_CONFIG) for contribution
|
||||
│ ├── linux.rs Linux drive discovery
|
||||
│ ├── macos.rs macOS drive discovery
|
||||
│ └── windows.rs Windows drive discovery
|
||||
├── disc/ Disc (scan, titles, AACS setup)
|
||||
├── disc/ Disc (scan, titles, AACS setup, sweep, patch)
|
||||
│ ├── mod.rs Disc struct, scan, titles, formats; Disc::copy + Disc::sweep (Pass 1)
|
||||
│ ├── sweep.rs Pass 1 internal helpers (pub(super))
|
||||
│ ├── patch.rs Disc::patch (Pass N retry over mapfile)
|
||||
│ ├── mapfile.rs ddrescue-format mapfile
|
||||
│ └── read_error.rs ReadCtx / ReadAction state machine
|
||||
├── scsi/ SCSI transport (Linux SG_IO, macOS IOKit, Windows SPTI)
|
||||
├── platform/ Drive unlock (MT1959 A/B)
|
||||
├── unlock.rs Unlocker trait + registry (pluggable unlock seam)
|
||||
├── aacs/ AACS decryption (handshake, keys, keydb, decrypt)
|
||||
├── css/ DVD CSS cipher
|
||||
├── decrypt.rs Unified decrypt dispatcher (AACS/CSS/None)
|
||||
├── pes.rs PES frame types, Stream trait
|
||||
├── sector.rs SectorReader trait
|
||||
├── pes.rs PES frame types, FrameSource / FrameSink traits
|
||||
├── sector/ Sector I/O
|
||||
│ ├── mod.rs SectorSource, SectorSink traits
|
||||
│ ├── file.rs FileSectorSource, FileSectorSink (ISO-backed)
|
||||
│ └── decrypting.rs DecryptingSectorSource decorator
|
||||
├── udf.rs UDF 2.50 filesystem parser
|
||||
├── mpls.rs MPLS playlist parser
|
||||
├── clpi.rs CLPI clip info parser
|
||||
@@ -196,18 +198,17 @@ libfreemkv/src/
|
||||
├── labels/ BD-J label extraction (5 format parsers)
|
||||
├── keydb.rs KEYDB download, parse, save
|
||||
├── identity.rs DriveId from INQUIRY
|
||||
├── profile.rs Bundled drive profiles
|
||||
├── speed.rs DriveSpeed enum
|
||||
├── mux/
|
||||
│ ├── mod.rs IOStream trait, public exports
|
||||
│ ├── resolve.rs URL parser + open_input/open_output + input/output
|
||||
│ ├── mod.rs Public mux exports
|
||||
│ ├── resolve.rs URL parser + input/output (Box<dyn FrameSource/Sink>)
|
||||
│ ├── meta.rs FMKV header format
|
||||
│ ├── disc.rs DiscStream (optical drive → PES)
|
||||
│ ├── iso.rs IsoStream (ISO image read/write)
|
||||
│ ├── iso.rs IsoStream (ISO image read)
|
||||
│ ├── isowriter.rs ISO image writer (UDF, AVDP, multi-extent)
|
||||
│ ├── mkvstream.rs MkvStream (bidirectional Matroska, IOStream)
|
||||
│ ├── mkvstream.rs MkvStream (bidirectional Matroska)
|
||||
│ ├── mkvout.rs MkvOutputStream (PES → MKV)
|
||||
│ ├── m2ts.rs M2tsStream (BD-TS, IOStream)
|
||||
│ ├── m2ts.rs M2tsStream (BD-TS)
|
||||
│ ├── pesout.rs PES output streams (M2ts, Network, Stdio, Null)
|
||||
│ ├── network.rs NetworkStream (TCP + FMKV header)
|
||||
│ ├── stdio.rs StdioStream (stdin/stdout pipe)
|
||||
|
||||
+32
-26
@@ -1,8 +1,10 @@
|
||||
# libfreemkv Architecture
|
||||
|
||||
Open source optical drive access library for 4K UHD Blu-ray, Blu-ray, and DVD.
|
||||
Rust library with no external dependencies at runtime -- profiles are bundled,
|
||||
AACS keys are derived internally, and all SCSI communication is handled in-process.
|
||||
Rust library with profiles bundled and all SCSI communication handled in-process.
|
||||
AACS decryption requires an external `keydb.cfg` (default
|
||||
`~/.config/freemkv/keydb.cfg`) — the derivation math is internal, but no AACS key
|
||||
material is compiled in; DVD CSS player keys are the only compiled-in keys.
|
||||
|
||||
**Repository:** <https://github.com/freemkv/libfreemkv>
|
||||
**License:** AGPL-3.0-only
|
||||
@@ -15,9 +17,10 @@ AACS keys are derived internally, and all SCSI communication is handled in-proce
|
||||
format handling live in the library. CLI binaries are thin wrappers that call
|
||||
`Drive::open()` and `Disc::scan()`.
|
||||
|
||||
2. **No external files.** Bundled drive profiles are compiled into the binary via
|
||||
`include_str!`. No configuration directory, no runtime file lookups for drive
|
||||
support.
|
||||
2. **Firmware-clean core.** libfreemkv ships no firmware, no unlock CDBs, and no
|
||||
drive profiles. Drive-unlock logic is plugged in by an external crate through
|
||||
the `Unlocker` trait + registry (`register_unlocker`); without one the library
|
||||
still rips via the host-certificate AACS handshake.
|
||||
|
||||
3. **Transparent AACS.** The `ContentReader` decrypts on the fly when keys are
|
||||
available. Callers read cleartext sectors without knowing whether the disc
|
||||
@@ -41,11 +44,9 @@ AACS keys are derived internally, and all SCSI communication is handled in-proce
|
||||
libfreemkv (lib.rs)
|
||||
│
|
||||
├── Drive Access
|
||||
│ ├── drive Drive — open, identify, init, unlock, single-shot read
|
||||
│ ├── drive Drive — open, identify, init, single-shot read
|
||||
│ ├── scsi ScsiTransport trait + platform backends (sg async, IOKit, SPTI)
|
||||
│ ├── platform/ Platform trait — per-chipset command handlers
|
||||
│ │ └── mt1959 MediaTek MT1959 driver (LG, ASUS, HP)
|
||||
│ ├── profile DriveProfile loading, matching, bundled JSON
|
||||
│ ├── unlock Unlocker trait + registry — the pluggable unlock seam
|
||||
│ ├── identity DriveId from INQUIRY + GET_CONFIG 010C
|
||||
│ ├── speed DriveSpeed enum, SET CD SPEED CDB builder
|
||||
│ └── event Event system for drive status callbacks
|
||||
@@ -65,13 +66,16 @@ libfreemkv (lib.rs)
|
||||
│
|
||||
├── Streaming
|
||||
│ ├── mux/ Stream implementations (Disc, ISO, MKV, M2TS, Network, Stdio, Null)
|
||||
│ ├── pes PES frame types, Stream trait (read/write frames)
|
||||
│ └── sector SectorReader trait — abstracts disc vs ISO vs file
|
||||
│ ├── pes PES frame types; the unified pes::Stream (PesStream) read/write trait
|
||||
│ └── sector/ SectorSource / SectorSink traits, FileSector{Source,Sink}, DecryptingSectorSource
|
||||
│
|
||||
├── I/O Primitives
|
||||
│ ├── halt Halt cancellation token (one Arc<AtomicBool>, cloneable)
|
||||
│ └── io/ Pipeline<I, R> + Sink trait + WritebackFile (bounded-cache writer)
|
||||
│
|
||||
├── Support
|
||||
│ ├── keydb KEYDB.cfg download, parse, verify, save
|
||||
│ ├── error Error enum with numeric codes E1000-E8000
|
||||
│ └── profile Bundled drive profiles
|
||||
│ └── error Error enum with numeric codes E1000-E8000
|
||||
│
|
||||
└── lib.rs Public API re-exports
|
||||
```
|
||||
@@ -85,13 +89,12 @@ Drive::open(Path::new("/dev/sg4"))
|
||||
│
|
||||
├─ scsi::open() Open /dev/sg4 (async write/poll/read)
|
||||
├─ DriveId::from_drive() INQUIRY + GET_CONFIG 010C
|
||||
├─ profile::find_by_drive_id() Match against bundled profiles
|
||||
├─ Platform::new() Instantiate chipset driver (Mt1959)
|
||||
└─ Drive ready for init/unlock/read
|
||||
└─ Drive ready for init/read
|
||||
```
|
||||
|
||||
After open:
|
||||
- `init()` -- unlock + firmware upload + speed calibration
|
||||
- `init()` -- routes to the matching registered unlocker (if any); otherwise
|
||||
a no-op and the cert handshake carries the disc
|
||||
- `probe_disc()` -- probe disc surface for optimal speeds
|
||||
- `read(lba, count, buf, recovery)` -- single-shot read; `recovery` only selects the per-CDB timeout (1.5 s vs. 30 s)
|
||||
- `wait_ready()` -- wait for disc insertion
|
||||
@@ -191,17 +194,20 @@ implementing `execute()` for that OS and wiring it into `scsi::open()`.
|
||||
|
||||
---
|
||||
|
||||
## Chipset Support
|
||||
## Drive Unlock
|
||||
|
||||
| Chipset | Drives | Status |
|
||||
|---------|--------|--------|
|
||||
| MediaTek MT1959 | LG, ASUS, HP | Supported (bundled profiles) |
|
||||
| Renesas RS8xxx/RS9xxx | Pioneer, some HL-DT-ST | Planned |
|
||||
libfreemkv carries no drive-unlock mechanism. The `Unlocker` trait + registry
|
||||
(`src/unlock.rs`) is the seam: an external crate implements `Unlocker` and
|
||||
registers it once via `register_unlocker(...)`. At drive-prep the registry is
|
||||
walked in order and the first unlocker whose `matches()` is true is asked to
|
||||
`unlock_drive()` over the raw `ScsiTransport`. If none match, the drive is left
|
||||
untouched and the host-certificate AACS handshake carries the disc.
|
||||
|
||||
The `Platform` trait abstracts chipset-specific commands. Each chipset implements
|
||||
handlers (unlock, config, register, calibrate, keepalive, status, probe,
|
||||
read_sectors, timing). All handlers are accessed via SCSI READ BUFFER with
|
||||
chipset-specific mode and buffer ID bytes.
|
||||
The implementor owns everything firmware-specific — drive profiles, vendor CDBs,
|
||||
variant logic. Concrete unlockers live in the separate
|
||||
**[freemkv-unlock](https://github.com/freemkv/freemkv-unlock)** repository, never
|
||||
in libfreemkv. See [`drive-access.md`](drive-access.md#drive-unlock-seam) for the
|
||||
trait definition and routing.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+1
-1
@@ -189,7 +189,7 @@ The full ripping pipeline chains three parsers:
|
||||
2. **CLPI** converts those timestamps to SPN ranges, then to sector extents.
|
||||
3. **UDF** provides the file's starting LBA on disc for absolute sector addressing.
|
||||
|
||||
The `Disc::scan()` method in `src/disc.rs` orchestrates this: for each play item in each playlist, it loads the corresponding CLPI, calls `get_extents()` with the play item's in/out times, and collects the resulting sector ranges into the title's extent list.
|
||||
The `Disc::scan()` method in `src/disc/mod.rs` orchestrates this: for each play item in each playlist, it loads the corresponding CLPI, calls `get_extents()` with the play item's in/out times, and collects the resulting sector ranges into the title's extent list.
|
||||
|
||||
## References
|
||||
|
||||
|
||||
+14
-10
@@ -10,14 +10,14 @@ Insert disc
|
||||
│
|
||||
▼
|
||||
1. Open drive (drive/mod.rs)
|
||||
│ INQUIRY → identify drive
|
||||
│ Match bundled profile → chipset, unlock parameters
|
||||
│ INQUIRY → identify drive (DriveId)
|
||||
│
|
||||
▼
|
||||
2. Init drive (drive/mod.rs → platform/mt1959)
|
||||
│ Firmware upload (if needed, 10s recovery wait)
|
||||
│ Unlock → vendor-specific command activates raw read mode
|
||||
│ Speed calibration → probe_disc()
|
||||
2. Init drive (drive/mod.rs → unlock seam)
|
||||
│ Walk the registered-unlocker registry; first match unlocks the drive
|
||||
│ (firmware/vendor handshakes are the unlocker's own business)
|
||||
│ No match → drive untouched; host-cert AACS handshake carries the disc
|
||||
│ Speed control → probe_disc()
|
||||
│
|
||||
▼
|
||||
3. AACS handshake (aacs/handshake.rs) — optional
|
||||
@@ -97,7 +97,9 @@ drive.probe_disc()?;
|
||||
// Scan disc (UDF + playlists + AACS — all automatic)
|
||||
let disc = Disc::scan(&mut drive, &ScanOptions::default())?;
|
||||
|
||||
// Stream pipeline — PES frames from any source to any output
|
||||
// Stream pipeline — PES frames from any source to any output.
|
||||
// input() returns Box<dyn FrameSource>, output() returns Box<dyn FrameSink>;
|
||||
// direction is type-checked, so calling .write() on an input is a compile error.
|
||||
let opts = InputOptions::default();
|
||||
let mut input = libfreemkv::input("disc:///dev/sg4", &opts)?;
|
||||
let title = input.info().clone();
|
||||
@@ -121,11 +123,13 @@ output.finish()?;
|
||||
| aacs/ | [aacs.md](aacs.md) | Key resolution + content decrypt + bus handshake |
|
||||
| css/ | -- | DVD CSS cipher |
|
||||
| decrypt.rs | -- | Unified decrypt dispatcher (AACS/CSS/None) |
|
||||
| disc/ | -- | High-level scan + read API |
|
||||
| disc/ | [rip-recovery.md](rip-recovery.md) | Disc::scan + Disc::sweep + Disc::patch + mapfile |
|
||||
| labels/ | -- | BD-J stream labels (5 format parsers) |
|
||||
| mux/ | -- | Stream implementations (7 stream types) |
|
||||
| pes.rs | -- | PES frame types + Stream trait |
|
||||
| sector.rs | -- | SectorReader trait |
|
||||
| pes.rs | -- | PES frame types + FrameSource / FrameSink traits |
|
||||
| sector/ | -- | SectorSource / SectorSink + DecryptingSectorSource decorator |
|
||||
| io/ | -- | Pipeline<I, R> + Sink trait + WritebackFile |
|
||||
| halt.rs | -- | Halt cancellation token |
|
||||
| keydb.rs | -- | KEYDB download, parse, save |
|
||||
| error.rs | -- | Error codes (E1xxx-E8xxx) |
|
||||
| event.rs | -- | Drive event system |
|
||||
|
||||
+65
-91
@@ -7,8 +7,9 @@ optical drives.
|
||||
|
||||
## Drive
|
||||
|
||||
`Drive` is the primary API. It owns the SCSI transport, the matched
|
||||
drive profile, and the chipset-specific platform driver.
|
||||
`Drive` is the primary API. It owns the SCSI transport and the drive
|
||||
identity (`DriveId`); any drive-specific unlock logic lives behind the
|
||||
pluggable [unlock seam](#drive-unlock-seam), not in `Drive` itself.
|
||||
|
||||
### Opening a Drive
|
||||
|
||||
@@ -16,15 +17,16 @@ drive profile, and the chipset-specific platform driver.
|
||||
let mut drive = Drive::open(Path::new("/dev/sg4"))?;
|
||||
```
|
||||
|
||||
`open()` performs: open device → send INQUIRY → match profile → instantiate
|
||||
platform driver. The drive is ready for `wait_ready()` and `init()`.
|
||||
`open()` performs: open device → send INQUIRY → build `DriveId`. The drive
|
||||
is ready for `wait_ready()` and `init()` (which routes through the unlock
|
||||
seam).
|
||||
|
||||
### Drive Operations
|
||||
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| `wait_ready()` | Wait for disc insertion (30s timeout, TUR polling) |
|
||||
| `init()` | Firmware upload + unlock + speed calibration |
|
||||
| `init()` | Route to the matching registered unlocker (if any), then prepare for reads |
|
||||
| `probe_disc()` | Probe disc surface for optimal speeds |
|
||||
| `read(lba, count, buf, recovery)` | Read sectors. Single-shot — no inline retries or reset. |
|
||||
| `reset()` | Eject-cycle escape hatch. Caller-invoked only; not on the read path. |
|
||||
@@ -32,17 +34,21 @@ platform driver. The drive is ready for `wait_ready()` and `init()`.
|
||||
| `unlock_tray()` | Allow tray ejection (also runs on Drop) |
|
||||
| `eject()` | Eject disc tray |
|
||||
| `drive_status()` | Query physical state (disc present, tray open, etc.) |
|
||||
| `has_profile()` | Whether a bundled profile matched |
|
||||
| `has_profile()` | Whether a registered unlocker matches this drive |
|
||||
| `close()` | Consume Drive, cleanup (also runs via Drop) |
|
||||
|
||||
### init() Sequence
|
||||
|
||||
`init()` orchestrates the full drive unlock:
|
||||
`init()` routes drive preparation through the unlock seam:
|
||||
|
||||
1. Platform driver `run_init()` — sends vendor-specific SCSI commands
|
||||
2. If firmware upload needed: upload, wait 10s for drive reset, retry
|
||||
3. Speed calibration after unlock
|
||||
4. Max 3 attempts before giving up
|
||||
1. Walk the registered-unlocker registry; the first whose `matches()` is true
|
||||
is asked to `unlock_drive()` over the raw transport.
|
||||
2. Whatever that unlocker needs (firmware upload, vendor handshakes, retries)
|
||||
is the unlocker's own business — libfreemkv only forwards the transport.
|
||||
3. If no unlocker matches, the drive is left untouched and the library uses
|
||||
the host-certificate AACS handshake.
|
||||
|
||||
See [Drive Unlock Seam](#drive-unlock-seam) for the trait and registry.
|
||||
|
||||
### read() — single-shot
|
||||
|
||||
@@ -52,8 +58,8 @@ selects the per-CDB timeout:
|
||||
|
||||
| `recovery` | Timeout | Used by |
|
||||
|------------|----------|------------------------------------------|
|
||||
| `false` | 1.5 s | `Disc::copy` fast skip-forward sweep, `DiscStream::fill_extents` |
|
||||
| `true` | 30 s | `Disc::patch` multi-pass over the mapfile |
|
||||
| `false` | 1.5 s | `Disc::sweep` fast skip-forward pass, `DiscStream::fill_extents` |
|
||||
| `true` | 30 s | `Disc::patch` retry pass over the mapfile |
|
||||
|
||||
On any SCSI failure or timeout, `read` returns `Err(DiscRead)` immediately.
|
||||
There are no inline retries, no SCSI reset, no Phase 1/2/3 escalation.
|
||||
@@ -164,100 +170,68 @@ date for drives where Feature 010C is unavailable.
|
||||
|
||||
---
|
||||
|
||||
## Drive Profiles
|
||||
## Drive Unlock Seam
|
||||
|
||||
Profiles are JSON objects compiled into the binary (`profiles.json`).
|
||||
Each profile contains:
|
||||
|
||||
| Field | Purpose |
|
||||
|-------|---------|
|
||||
| `vendor_id`, `product_revision`, `vendor_specific`, `firmware_date` | Matching fields |
|
||||
| `chipset` | `"mediatek"` or `"renesas"` |
|
||||
| `unlock_mode`, `unlock_buf_id` | READ BUFFER CDB parameters |
|
||||
| `signature` | Expected 4-byte response signature |
|
||||
| `unlock_cdb` | Pre-built unlock CDB (hex-encoded) |
|
||||
| `register_offsets` | Offsets for hardware register reads |
|
||||
| `capabilities` | Feature flags: `bd_raw_read`, `dvd_all_regions`, etc. |
|
||||
|
||||
Loading:
|
||||
libfreemkv ships **no firmware, no unlock CDBs, and no drive profiles.** It
|
||||
knows only the *seam*, never the *mechanism*. The seam is the `Unlocker`
|
||||
trait plus a small process-wide registry (`src/unlock.rs`):
|
||||
|
||||
```rust
|
||||
// Bundled (compiled-in) -- no file I/O
|
||||
let profiles = profile::load_bundled()?;
|
||||
pub trait Unlocker: Send + Sync {
|
||||
/// Stable, language-neutral identifier (logged).
|
||||
fn name(&self) -> &str;
|
||||
|
||||
// External file
|
||||
let profiles = profile::load_all(Path::new("/path/to/profiles.json"))?;
|
||||
/// True if this unlocker handles the given drive.
|
||||
fn matches(&self, id: &DriveId) -> bool;
|
||||
|
||||
/// Put the drive into extended-access mode. The one required capability.
|
||||
fn unlock_drive(&self, scsi: &mut dyn ScsiTransport, id: &DriveId) -> Result<()>;
|
||||
|
||||
/// Read the disc Volume ID via the drive's OEM path. Default: no-op.
|
||||
fn read_volume_id(&self, _scsi: &mut dyn ScsiTransport, _id: &DriveId)
|
||||
-> Result<Option<[u8; 16]>> { Ok(None) }
|
||||
|
||||
/// Raise the drive to its maximum read speed. Default: no-op.
|
||||
fn set_max_read_speed(&self, _scsi: &mut dyn ScsiTransport, _id: &DriveId)
|
||||
-> Result<()> { Ok(()) }
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
An unlocker is supplied by an **external crate** and registered once at
|
||||
process start:
|
||||
|
||||
## Chipsets
|
||||
```rust
|
||||
libfreemkv::register_unlocker(Box::new(some_unlocker::Plugin::new()));
|
||||
```
|
||||
|
||||
### MediaTek MT1959
|
||||
The implementor owns everything about *how* a particular drive family is
|
||||
driven — drive identification against its own profile database, firmware
|
||||
upload, vendor CDBs, variant logic. libfreemkv only hands over the raw
|
||||
`ScsiTransport` and the `DriveId`.
|
||||
|
||||
Covers all LG, ASUS, and HP optical drives. Two sub-variants share identical
|
||||
logic with different SCSI parameters:
|
||||
### Routing
|
||||
|
||||
| Variant | READ BUFFER mode | Buffer ID |
|
||||
|---------|------------------|-----------|
|
||||
| MT1959-A | 0x01 | 0x44 |
|
||||
| MT1959-B | 0x02 | 0x77 |
|
||||
At drive-prep the registry is walked in registration order; the first
|
||||
unlocker whose `matches()` returns true is asked to `unlock_drive()` (and,
|
||||
when needed, `read_volume_id()` / `set_max_read_speed()`). If no unlocker
|
||||
matches, the drive is left untouched and the library falls back to the
|
||||
standard host-certificate AACS handshake (the "OEM route"). The
|
||||
`register_unlocker(...)` line is the entire plug: drop it (and the unlocker
|
||||
crate) and libfreemkv still compiles and rips via the cert handshake.
|
||||
|
||||
The Platform trait maps to command handlers:
|
||||
|
||||
| Handler | Function | Description |
|
||||
|---------|----------|-------------|
|
||||
| 0 | `unlock()` | Send READ BUFFER, verify signature + verification bytes |
|
||||
| 1 | `read_config()` | Read 1888-byte configuration block + 4-byte status |
|
||||
| 2-3 | `read_register()` | Read hardware registers at profile-specified offsets |
|
||||
| 4 | `calibrate()` | Probe disc surface, build 64-entry speed table |
|
||||
| 5 | `keepalive()` | Periodic session maintenance |
|
||||
| 6 | `status()` | Query current mode and feature flags |
|
||||
| 7 | `probe()` | Generic READ BUFFER with dynamic parameters |
|
||||
| 8 | `read_sectors()` | Speed lookup + SET CD SPEED + READ(10) with flag 0x08 |
|
||||
| 9 | `timing()` | Timing calibration |
|
||||
|
||||
### Renesas (Planned)
|
||||
|
||||
RS8xxx/RS9xxx chipsets used in Pioneer and some HL-DT-ST drives.
|
||||
Currently returns `Error::UnsupportedDrive` when a Renesas profile is matched.
|
||||
|
||||
---
|
||||
|
||||
## Why Unlock Is Needed
|
||||
|
||||
Optical drive firmware restricts what applications can read from disc. Without
|
||||
unlock:
|
||||
|
||||
- **READ(10) works for unencrypted filesystem data.** UDF structures, MPLS
|
||||
playlists, and CLPI clip info are readable without unlock. Standard READ(10)
|
||||
works on any drive.
|
||||
|
||||
- **READ(10) fails for encrypted content sectors.** The drive firmware returns
|
||||
SCSI errors (sense key 0x05, illegal request) when an application attempts to
|
||||
read sectors containing encrypted m2ts content without prior AACS
|
||||
authentication via the bus key.
|
||||
|
||||
- **Raw mode bypasses firmware restrictions.** After unlock, the drive accepts
|
||||
READ(10) with the raw read flag (CDB byte 1 = 0x08) for all sectors,
|
||||
regardless of encryption status.
|
||||
|
||||
### AACS Before Unlock
|
||||
|
||||
AACS bus authentication uses standard MMC REPORT KEY / SEND KEY commands.
|
||||
On some drives these must execute before unlock. The `Disc::scan()` handles
|
||||
this internally — it manages the handshake/unlock ordering automatically.
|
||||
Concrete unlockers — including the firmware-unlock profile databases,
|
||||
variant logic, and vendor CDBs that used to live in-tree — are maintained
|
||||
in the separate **[freemkv-unlock](https://github.com/freemkv/freemkv-unlock)**
|
||||
repository, never here.
|
||||
|
||||
---
|
||||
|
||||
## Speed Control
|
||||
|
||||
After `probe_disc()`, the platform driver maintains a speed lookup table
|
||||
built by probing the disc surface. On each `read()` call, the driver:
|
||||
|
||||
1. Looks up the optimal speed for the target LBA.
|
||||
2. Issues SET CD SPEED (0xBB) if the speed differs from current.
|
||||
3. Performs the READ(10).
|
||||
A matching unlocker may raise the drive to its maximum read speed via
|
||||
`set_max_read_speed()` (a no-op when no unlocker matches or the unlocker
|
||||
declines). The library issues SET CD SPEED (0xBB) through the generic CDB
|
||||
builder; the concrete speed policy lives in the unlocker.
|
||||
|
||||
Available speeds:
|
||||
|
||||
|
||||
+48
-27
@@ -12,21 +12,24 @@ reach into the others.
|
||||
|
||||
| Layer | Where it lives | What it does |
|
||||
|-------|---------------|--------------|
|
||||
| 1 — Bad-range retry | `Disc::patch` (multi-pass over the mapfile) | Re-reads non-`+` ranges with the long timeout. Idempotent; call N times. |
|
||||
| 1 — Bad-range retry | `Disc::patch` (one pass over the mapfile per call) | Re-reads non-`+` ranges with the long timeout. Idempotent; caller invokes N times. |
|
||||
| 2 — Single-shot primitive | `Drive::read` in `src/drive/mod.rs` | One CDB, one timeout, one result. No inline retries, no SCSI reset. |
|
||||
| 3 — In-loop request adaptation | `DiscStream::fill_extents` adaptive batch sizer | Halves the batch on failure, retries at the same LBA, walks back up on a clean-read streak. |
|
||||
|
||||
The caller orchestrates layer 1. Autorip's `rip_disc` loops `copy` then
|
||||
N × `patch` per the `MAX_RETRIES` config, then hands the ISO off to the
|
||||
existing mux pipeline. Layer 3 runs inside any consumer of `DiscStream`
|
||||
(direct PES pipeline, ISO playback, etc.) without caller involvement.
|
||||
The library exposes flat verbs; the caller drives the multipass loop. Autorip
|
||||
runs `Disc::sweep` once, then loops `Disc::patch` until either the mapfile is
|
||||
clean or the configured retry budget is exhausted, then hands the ISO off to
|
||||
the mux pipeline. The `freemkv` CLI does the same shape with a
|
||||
terminal-output progress sink. Layer 3 runs inside any consumer of
|
||||
`DiscStream` (direct PES pipeline, ISO playback, etc.) without caller
|
||||
involvement.
|
||||
|
||||
Three primitives compose the disc-side flow:
|
||||
|
||||
| Primitive | What it does |
|
||||
|---------------------------|-----------------------------------------------------------------------|
|
||||
| `Disc::copy` | disc → ISO. Writes a sidecar `.mapfile`. Opt-in skip-forward on failure. |
|
||||
| `Disc::patch` | Re-reads bad ranges from the drive. Idempotent; call N times. |
|
||||
| `Disc::sweep` | disc → ISO, one forward pass. Writes a sidecar `.mapfile`. Opt-in skip-on-error. |
|
||||
| `Disc::patch` | Re-reads bad ranges from the drive. One pass per call; caller invokes N times. |
|
||||
| `DiscStream` (ISO source) | Reads sectors from the ISO, feeds decrypt → demux → codec → mux. |
|
||||
|
||||
## Data model
|
||||
@@ -60,42 +63,60 @@ Status characters match ddrescue:
|
||||
|
||||
Position and size are hex byte offsets into the ISO.
|
||||
|
||||
### `CopyOptions` and `PatchOptions`
|
||||
### `SweepOptions` and `PatchOptions`
|
||||
|
||||
`Disc::copy()` auto-detects the pass from mapfile state:
|
||||
The library no longer dispatches between sweep and patch internally — the
|
||||
caller picks the verb explicitly per pass. The two option structs are flat
|
||||
and have no overlap:
|
||||
|
||||
```rust
|
||||
CopyOptions {
|
||||
decrypt: true, // decrypt AACS/CSS sectors
|
||||
multipass: true, // enable mapfile + skip-on-error + damage-jump
|
||||
progress: Some(&reporter), // progress callback
|
||||
halt: Some(flag), // halt flag for graceful stop
|
||||
SweepOptions {
|
||||
decrypt: true,
|
||||
resume: false,
|
||||
batch_sectors: None,
|
||||
skip_on_error: true, // damage-jump + zero-fill on read failure
|
||||
progress: Some(&reporter),
|
||||
halt: Some(flag),
|
||||
}
|
||||
|
||||
PatchOptions {
|
||||
decrypt: true,
|
||||
block_sectors: None,
|
||||
full_recovery: true,
|
||||
reverse: true, // walk bad ranges high → low LBA
|
||||
wedged_threshold: 50,
|
||||
progress: Some(&reporter),
|
||||
halt: Some(flag),
|
||||
}
|
||||
```
|
||||
|
||||
Dispatch logic:
|
||||
- No mapfile → sweep (fresh Pass 1)
|
||||
- Mapfile with NonTried (`?`) → sweep with resume
|
||||
- Mapfile covering full disc, only NonTrimmed/NonScraped/Unreadable → patch
|
||||
- Mapfile clean → no-op
|
||||
Caller-orchestrated dispatch (the policy `Disc::copy` used to embed):
|
||||
|
||||
- No mapfile → `sweep` (fresh Pass 1).
|
||||
- Mapfile with `?` ranges → `sweep` with `resume: true`.
|
||||
- Mapfile covers full disc, only `*` / `/` / `-` ranges → `patch`.
|
||||
- Mapfile clean → done; no further pass needed.
|
||||
|
||||
Each consumer (autorip, `freemkv` CLI) implements the loop in roughly five
|
||||
lines of `Mapfile::stats()` checks.
|
||||
|
||||
## Algorithm
|
||||
|
||||
### Pass 1 — fast sweep (`Disc::copy` → `sweep_internal`)
|
||||
### Pass 1 — fast sweep (`Disc::sweep`)
|
||||
|
||||
1. Read one ECC block (32 sectors for UHD, 16 for BD/DVD) at the current LBA.
|
||||
2. On success: write data to ISO, mark `+`, advance.
|
||||
3. On failure (with `multipass`): zero-fill, mark `*`, advance.
|
||||
4. Track a sliding window of the last 16 ECC block results. When ≥12% are failures
|
||||
→ **damage-jump**: skip ahead by `256×batch×multiplier` sectors (8 MB base for
|
||||
UHD). Double the multiplier on each jump (8→16→32→64 MB...). Zero-fill the gap as `*`.
|
||||
→ **damage-jump**: skip ahead by `1024×batch×multiplier` sectors (64 MB base for
|
||||
UHD). Double the multiplier on each jump (64→128→256→512 MB...). Zero-fill the gap as `*`.
|
||||
5. On 16 consecutive good reads: reset jump multiplier to 1, restore max read speed.
|
||||
6. Speed control: damage zone entry → minimum speed, exit → maximum speed.
|
||||
7. Only transport failures (USB bridge crash) abort the pass.
|
||||
|
||||
Pass 1 completes when every byte has been visited (either `+` or `*`).
|
||||
|
||||
### Pass 2+ — patch (`Disc::copy` → `patch_internal`)
|
||||
### Pass 2+ — patch (`Disc::patch`)
|
||||
|
||||
`Disc::patch` reads the mapfile and iterates every non-`+` range. Default: **reverse** mode
|
||||
(walks ranges from highest LBA to lowest, within each range from end to start).
|
||||
@@ -140,8 +161,8 @@ recoveries to show for it. Recovery responsibility is now layered: layer 1
|
||||
handles ranges, layer 3 handles request size, neither touches the
|
||||
wedge-prone reset path.
|
||||
|
||||
**No `MODE SELECT` to disable drive retries.** Research showed neither ddrescue
|
||||
nor MakeMKV does this. Drive firmware has access to raw analog signal, laser
|
||||
**No `MODE SELECT` to disable drive retries.** Neither ddrescue
|
||||
nor any consumer ripper does this. Drive firmware has access to raw analog signal, laser
|
||||
power control, and drive-specific ECC tuning that userspace can't replicate —
|
||||
disabling it throws away recovery headroom on marginal sectors. We fail fast
|
||||
via short SG_IO timeouts in pass 1 and let the firmware work the long timeout
|
||||
@@ -157,7 +178,7 @@ explicitly by callers that need an eject-cycle escape hatch — it is never
|
||||
reached from a read path.
|
||||
|
||||
**ISO intermediate, even for single-pass.** Pass 1 always writes an ISO. The
|
||||
mux stage reads the ISO via `IsoSectorReader`. For single-pass (no retries),
|
||||
mux stage reads the ISO via `FileSectorSource`. For single-pass (no retries),
|
||||
this adds ~2-3 min (local disk mux) but gains resumability across crashes,
|
||||
re-muxability without re-ripping, and a persistent forensic artifact. Callers
|
||||
who need pure speed can bypass and use `DiscStream::new(Box::new(drive), …)`
|
||||
@@ -177,4 +198,4 @@ scrape vs. retry with direction reversal) if there's measured benefit.
|
||||
|
||||
- [ddrescue manual, Algorithm chapter](https://www.gnu.org/software/ddrescue/manual/ddrescue_manual.html)
|
||||
- [ddrescue optical media notes](https://www.electric-spoon.com/doc/gddrescue/html/Optical-media.html)
|
||||
- Source: [`src/disc/mapfile.rs`](../src/disc/mapfile.rs), [`src/disc/mod.rs`](../src/disc/mod.rs) (`Disc::copy`, `Disc::patch`), [`src/drive/mod.rs`](../src/drive/mod.rs) (`Drive::read`), [`src/mux/disc.rs`](../src/mux/disc.rs) (`DiscStream::fill_extents`).
|
||||
- Source: [`src/disc/mapfile.rs`](../src/disc/mapfile.rs), [`src/disc/mod.rs`](../src/disc/mod.rs) (`Disc::sweep`), [`src/disc/patch.rs`](../src/disc/patch.rs) (`Disc::patch`), [`src/drive/mod.rs`](../src/drive/mod.rs) (`Drive::read`), [`src/mux/disc.rs`](../src/mux/disc.rs) (`DiscStream::fill_extents`).
|
||||
|
||||
+1
-1
@@ -126,7 +126,7 @@ USB optical drives have ~500ms round-trip latency per SCSI command. Since `read_
|
||||
|
||||
`Disc::scan()` wraps the drive in a `BufferedSectorReader` before reading. On a single-sector read, the buffer prefetches a batch of sectors (sized from the kernel's `max_hw_sectors_kb` for the device) and caches them. Subsequent reads to nearby LBAs return from cache with zero SCSI overhead. After parsing the UDF directory structure, the entire metadata partition is pre-read into the cache, so all ICB lookups during title scanning and encryption resolution are instant.
|
||||
|
||||
The buffer is transparent -- `read_filesystem()`, `read_file()`, and all downstream code still call `read_sectors(lba, 1, buf)` as before. The batching happens inside the `SectorReader` implementation.
|
||||
The buffer is transparent -- `read_filesystem()`, `read_file()`, and all downstream code still call `read_sectors(lba, 1, buf)` as before. The batching happens inside the `SectorSource` implementation.
|
||||
|
||||
### UDF Filename Encoding
|
||||
|
||||
|
||||
-2067
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,308 @@
|
||||
//! AACS derivation "boil-down" — one public home for the key chain.
|
||||
//!
|
||||
//! Thin newtypes at the API boundary and three wrapper functions over the
|
||||
//! existing crypto. Nothing here re-implements a primitive: every function
|
||||
//! delegates to the already-audited code in [`super::keys`] and
|
||||
//! [`super::variants`], so the boil-down cannot drift from production math.
|
||||
//!
|
||||
//! The newtypes wrap bare `[u8; 16]` ONLY at this boundary — the crypto
|
||||
//! internals continue to operate on raw arrays. They exist so a caller threads
|
||||
//! the chain `DK → MK → VUK → UK` without confusing one 16-byte secret for
|
||||
//! another, not to refactor the resolver.
|
||||
//!
|
||||
//! Chain (matches `aacs::keys::resolve_keys_classical` path 1 and
|
||||
//! `aacs::keys::resolve_keys_v21` path 1 byte-for-byte):
|
||||
//!
|
||||
//! ```text
|
||||
//! mk_from_dk(device_keys, mkb, vid) → MediaKey (Km)
|
||||
//! mk_from_pk(processing_keys, mkb) → MediaKey (Km)
|
||||
//! vuk_from_mk(MediaKey, Vid) → Vuk (= AES-G(Km, VID))
|
||||
//! uk_from_vuk(Vuk, enc_title_keys) → [UnitKey] (decrypt_unit_key each)
|
||||
//! ```
|
||||
//!
|
||||
//! `mk_from_dk` and `mk_from_pk` are two entry points to the SAME Media Key:
|
||||
//! the device-key path walks the MKB's Media-Key-Variant chain, the
|
||||
//! processing-key path walks the MKB's Subset-Difference cvalue tables. Neither
|
||||
//! needs a VID (the VID enters at `vuk_from_mk`).
|
||||
|
||||
use super::keys::{decrypt_unit_key, derive_media_key_from_pk, derive_vuk};
|
||||
use super::types::DeviceKey;
|
||||
use super::variants::{KEY_CORRECTION_DATA_PLACEHOLDER, derive_media_key_variant, walk_mkb};
|
||||
|
||||
/// Volume ID (16 bytes) — read from the disc via the SCSI handshake / OEM path.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Vid(pub [u8; 16]);
|
||||
|
||||
/// Media Key (Km, 16 bytes) — the MKB-scoped key derived from device keys.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct MediaKey(pub [u8; 16]);
|
||||
|
||||
/// Volume Unique Key (VUK / Kvu, 16 bytes) — derived from `MediaKey` + `Vid`,
|
||||
/// decrypts the per-disc encrypted title keys in `Unit_Key_RO.inf`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Vuk(pub [u8; 16]);
|
||||
|
||||
/// One decrypted per-CPS-unit AACS title key.
|
||||
///
|
||||
/// `idx` is the POSITIONAL index of the encrypted title key within the slice
|
||||
/// handed to [`uk_from_vuk`] (i.e. its order in `Unit_Key_RO.inf`'s key-storage
|
||||
/// area). The CPS-unit *number* association (the `u32` in
|
||||
/// `ResolvedKeys::unit_keys`) is a higher-level concern owned by
|
||||
/// [`super::keys::parse_unit_key_ro`], which pairs each positional key with its
|
||||
/// declared CPS unit; this primitive only does the AES, so it surfaces position.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct UnitKey {
|
||||
pub idx: u32,
|
||||
pub key: [u8; 16],
|
||||
}
|
||||
|
||||
/// Derive the Volume Unique Key from a Media Key and Volume ID.
|
||||
///
|
||||
/// Wraps [`derive_vuk`] verbatim: `VUK = AES-128-ECB-DECRYPT(MK, VID) XOR VID`.
|
||||
/// This is byte-identical to the inline `derive_vuk(&mk, ctx.volume_id)` call in
|
||||
/// every classical resolver path AND to the `Kvu = AES-G(Km, VID)` step inside
|
||||
/// [`derive_media_key_variant`] (AES-G and `derive_vuk` are the same math), so
|
||||
/// `vuk_from_mk(mk_from_dk(..)?, vid)` reproduces the V21 variant VUK exactly.
|
||||
pub fn vuk_from_mk(mk: MediaKey, vid: Vid) -> Vuk {
|
||||
Vuk(derive_vuk(&mk.0, &vid.0))
|
||||
}
|
||||
|
||||
/// Decrypt the disc's encrypted title keys with a VUK.
|
||||
///
|
||||
/// Wraps [`decrypt_unit_key`] (AES-128-ECB-DECRYPT) per entry, mirroring the
|
||||
/// `derive_uks` closure in `resolve_keys_classical` / `resolve_keys_v21`. The
|
||||
/// returned `UnitKey::idx` is the slice position; pair with CPS-unit numbers via
|
||||
/// [`super::keys::parse_unit_key_ro`] when the numbering matters.
|
||||
pub fn uk_from_vuk(vuk: Vuk, enc_title_keys: &[[u8; 16]]) -> Vec<UnitKey> {
|
||||
enc_title_keys
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, enc)| UnitKey {
|
||||
idx: i as u32,
|
||||
key: decrypt_unit_key(&vuk.0, enc),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Derive the Media Key (Km) from device keys via the Media Key Variant chain.
|
||||
///
|
||||
/// Wraps [`walk_mkb`] + [`derive_media_key_variant`] with exactly the arguments
|
||||
/// `resolve_keys_v21` path 1 passes: the placeholder Key Correction Data and the
|
||||
/// disc Volume ID. Returns the FIRST tuple element `Km` (the Media Key) — the
|
||||
/// resolver treats `Km` as the media key and derives the VUK from it as
|
||||
/// `Kvu = AES-G(Km, VID)`, which equals [`vuk_from_mk`]`(MediaKey(km), vid)`. The
|
||||
/// variant fn's second element is that already-derived `Kvu`; returning `Km`
|
||||
/// keeps this primitive at the "media key" level so the chain composes.
|
||||
///
|
||||
/// Because the integrator KCD is unavailable in-tree (the placeholder is
|
||||
/// rejected by the variant chain), this returns `Err` for every real disc today
|
||||
/// — byte-for-byte identical to `resolve_keys_v21` path 1, which the resolver
|
||||
/// also leaves unreachable in production. All variant-chain failures collapse to
|
||||
/// [`Error::AacsMkUnavailable`] (E7018): no numeric distinction is load-bearing
|
||||
/// at this boundary, and the variant error carries no English to preserve.
|
||||
pub fn mk_from_dk(
|
||||
device_keys: &[DeviceKey],
|
||||
mkb: &[u8],
|
||||
vid: Vid,
|
||||
) -> Result<MediaKey, crate::error::Error> {
|
||||
let records = walk_mkb(mkb);
|
||||
match derive_media_key_variant(
|
||||
&records,
|
||||
device_keys,
|
||||
&KEY_CORRECTION_DATA_PLACEHOLDER,
|
||||
&vid.0,
|
||||
) {
|
||||
Ok((km, _kvu)) => Ok(MediaKey(km)),
|
||||
Err(_) => Err(crate::error::Error::AacsMkUnavailable),
|
||||
}
|
||||
}
|
||||
|
||||
/// Derive the Media Key (Km) from one or more Processing Keys and the disc MKB.
|
||||
///
|
||||
/// Wraps [`derive_media_key_from_pk`] — the Subset-Difference PK→MK walk: each
|
||||
/// processing key is validated (and tree-walked) against the MKB's cvalue tables
|
||||
/// (records `0x04`/`0x05`) until one yields the Media Key whose verify record
|
||||
/// (`0x81`/`0x86`) matches. Unlike [`mk_from_dk`] this path is reachable for
|
||||
/// real discs — a leaked/precomputed AACS Processing Key in the keydb resolves
|
||||
/// the Media Key directly. No VID is involved at this step; the VID enters at
|
||||
/// [`vuk_from_mk`].
|
||||
///
|
||||
/// Returns [`Error::AacsMkUnavailable`] (E7018) when no processing key resolves
|
||||
/// the MKB — the same terminal error as [`mk_from_dk`]; no numeric distinction
|
||||
/// is load-bearing at this boundary.
|
||||
///
|
||||
/// [`Error::AacsMkUnavailable`]: crate::error::Error::AacsMkUnavailable
|
||||
pub fn mk_from_pk(
|
||||
processing_keys: &[[u8; 16]],
|
||||
mkb: &[u8],
|
||||
) -> Result<MediaKey, crate::error::Error> {
|
||||
match derive_media_key_from_pk(mkb, processing_keys) {
|
||||
Some(km) => Ok(MediaKey(km)),
|
||||
None => Err(crate::error::Error::AacsMkUnavailable),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::aacs::decrypt::aes_ecb_encrypt;
|
||||
use crate::aacs::keys::{decrypt_unit_key, derive_vuk};
|
||||
|
||||
/// `vuk_from_mk` must equal the inline `derive_vuk` path bit-for-bit, for
|
||||
/// several known (MK, VID) vectors.
|
||||
#[test]
|
||||
fn vuk_from_mk_matches_inline_derive_vuk() {
|
||||
let cases: [([u8; 16], [u8; 16]); 3] = [
|
||||
([0x5A; 16], [0xA5; 16]),
|
||||
([0x11; 16], [0x22; 16]),
|
||||
(
|
||||
[
|
||||
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C,
|
||||
0x0D, 0x0E, 0x0F,
|
||||
],
|
||||
[
|
||||
0xF0, 0xE1, 0xD2, 0xC3, 0xB4, 0xA5, 0x96, 0x87, 0x78, 0x69, 0x5A, 0x4B, 0x3C,
|
||||
0x2D, 0x1E, 0x0F,
|
||||
],
|
||||
),
|
||||
];
|
||||
for (mk, vid) in cases {
|
||||
let inline = derive_vuk(&mk, &vid);
|
||||
let boiled = vuk_from_mk(MediaKey(mk), Vid(vid));
|
||||
assert_eq!(boiled.0, inline, "vuk_from_mk must equal derive_vuk");
|
||||
}
|
||||
}
|
||||
|
||||
/// `uk_from_vuk` must equal the inline `decrypt_unit_key` path bit-for-bit
|
||||
/// and carry positional indices 0..n. Built by encrypting known plaintext
|
||||
/// title keys under the VUK (the same primitive the resolver inverts).
|
||||
#[test]
|
||||
fn uk_from_vuk_matches_inline_decrypt_unit_key() {
|
||||
let vuk = [0x5Au8; 16];
|
||||
let plain_keys = [[0x11u8; 16], [0x22u8; 16], [0xCDu8; 16]];
|
||||
let enc: Vec<[u8; 16]> = plain_keys
|
||||
.iter()
|
||||
.map(|k| aes_ecb_encrypt(&vuk, k))
|
||||
.collect();
|
||||
|
||||
let boiled = uk_from_vuk(Vuk(vuk), &enc);
|
||||
assert_eq!(boiled.len(), enc.len());
|
||||
for (i, uk) in boiled.iter().enumerate() {
|
||||
assert_eq!(uk.idx, i as u32, "idx must be the positional index");
|
||||
// Matches the inline derive_uks closure: decrypt_unit_key(vuk, enc).
|
||||
assert_eq!(uk.key, decrypt_unit_key(&vuk, &enc[i]));
|
||||
// And recovers the original plaintext title key.
|
||||
assert_eq!(
|
||||
uk.key, plain_keys[i],
|
||||
"VUK roundtrip recovers the title key"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// `uk_from_vuk` on an empty slice yields no keys (no panic, no phantom idx).
|
||||
#[test]
|
||||
fn uk_from_vuk_empty_is_empty() {
|
||||
assert!(uk_from_vuk(Vuk([0u8; 16]), &[]).is_empty());
|
||||
}
|
||||
|
||||
/// `mk_from_dk` returns `Err(AacsMkUnavailable)` for the placeholder-KCD
|
||||
/// path that production also leaves unreachable — never a wrong key, never a
|
||||
/// panic — on both an empty MKB and a non-variant MKB.
|
||||
#[test]
|
||||
fn mk_from_dk_errors_without_integrator_kcd() {
|
||||
let dk = DeviceKey {
|
||||
key: [0x11; 16],
|
||||
node: 1,
|
||||
uv: 1,
|
||||
u_mask_shift: 0,
|
||||
};
|
||||
// Empty MKB → not a variant MKB → Err.
|
||||
let e = mk_from_dk(std::slice::from_ref(&dk), &[], Vid([0x09; 16]));
|
||||
assert!(matches!(e, Err(crate::error::Error::AacsMkUnavailable)));
|
||||
|
||||
// A variant-looking MKB (0x82 record) still cannot complete without the
|
||||
// integrator KCD, so it also errors — never silently yields a key.
|
||||
let mut mkb: Vec<u8> = Vec::new();
|
||||
mkb.extend_from_slice(&[0x82, 0x00, 0x00, 0x14]); // variant data record
|
||||
mkb.extend_from_slice(&[0xAB; 16]);
|
||||
let e2 = mk_from_dk(&[dk], &mkb, Vid([0x09; 16]));
|
||||
assert!(matches!(e2, Err(crate::error::Error::AacsMkUnavailable)));
|
||||
}
|
||||
|
||||
/// Build a 4-byte MKB record header (type + 3-byte big-endian total length,
|
||||
/// header included) and append `body`. Mirrors the MKB record framing the
|
||||
/// parser expects; no crypto.
|
||||
fn mkb_record(rec_type: u8, body: &[u8]) -> Vec<u8> {
|
||||
let total = 4 + body.len();
|
||||
let mut rec = Vec::with_capacity(total);
|
||||
rec.push(rec_type);
|
||||
rec.push(((total >> 16) & 0xFF) as u8);
|
||||
rec.push(((total >> 8) & 0xFF) as u8);
|
||||
rec.push((total & 0xFF) as u8);
|
||||
rec.extend_from_slice(body);
|
||||
rec
|
||||
}
|
||||
|
||||
/// `mk_from_pk` resolves a planted Processing Key against a synthetic MKB and
|
||||
/// drives the FULL boil chain PK → MK → VUK → UK. The MKB is built with the
|
||||
/// same (pk, cv, mk_dv, uv) construction the production SD walk validates, so
|
||||
/// this proves a PK entry yields real Unit Keys — not just an `Ok`.
|
||||
#[test]
|
||||
fn mk_from_pk_drives_full_chain_to_uks() {
|
||||
let pk: [u8; 16] = [
|
||||
0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE,
|
||||
0xFF, 0x00,
|
||||
];
|
||||
let mk: [u8; 16] = [
|
||||
0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD,
|
||||
0xAE, 0xAF,
|
||||
];
|
||||
let uv: [u8; 4] = [0x00, 0x00, 0x04, 0x00];
|
||||
|
||||
// cv = AES-E(pk, mk_raw), where mk_raw is mk with the last-4-bytes-uv XOR
|
||||
// pre-undone, so the validate step XORs uv back in and recovers mk.
|
||||
let mut mk_raw = mk;
|
||||
for a in 0..4 {
|
||||
mk_raw[12 + a] ^= uv[a];
|
||||
}
|
||||
let cv = aes_ecb_encrypt(&pk, &mk_raw);
|
||||
|
||||
// mk_dv = AES-E(mk, magic||pad): AES-D(mk, mk_dv) starts with the AACS
|
||||
// verify sentinel.
|
||||
let mut vd = [0x11u8; 16];
|
||||
vd[..8].copy_from_slice(&[0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF]);
|
||||
let mk_dv = aes_ecb_encrypt(&mk, &vd);
|
||||
|
||||
// Synthetic MKB: type/version (0x10), verify record (0x86 = mk_dv),
|
||||
// one-entry SD index (0x04 = [u_mask_shift=0][uv]), one-entry cvalue
|
||||
// table (0x05 = cv).
|
||||
let mut sd = vec![0u8];
|
||||
sd.extend_from_slice(&uv);
|
||||
let mut mkb = Vec::new();
|
||||
mkb.extend_from_slice(&mkb_record(0x10, &[0, 0, 0, 0x20, 0, 0, 0, 0x52]));
|
||||
mkb.extend_from_slice(&mkb_record(0x86, &mk_dv));
|
||||
mkb.extend_from_slice(&mkb_record(0x04, &sd));
|
||||
mkb.extend_from_slice(&mkb_record(0x05, &cv));
|
||||
|
||||
// PK → MK.
|
||||
let got_mk = mk_from_pk(std::slice::from_ref(&pk), &mkb).expect("planted PK resolves MK");
|
||||
assert_eq!(got_mk, MediaKey(mk), "mk_from_pk recovers the planted MK");
|
||||
|
||||
// MK → VUK → UK over an encrypted title key.
|
||||
let vid = Vid([0x42u8; 16]);
|
||||
let plain_uk = [0x7Eu8; 16];
|
||||
let vuk = vuk_from_mk(got_mk, vid);
|
||||
let enc = aes_ecb_encrypt(&vuk.0, &plain_uk);
|
||||
let uks = uk_from_vuk(vuk, std::slice::from_ref(&enc));
|
||||
assert_eq!(uks.len(), 1);
|
||||
assert_eq!(uks[0].key, plain_uk, "PK chain recovers the title key");
|
||||
|
||||
// A corrupt PK resolves nothing.
|
||||
let mut bad = pk;
|
||||
bad[0] ^= 0xFF;
|
||||
assert!(matches!(
|
||||
mk_from_pk(std::slice::from_ref(&bad), &mkb),
|
||||
Err(crate::error::Error::AacsMkUnavailable)
|
||||
));
|
||||
}
|
||||
}
|
||||
+949
-48
File diff suppressed because it is too large
Load Diff
+727
-97
File diff suppressed because it is too large
Load Diff
@@ -1,425 +0,0 @@
|
||||
//! AACS Key Database parsing — KEYDB.cfg format.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Parsed AACS key database.
|
||||
#[derive(Debug)]
|
||||
pub struct KeyDb {
|
||||
/// Device keys for MKB processing
|
||||
pub device_keys: Vec<DeviceKey>,
|
||||
/// Processing keys (pre-computed media keys for specific MKB versions)
|
||||
pub processing_keys: Vec<[u8; 16]>,
|
||||
/// Host certificate + private key for SCSI authentication
|
||||
pub host_certs: Vec<HostCert>,
|
||||
/// Per-disc VUK entries indexed by disc hash (hex lowercase)
|
||||
pub disc_entries: HashMap<String, DiscEntry>,
|
||||
}
|
||||
|
||||
/// A device key for MKB subset-difference tree processing.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DeviceKey {
|
||||
pub key: [u8; 16],
|
||||
pub node: u16,
|
||||
pub uv: u32,
|
||||
pub u_mask_shift: u8,
|
||||
}
|
||||
|
||||
/// Host certificate + private key for AACS SCSI authentication.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HostCert {
|
||||
/// AACS 1.0: 20 bytes. AACS 2.0: 32 bytes.
|
||||
pub private_key: [u8; 20],
|
||||
/// AACS 1.0: 92 bytes. AACS 2.0: 132 bytes.
|
||||
pub certificate: Vec<u8>,
|
||||
/// AACS 2.0 host private key (P-256, 32 bytes). None for AACS 1.0 only.
|
||||
pub private_key_v2: Option<[u8; 32]>,
|
||||
/// AACS 2.0 host certificate (type 0x11). None for AACS 1.0 only.
|
||||
pub certificate_v2: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
/// A per-disc entry from the key database.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DiscEntry {
|
||||
/// Disc hash (20 bytes, hex)
|
||||
pub disc_hash: String,
|
||||
/// Disc title
|
||||
pub title: String,
|
||||
/// Media Key (16 bytes) — from MKB processing
|
||||
pub media_key: Option<[u8; 16]>,
|
||||
/// Disc ID (16 bytes)
|
||||
pub disc_id: Option<[u8; 16]>,
|
||||
/// Volume Unique Key (16 bytes) — decrypts title keys
|
||||
pub vuk: Option<[u8; 16]>,
|
||||
/// Unit keys (title keys) indexed by CPS unit number
|
||||
pub unit_keys: Vec<(u32, [u8; 16])>,
|
||||
}
|
||||
|
||||
/// Parse a hex string like "0xABCD..." into bytes.
|
||||
pub(crate) fn parse_hex(s: &str) -> Option<Vec<u8>> {
|
||||
let s = s.trim().trim_start_matches("0x").trim_start_matches("0X");
|
||||
if s.len() % 2 != 0 {
|
||||
return None;
|
||||
}
|
||||
let mut out = Vec::with_capacity(s.len() / 2);
|
||||
for i in (0..s.len()).step_by(2) {
|
||||
out.push(u8::from_str_radix(&s[i..i + 2], 16).ok()?);
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Parse hex into a fixed-size array.
|
||||
pub(crate) fn parse_hex16(s: &str) -> Option<[u8; 16]> {
|
||||
let v = parse_hex(s)?;
|
||||
if v.len() != 16 {
|
||||
return None;
|
||||
}
|
||||
let mut out = [0u8; 16];
|
||||
out.copy_from_slice(&v);
|
||||
Some(out)
|
||||
}
|
||||
|
||||
pub(crate) fn parse_hex20(s: &str) -> Option<[u8; 20]> {
|
||||
let v = parse_hex(s)?;
|
||||
if v.len() != 20 {
|
||||
return None;
|
||||
}
|
||||
let mut out = [0u8; 20];
|
||||
out.copy_from_slice(&v);
|
||||
Some(out)
|
||||
}
|
||||
|
||||
impl KeyDb {
|
||||
/// Parse a KEYDB.cfg file from a string.
|
||||
pub fn parse(data: &str) -> Self {
|
||||
let mut db = KeyDb {
|
||||
device_keys: Vec::new(),
|
||||
processing_keys: Vec::new(),
|
||||
host_certs: Vec::new(),
|
||||
disc_entries: HashMap::new(),
|
||||
};
|
||||
|
||||
for line in data.lines() {
|
||||
let line = line.trim();
|
||||
|
||||
// Skip comments and empty lines
|
||||
if line.is_empty() || line.starts_with(';') || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Device Key
|
||||
if line.starts_with("| DK") {
|
||||
if let Some(dk) = Self::parse_device_key(line) {
|
||||
db.device_keys.push(dk);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Processing Key
|
||||
if line.starts_with("| PK") {
|
||||
if let Some(pk) = Self::parse_processing_key(line) {
|
||||
db.processing_keys.push(pk);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Host Certificate (AACS 2.0)
|
||||
if line.starts_with("| HC2") {
|
||||
if let Some(hc) = db.host_certs.last_mut() {
|
||||
if let Some((pk, cert)) = Self::parse_host_cert_v2(line) {
|
||||
hc.private_key_v2 = Some(pk);
|
||||
hc.certificate_v2 = Some(cert);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Host Certificate (AACS 1.0)
|
||||
if line.starts_with("| HC") {
|
||||
if let Some(hc) = Self::parse_host_cert(line) {
|
||||
db.host_certs.push(hc);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Disc entry: starts with 0x
|
||||
if line.starts_with("0x") && line.contains(" = ") {
|
||||
if let Some(entry) = Self::parse_disc_entry(line) {
|
||||
db.disc_entries.insert(entry.disc_hash.clone(), entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
db
|
||||
}
|
||||
|
||||
/// Load KEYDB.cfg from a file path.
|
||||
pub fn load(path: &std::path::Path) -> std::io::Result<Self> {
|
||||
let data = std::fs::read_to_string(path)?;
|
||||
Ok(Self::parse(&data))
|
||||
}
|
||||
|
||||
/// Look up a disc by its hash. Returns the VUK if found.
|
||||
pub fn find_vuk(&self, disc_hash: &str) -> Option<[u8; 16]> {
|
||||
let hash = disc_hash
|
||||
.trim()
|
||||
.to_lowercase()
|
||||
.trim_start_matches("0x")
|
||||
.to_string();
|
||||
// Try with 0x prefix and without
|
||||
self.disc_entries
|
||||
.get(&format!("0x{hash}"))
|
||||
.or_else(|| self.disc_entries.get(&hash))
|
||||
.and_then(|e| e.vuk)
|
||||
}
|
||||
|
||||
/// Look up a disc by its hash. Returns the full entry.
|
||||
pub fn find_disc(&self, disc_hash: &str) -> Option<&DiscEntry> {
|
||||
let hash = disc_hash
|
||||
.trim()
|
||||
.to_lowercase()
|
||||
.trim_start_matches("0x")
|
||||
.to_string();
|
||||
self.disc_entries
|
||||
.get(&format!("0x{hash}"))
|
||||
.or_else(|| self.disc_entries.get(&hash))
|
||||
}
|
||||
|
||||
// ── Parsers ─────────────────────────────────────────────────────────────
|
||||
|
||||
fn parse_device_key(line: &str) -> Option<DeviceKey> {
|
||||
// | DK | DEVICE_KEY 0x... | DEVICE_NODE 0x... | KEY_UV 0x... | KEY_U_MASK_SHIFT 0x...
|
||||
let key_str = line.split("DEVICE_KEY").nth(1)?.split('|').next()?.trim();
|
||||
let node_str = line.split("DEVICE_NODE").nth(1)?.split('|').next()?.trim();
|
||||
let uv_str = line.split("KEY_UV").nth(1)?.split('|').next()?.trim();
|
||||
let shift_str = line
|
||||
.split("KEY_U_MASK_SHIFT")
|
||||
.nth(1)?
|
||||
.split(';')
|
||||
.next()?
|
||||
.split('|')
|
||||
.next()?
|
||||
.trim();
|
||||
|
||||
Some(DeviceKey {
|
||||
key: parse_hex16(key_str)?,
|
||||
node: u16::from_str_radix(node_str.trim_start_matches("0x"), 16).ok()?,
|
||||
uv: u32::from_str_radix(uv_str.trim_start_matches("0x"), 16).ok()?,
|
||||
u_mask_shift: u8::from_str_radix(shift_str.trim_start_matches("0x"), 16).ok()?,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_processing_key(line: &str) -> Option<[u8; 16]> {
|
||||
// | PK | 0x...
|
||||
let parts: Vec<&str> = line.split('|').collect();
|
||||
if parts.len() >= 3 {
|
||||
let key_str = parts[2].split(';').next()?.trim();
|
||||
return parse_hex16(key_str);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn parse_host_cert(line: &str) -> Option<HostCert> {
|
||||
// | HC | HOST_PRIV_KEY 0x... | HOST_CERT 0x...
|
||||
let priv_str = line
|
||||
.split("HOST_PRIV_KEY")
|
||||
.nth(1)?
|
||||
.split('|')
|
||||
.next()?
|
||||
.trim();
|
||||
let cert_str = line
|
||||
.split("HOST_CERT")
|
||||
.nth(1)?
|
||||
.split(';')
|
||||
.next()?
|
||||
.split('|')
|
||||
.next()?
|
||||
.trim();
|
||||
|
||||
Some(HostCert {
|
||||
private_key: parse_hex20(priv_str)?,
|
||||
certificate: parse_hex(cert_str)?,
|
||||
private_key_v2: None,
|
||||
certificate_v2: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse AACS 2.0 host cert: `| HC2 | HOST_PRIV_KEY 0x... | HOST_CERT 0x...`
|
||||
fn parse_host_cert_v2(line: &str) -> Option<([u8; 32], Vec<u8>)> {
|
||||
let priv_str = line
|
||||
.split("HOST_PRIV_KEY")
|
||||
.nth(1)?
|
||||
.split('|')
|
||||
.next()?
|
||||
.trim();
|
||||
let cert_str = line
|
||||
.split("HOST_CERT")
|
||||
.nth(1)?
|
||||
.split(';')
|
||||
.next()?
|
||||
.split('|')
|
||||
.next()?
|
||||
.trim();
|
||||
|
||||
let priv_bytes = parse_hex(priv_str)?;
|
||||
if priv_bytes.len() != 32 {
|
||||
return None;
|
||||
}
|
||||
let mut pk = [0u8; 32];
|
||||
pk.copy_from_slice(&priv_bytes);
|
||||
|
||||
let cert = parse_hex(cert_str)?;
|
||||
if cert.len() < 132 {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some((pk, cert))
|
||||
}
|
||||
|
||||
fn parse_disc_entry(line: &str) -> Option<DiscEntry> {
|
||||
// 0x<hash> = <title> | D | <date> | M | 0x<mk> | I | 0x<id> | V | 0x<vuk> | U | <unit_keys>
|
||||
let (hash_part, rest) = line.split_once(" = ")?;
|
||||
let disc_hash = hash_part.trim().to_lowercase();
|
||||
|
||||
// Extract title (before first |)
|
||||
let title_part = rest.split(" | ").next().unwrap_or("").trim();
|
||||
// Clean title: "TITLE_NAME (Display Title)" → use display title if present
|
||||
let title = if let Some(start) = title_part.find('(') {
|
||||
if let Some(end) = title_part.rfind(')') {
|
||||
title_part[start + 1..end].to_string()
|
||||
} else {
|
||||
title_part.to_string()
|
||||
}
|
||||
} else {
|
||||
title_part.to_string()
|
||||
};
|
||||
|
||||
// Parse fields by tag
|
||||
let mut media_key = None;
|
||||
let mut disc_id = None;
|
||||
let mut vuk = None;
|
||||
let mut unit_keys = Vec::new();
|
||||
|
||||
let parts: Vec<&str> = rest.split(" | ").collect();
|
||||
let mut i = 0;
|
||||
while i < parts.len() {
|
||||
match parts[i].trim() {
|
||||
"M" => {
|
||||
if i + 1 < parts.len() {
|
||||
media_key = parse_hex16(parts[i + 1].trim());
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
"I" => {
|
||||
if i + 1 < parts.len() {
|
||||
disc_id = parse_hex16(parts[i + 1].trim());
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
"V" => {
|
||||
if i + 1 < parts.len() {
|
||||
vuk = parse_hex16(parts[i + 1].trim());
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
"U" => {
|
||||
if i + 1 < parts.len() {
|
||||
// Unit keys: "1-0xKEY" or "1-0xKEY ; comment"
|
||||
let uk_str = parts[i + 1].split(';').next().unwrap_or("").trim();
|
||||
for uk in uk_str.split(' ') {
|
||||
let uk = uk.trim();
|
||||
if let Some((num, key)) = uk.split_once('-') {
|
||||
if let Ok(n) = num.parse::<u32>() {
|
||||
if let Some(k) = parse_hex16(key) {
|
||||
unit_keys.push((n, k));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
|
||||
Some(DiscEntry {
|
||||
disc_hash,
|
||||
title,
|
||||
media_key,
|
||||
disc_id,
|
||||
vuk,
|
||||
unit_keys,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Get KEYDB path from KEYDB_PATH environment variable. Returns None if not set or not found.
|
||||
fn keydb_path() -> Option<std::path::PathBuf> {
|
||||
let path = std::path::PathBuf::from(std::env::var("KEYDB_PATH").ok()?);
|
||||
if path.exists() { Some(path) } else { None }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_disc_entry() {
|
||||
let line = r#"***REMOVED*** = DUNE_PART_TWO (Dune: Part Two) | D | 2024-04-02 | M | ***REMOVED*** | I | ***REMOVED*** | V | ***REMOVED*** | U | 1-***REMOVED*** ; MKBv77"#;
|
||||
let entry = KeyDb::parse_disc_entry(line).unwrap();
|
||||
assert_eq!(entry.title, "Dune: Part Two");
|
||||
assert!(entry.media_key.is_some());
|
||||
assert!(entry.vuk.is_some());
|
||||
assert_eq!(entry.unit_keys.len(), 1);
|
||||
assert_eq!(entry.unit_keys[0].0, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_device_key() {
|
||||
let line = "| DK | DEVICE_KEY ***REMOVED*** | DEVICE_NODE 0x0800 | KEY_UV 0x00000400 | KEY_U_MASK_SHIFT 0x17 ; MKBv01-MKBv48";
|
||||
let dk = KeyDb::parse_device_key(line).unwrap();
|
||||
assert_eq!(dk.node, 0x0800);
|
||||
assert_eq!(dk.u_mask_shift, 0x17);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_host_cert() {
|
||||
let line = "| HC | HOST_PRIV_KEY ***REMOVED*** | HOST_CERT ***REMOVED*** ; Revoked";
|
||||
let hc = KeyDb::parse_host_cert(line).unwrap();
|
||||
assert_eq!(hc.private_key[0], 0x90);
|
||||
assert_eq!(hc.certificate.len(), 92);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_full_keydb() {
|
||||
let path = match keydb_path() {
|
||||
Some(p) => p,
|
||||
None => return,
|
||||
}; // skip if not available
|
||||
|
||||
let db = KeyDb::load(&path).unwrap();
|
||||
|
||||
assert_eq!(db.device_keys.len(), 4);
|
||||
assert_eq!(db.processing_keys.len(), 3);
|
||||
assert!(!db.host_certs.is_empty());
|
||||
assert!(db.disc_entries.len() > 170000);
|
||||
|
||||
// Look up Dune: Part Two
|
||||
let dune = db
|
||||
.disc_entries
|
||||
.values()
|
||||
.find(|e| e.title.contains("Dune: Part Two") && e.vuk.is_some())
|
||||
.expect("Dune: Part Two not found");
|
||||
assert!(dune.media_key.is_some());
|
||||
assert!(dune.vuk.is_some());
|
||||
assert!(!dune.unit_keys.is_empty());
|
||||
|
||||
eprintln!(
|
||||
"Parsed {} disc entries, {} DK, {} PK",
|
||||
db.disc_entries.len(),
|
||||
db.device_keys.len(),
|
||||
db.processing_keys.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
+2523
-327
File diff suppressed because it is too large
Load Diff
+82
-7
@@ -8,25 +8,100 @@
|
||||
//! | DK | DEVICE_KEY 0x... | DEVICE_NODE 0x... | KEY_UV 0x... | KEY_U_MASK_SHIFT 0x...
|
||||
//! | PK | 0x...
|
||||
//! | HC | HOST_PRIV_KEY 0x... | HOST_CERT 0x...
|
||||
//! | HC2 | HOST_PRIV_KEY 0x... | HOST_CERT 0x...
|
||||
//! 0x<disc_hash> = <title> | D | <date> | M | 0x<media_key> | I | 0x<disc_id> | V | 0x<vuk> | U | <unit_keys>
|
||||
//!
|
||||
//! The VUK decrypts title keys from AACS/Unit_Key_RO.inf on disc.
|
||||
//! Title keys decrypt m2ts stream content (AES-128-CBC).
|
||||
|
||||
pub mod boil;
|
||||
pub mod decrypt;
|
||||
pub mod handshake;
|
||||
pub mod keydb;
|
||||
pub mod keys;
|
||||
pub mod provider;
|
||||
pub mod trace;
|
||||
pub mod types;
|
||||
pub mod variants;
|
||||
|
||||
// Boil-down derivation primitives (thin newtypes + wrappers over the crypto).
|
||||
pub use boil::{MediaKey, UnitKey, Vid, Vuk, mk_from_dk, mk_from_pk, uk_from_vuk, vuk_from_mk};
|
||||
// Structured, English-free resolution trace.
|
||||
pub use trace::{KeyNode, KeyOutcome, KeyStep, ResolutionTrace, UnlockOutcome, UnlockStep};
|
||||
|
||||
// Explicit re-exports — only items needed by external consumers and sibling crate modules.
|
||||
// AES primitives (aes_ecb_encrypt, aes_ecb_decrypt, aes_cbc_decrypt) are pub(crate) in decrypt.rs.
|
||||
pub use decrypt::{
|
||||
ALIGNED_UNIT_LEN, decrypt_bus, decrypt_unit, decrypt_unit_full, decrypt_unit_try_keys,
|
||||
is_unit_encrypted,
|
||||
ALIGNED_UNIT_LEN, ALIGNED_UNIT_SECTORS, UnitKeyResult, aacs_unit_encrypted,
|
||||
aacs_unit_needs_decrypt, decrypt_bus, decrypt_unit, decrypt_unit_checked, decrypt_unit_full,
|
||||
decrypt_unit_try_keys, is_unit_aligned, ts_packet_total, ts_sync_count, ts_sync_destroyed,
|
||||
unit_is_clean_ps, unit_is_clean_ts, unit_key_validates,
|
||||
};
|
||||
pub use keydb::{DeviceKey, DiscEntry, HostCert, KeyDb};
|
||||
// `probe` is a reproduction-harness helper (see keys.rs), not part of the
|
||||
// documented 1.0 surface; keep it reachable but off the rendered docs so we
|
||||
// don't commit semver stability to test primitives.
|
||||
#[doc(hidden)]
|
||||
pub use keys::probe;
|
||||
pub use keys::{
|
||||
ContentCert, ResolvedKeys, UnitKeyFile, decrypt_unit_key, derive_media_key_from_dk,
|
||||
derive_media_key_from_pk, derive_vuk, disc_hash, disc_hash_hex, mkb_version,
|
||||
parse_content_cert, parse_unit_key_ro, read_mkb_from_drive, resolve_keys,
|
||||
AacsVersion, ContentCert, MKB_20_CATEGORY_C, MKB_21_CATEGORY_C, MKB_TYPE_3_RECORDABLE,
|
||||
MKB_TYPE_4_PRERECORDED, MKB_TYPE_10_CLASS_II, MkbType, ResolveContext, ResolveFailure,
|
||||
ResolvedKeys, UnitKeyFile, decrypt_unit_key, derive_media_key_and_pk_from_dk,
|
||||
derive_media_key_from_dk, derive_media_key_from_pk, derive_vuk, disc_hash, disc_hash_hex,
|
||||
mkb_content_len, mkb_is_uhd, mkb_type, mkb_type_raw, mkb_version, parse_content_cert,
|
||||
parse_unit_key_ro, read_mkb_from_drive, recover_dk_position, resolve_keys_v1, resolve_keys_v2,
|
||||
resolve_keys_v21, resolve_keys_with_reason, trim_mkb,
|
||||
};
|
||||
pub use provider::KeyProvider;
|
||||
pub use types::{DeviceKey, DiscEntry, HostCert};
|
||||
pub use variants::{
|
||||
KEY_CORRECTION_DATA_PLACEHOLDER, MediaKeyVariantError, MkbRecord, ProcessingKeyMatch,
|
||||
derive_media_key_variant, is_variant_mkb, variant_nonce, walk_mkb, walk_processing_key,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
//! Re-export surface guards. The module's public API is the set of
|
||||
//! `pub use` items above. A regression that drops or renames an export
|
||||
//! (the class of bug that shipped in 0.31.0 by silently changing a
|
||||
//! surface) breaks compilation of these references, so they act as a
|
||||
//! compile-time contract for the crate's AACS surface.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn aligned_unit_len_is_three_2048_byte_sectors() {
|
||||
// ALIGNED_UNIT_LEN is the AACS aligned-unit size: 3 × 2048 = 6144.
|
||||
// Re-exported from decrypt; pin the value here so the public constant
|
||||
// and the spec stay in lockstep.
|
||||
assert_eq!(ALIGNED_UNIT_LEN, 6144);
|
||||
assert_eq!(ALIGNED_UNIT_LEN, 3 * 2048);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_strides_are_reexported_and_distinct() {
|
||||
// The three AACS generations are part of the public surface, and the
|
||||
// V10 (48) vs V20/V21 (64) stride distinction is the load-bearing
|
||||
// difference. Confirm the enum re-export is usable and the variants
|
||||
// are distinct values.
|
||||
assert_ne!(AacsVersion::V10, AacsVersion::V20);
|
||||
assert_ne!(AacsVersion::V20, AacsVersion::V21);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn key_correction_data_placeholder_is_all_zero() {
|
||||
// The variant chain refuses to run against this all-zero placeholder
|
||||
// KCD; the public constant must therefore be exactly 16 zero bytes.
|
||||
assert_eq!(KEY_CORRECTION_DATA_PLACEHOLDER, [0u8; 16]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_helpers_are_callable_through_the_facade() {
|
||||
// Touch a representative function from each re-export group so a
|
||||
// dropped/renamed export fails to compile. These are smoke calls, not
|
||||
// behavioural assertions (behaviour is covered in each module).
|
||||
let _ = ts_sync_destroyed(&[0u8; ALIGNED_UNIT_LEN]);
|
||||
let _ = mkb_content_len(&[]);
|
||||
let _ = is_variant_mkb(&walk_mkb(&[]));
|
||||
let _ = disc_hash_hex(&disc_hash(b"x"));
|
||||
let _ = mk_from_pk(&[[0u8; 16]], &[]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
//! Key source abstraction for the AACS resolve chain.
|
||||
//!
|
||||
//! libfreemkv keeps all crypto (AES-G primitives, SD-tree walking,
|
||||
//! validation, MK/VUK/TK derivation) but accepts key material from
|
||||
//! arbitrary backends via [`KeyProvider`].
|
||||
//!
|
||||
//! Methods come in two flavors:
|
||||
//!
|
||||
//! - **Bulk material** ([`device_keys`], [`processing_keys`],
|
||||
//! [`media_keys`]) — the resolver unions (and dedups) results
|
||||
//! across all providers and tries each candidate.
|
||||
//! - **Disc-keyed lookup** ([`lookup_disc_by_hash`],
|
||||
//! [`lookup_disc_by_vid`]) — the resolver short-circuits on the
|
||||
//! first hit, so providers are queried in array order with
|
||||
//! fastest/closest first.
|
||||
//!
|
||||
//! [`host_certs`] is a sixth method but is NOT consumed by the
|
||||
//! resolver chain: the SCSI handshake reads host certs directly from
|
||||
//! the caller-supplied credentials, not from the provider array. A
|
||||
//! provider that overrides `host_certs` today has no effect on the
|
||||
//! handshake; the method is retained as a forward-looking extension
|
||||
//! point only.
|
||||
//!
|
||||
//! Default impls return empty / `None` so backends only override
|
||||
//! the methods they actually support — an external key service might
|
||||
//! implement only `lookup_disc_by_hash`, while a local file might
|
||||
//! implement all six.
|
||||
//!
|
||||
//! Calls may block (disk I/O, network round-trips). The resolver
|
||||
//! invokes each method at most a handful of times per scan; for
|
||||
//! per-disc memoization, implementations should cache internally.
|
||||
//!
|
||||
//! [`device_keys`]: KeyProvider::device_keys
|
||||
//! [`processing_keys`]: KeyProvider::processing_keys
|
||||
//! [`media_keys`]: KeyProvider::media_keys
|
||||
//! [`host_certs`]: KeyProvider::host_certs
|
||||
//! [`lookup_disc_by_hash`]: KeyProvider::lookup_disc_by_hash
|
||||
//! [`lookup_disc_by_vid`]: KeyProvider::lookup_disc_by_vid
|
||||
|
||||
use super::types::{DeviceKey, DiscEntry, HostCert};
|
||||
|
||||
/// Source of AACS key material.
|
||||
///
|
||||
/// Implementors return raw material only — the resolver in
|
||||
/// `aacs::keys` owns all the crypto (DK→PK walking, PK validation,
|
||||
/// MK→VUK→TK derivation). See module docs for method semantics.
|
||||
pub trait KeyProvider: Send + Sync {
|
||||
/// Device keys (top-of-tree, walked by the resolver).
|
||||
fn device_keys(&self) -> Vec<DeviceKey> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// Processing keys — terminal PKs or walk-input PKs. The
|
||||
/// resolver tries each as a terminal first (cheap validate).
|
||||
fn processing_keys(&self) -> Vec<[u8; 16]> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// Every Media Key this provider holds, regardless of which disc it was
|
||||
/// filed under. An MK is MKB-scoped (shared across a pressing/MKB-family),
|
||||
/// so the resolver can verify each against the disc's MKB (`km_verifies`)
|
||||
/// and resolve a disc whose own hash/VID isn't directly keyed.
|
||||
fn media_keys(&self) -> Vec<[u8; 16]> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// AACS host certificates (with their private keys) for drive
|
||||
/// authentication. Multiple in case some are revoked.
|
||||
///
|
||||
/// NOTE: not consumed by the resolver chain — the handshake reads
|
||||
/// host certs from the caller-supplied credentials directly, so
|
||||
/// overriding this method has no effect on drive authentication
|
||||
/// today. Retained as a forward-looking extension point.
|
||||
fn host_certs(&self) -> Vec<HostCert> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// Direct per-disc lookup by SHA-1 of `Unit_Key_RO.inf`. Returns
|
||||
/// `Some(entry)` if this provider has pre-computed material for
|
||||
/// the disc (paths 4 and 5). Short-circuits the resolver.
|
||||
fn lookup_disc_by_hash(&self, _disc_hash: &[u8; 20]) -> Option<DiscEntry> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Lookup by Volume ID (path 3 — pre-computed MK + matching
|
||||
/// VID). Short-circuits the resolver on hit.
|
||||
fn lookup_disc_by_vid(&self, _volume_id: &[u8; 16]) -> Option<DiscEntry> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolver-side helpers that aggregate across a provider array.
|
||||
///
|
||||
/// The resolver wraps `ctx.providers` (`&[&dyn KeyProvider]`) in this
|
||||
/// struct; these helpers apply the union-vs-short-circuit policy per
|
||||
/// method. The bulk unions dedup so overlapping providers don't make
|
||||
/// the resolver re-walk/re-validate identical material.
|
||||
pub(crate) struct Providers<'a>(pub &'a [&'a dyn KeyProvider]);
|
||||
|
||||
impl Providers<'_> {
|
||||
/// Union (deduped) — gather DKs from every provider.
|
||||
pub fn device_keys(&self) -> Vec<DeviceKey> {
|
||||
let mut v: Vec<DeviceKey> = self.0.iter().flat_map(|p| p.device_keys()).collect();
|
||||
// DeviceKey has no Ord/Hash; dedup on the value-defining tuple.
|
||||
v.sort_unstable_by_key(|d| (d.key, d.node, d.uv, d.u_mask_shift));
|
||||
v.dedup_by_key(|d| (d.key, d.node, d.uv, d.u_mask_shift));
|
||||
v
|
||||
}
|
||||
|
||||
/// Union (deduped) — gather PKs from every provider.
|
||||
pub fn processing_keys(&self) -> Vec<[u8; 16]> {
|
||||
let mut v: Vec<[u8; 16]> = self.0.iter().flat_map(|p| p.processing_keys()).collect();
|
||||
v.sort_unstable();
|
||||
v.dedup();
|
||||
v
|
||||
}
|
||||
|
||||
/// Union of distinct Media Keys across every provider, for the MK-pool
|
||||
/// brute (`km_verifies` against the disc's MKB).
|
||||
pub fn media_keys(&self) -> Vec<[u8; 16]> {
|
||||
let mut v: Vec<[u8; 16]> = self.0.iter().flat_map(|p| p.media_keys()).collect();
|
||||
v.sort_unstable();
|
||||
v.dedup();
|
||||
v
|
||||
}
|
||||
|
||||
/// Union — gather host certs from every provider. The SCSI handshake
|
||||
/// reads host certs from the caller-supplied credentials directly and
|
||||
/// does not call this, so it is currently unused by the resolver chain.
|
||||
#[allow(dead_code)]
|
||||
pub fn host_certs(&self) -> Vec<HostCert> {
|
||||
self.0.iter().flat_map(|p| p.host_certs()).collect()
|
||||
}
|
||||
|
||||
/// Short-circuit — query providers in array order, first hit wins.
|
||||
pub fn lookup_disc_by_hash(&self, disc_hash: &[u8; 20]) -> Option<DiscEntry> {
|
||||
self.0.iter().find_map(|p| p.lookup_disc_by_hash(disc_hash))
|
||||
}
|
||||
|
||||
/// Short-circuit — query providers in array order, first hit wins.
|
||||
pub fn lookup_disc_by_vid(&self, volume_id: &[u8; 16]) -> Option<DiscEntry> {
|
||||
self.0.iter().find_map(|p| p.lookup_disc_by_vid(volume_id))
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`KeyProvider`] backed by a single caller-supplied key's raw material —
|
||||
/// the bridge for [`crate::disc::Disc::decrypt_with`].
|
||||
///
|
||||
/// The application's key source did the lookup and handed in material at one
|
||||
/// level (DK / PK / MK / VUK). This exposes exactly that material to the
|
||||
/// version-dispatched resolver, which owns ALL derivation — so a source never
|
||||
/// derives, and the lib remains the single home for the AACS chain across
|
||||
/// 1.0 / 2.0 / 2.1 / 2.x.
|
||||
///
|
||||
/// Each level fills only its own field; the rest stay empty, so the resolver
|
||||
/// naturally runs the matching path (DK→…, PK→…, MK-pool brute, or a
|
||||
/// disc-keyed VUK hit). `decrypt_with` already knows the disc, so the
|
||||
/// `lookup_disc_by_*` hash/VID arguments are irrelevant — a present
|
||||
/// `disc_entry` is returned for any query.
|
||||
pub(crate) struct SuppliedKey {
|
||||
pub device_keys: Vec<DeviceKey>,
|
||||
pub processing_keys: Vec<[u8; 16]>,
|
||||
pub media_keys: Vec<[u8; 16]>,
|
||||
pub disc_entry: Option<DiscEntry>,
|
||||
}
|
||||
|
||||
impl KeyProvider for SuppliedKey {
|
||||
fn device_keys(&self) -> Vec<DeviceKey> {
|
||||
self.device_keys.clone()
|
||||
}
|
||||
fn processing_keys(&self) -> Vec<[u8; 16]> {
|
||||
self.processing_keys.clone()
|
||||
}
|
||||
fn media_keys(&self) -> Vec<[u8; 16]> {
|
||||
self.media_keys.clone()
|
||||
}
|
||||
fn lookup_disc_by_hash(&self, _disc_hash: &[u8; 20]) -> Option<DiscEntry> {
|
||||
self.disc_entry.clone()
|
||||
}
|
||||
fn lookup_disc_by_vid(&self, _volume_id: &[u8; 16]) -> Option<DiscEntry> {
|
||||
self.disc_entry.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn entry(hash: &str, vuk: u8) -> DiscEntry {
|
||||
DiscEntry {
|
||||
disc_hash: hash.to_string(),
|
||||
title: "t".to_string(),
|
||||
media_key: None,
|
||||
disc_id: None,
|
||||
vuk: Some([vuk; 16]),
|
||||
unit_keys: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn dk(byte: u8, node: u16) -> DeviceKey {
|
||||
DeviceKey {
|
||||
key: [byte; 16],
|
||||
node,
|
||||
uv: 1,
|
||||
u_mask_shift: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// A provider that returns fixed bulk material and an optional disc entry
|
||||
/// keyed unconditionally (used to test array-order short-circuiting).
|
||||
#[derive(Default)]
|
||||
struct Fixed {
|
||||
dks: Vec<DeviceKey>,
|
||||
pks: Vec<[u8; 16]>,
|
||||
mks: Vec<[u8; 16]>,
|
||||
hash_hit: Option<DiscEntry>,
|
||||
vid_hit: Option<DiscEntry>,
|
||||
}
|
||||
impl KeyProvider for Fixed {
|
||||
fn device_keys(&self) -> Vec<DeviceKey> {
|
||||
self.dks.clone()
|
||||
}
|
||||
fn processing_keys(&self) -> Vec<[u8; 16]> {
|
||||
self.pks.clone()
|
||||
}
|
||||
fn media_keys(&self) -> Vec<[u8; 16]> {
|
||||
self.mks.clone()
|
||||
}
|
||||
fn lookup_disc_by_hash(&self, _h: &[u8; 20]) -> Option<DiscEntry> {
|
||||
self.hash_hit.clone()
|
||||
}
|
||||
fn lookup_disc_by_vid(&self, _v: &[u8; 16]) -> Option<DiscEntry> {
|
||||
self.vid_hit.clone()
|
||||
}
|
||||
}
|
||||
|
||||
// ── KeyProvider default methods all return empty ───────────────────────
|
||||
|
||||
#[test]
|
||||
fn default_provider_methods_return_empty() {
|
||||
// A bare provider that overrides nothing must yield empty material so
|
||||
// the resolver simply finds nothing through it (no surprise hits).
|
||||
struct Empty;
|
||||
impl KeyProvider for Empty {}
|
||||
let e = Empty;
|
||||
assert!(e.device_keys().is_empty());
|
||||
assert!(e.processing_keys().is_empty());
|
||||
assert!(e.media_keys().is_empty());
|
||||
assert!(e.host_certs().is_empty());
|
||||
assert!(e.lookup_disc_by_hash(&[0u8; 20]).is_none());
|
||||
assert!(e.lookup_disc_by_vid(&[0u8; 16]).is_none());
|
||||
}
|
||||
|
||||
// ── Providers::processing_keys: union + dedup ──────────────────────────
|
||||
|
||||
#[test]
|
||||
fn providers_processing_keys_union_and_dedup() {
|
||||
// Two providers each carrying overlapping PKs → the aggregate is the
|
||||
// deduped union (the resolver must not re-validate identical material).
|
||||
let a = Fixed {
|
||||
pks: vec![[0x01u8; 16], [0x02u8; 16]],
|
||||
..Default::default()
|
||||
};
|
||||
let b = Fixed {
|
||||
pks: vec![[0x02u8; 16], [0x03u8; 16]],
|
||||
..Default::default()
|
||||
};
|
||||
let arr: &[&dyn KeyProvider] = &[&a, &b];
|
||||
let mut got = Providers(arr).processing_keys();
|
||||
got.sort();
|
||||
assert_eq!(got, vec![[0x01u8; 16], [0x02u8; 16], [0x03u8; 16]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn providers_media_keys_union_and_dedup() {
|
||||
let a = Fixed {
|
||||
mks: vec![[0xAAu8; 16]],
|
||||
..Default::default()
|
||||
};
|
||||
let b = Fixed {
|
||||
mks: vec![[0xAAu8; 16], [0xBBu8; 16]],
|
||||
..Default::default()
|
||||
};
|
||||
let arr: &[&dyn KeyProvider] = &[&a, &b];
|
||||
let mut got = Providers(arr).media_keys();
|
||||
got.sort();
|
||||
assert_eq!(got, vec![[0xAAu8; 16], [0xBBu8; 16]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn providers_device_keys_dedup_on_value_tuple() {
|
||||
// DeviceKey has no Hash/Ord; dedup keys on (key,node,uv,u_mask_shift).
|
||||
// Two identical DKs across providers collapse to one; a DK differing
|
||||
// only in node is kept.
|
||||
let a = Fixed {
|
||||
dks: vec![dk(0x11, 5), dk(0x11, 5)],
|
||||
..Default::default()
|
||||
};
|
||||
let b = Fixed {
|
||||
dks: vec![dk(0x11, 5), dk(0x11, 6)],
|
||||
..Default::default()
|
||||
};
|
||||
let arr: &[&dyn KeyProvider] = &[&a, &b];
|
||||
let got = Providers(arr).device_keys();
|
||||
assert_eq!(got.len(), 2, "identical DKs dedup; differing node kept");
|
||||
let nodes: Vec<u16> = got.iter().map(|d| d.node).collect();
|
||||
assert!(nodes.contains(&5) && nodes.contains(&6));
|
||||
}
|
||||
|
||||
// ── Disc-keyed lookups: array-order short-circuit ──────────────────────
|
||||
|
||||
#[test]
|
||||
fn providers_lookup_by_hash_first_hit_wins() {
|
||||
// Querying providers in array order, the FIRST hit wins (closest /
|
||||
// fastest first). Provider 0 hits → its entry is returned even though
|
||||
// provider 1 also has one.
|
||||
let a = Fixed {
|
||||
hash_hit: Some(entry("first", 0x01)),
|
||||
..Default::default()
|
||||
};
|
||||
let b = Fixed {
|
||||
hash_hit: Some(entry("second", 0x02)),
|
||||
..Default::default()
|
||||
};
|
||||
let arr: &[&dyn KeyProvider] = &[&a, &b];
|
||||
let got = Providers(arr).lookup_disc_by_hash(&[0u8; 20]).unwrap();
|
||||
assert_eq!(got.disc_hash, "first");
|
||||
assert_eq!(got.vuk, Some([0x01u8; 16]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn providers_lookup_by_hash_falls_through_to_later_provider() {
|
||||
// Provider 0 misses, provider 1 hits → the later provider's entry is
|
||||
// used (find_map continues past None).
|
||||
let a = Fixed::default(); // hash_hit None
|
||||
let b = Fixed {
|
||||
hash_hit: Some(entry("second", 0x02)),
|
||||
..Default::default()
|
||||
};
|
||||
let arr: &[&dyn KeyProvider] = &[&a, &b];
|
||||
let got = Providers(arr).lookup_disc_by_hash(&[0u8; 20]).unwrap();
|
||||
assert_eq!(got.disc_hash, "second");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn providers_lookup_by_vid_first_hit_wins() {
|
||||
let a = Fixed {
|
||||
vid_hit: Some(entry("vid-a", 0x07)),
|
||||
..Default::default()
|
||||
};
|
||||
let b = Fixed {
|
||||
vid_hit: Some(entry("vid-b", 0x08)),
|
||||
..Default::default()
|
||||
};
|
||||
let arr: &[&dyn KeyProvider] = &[&a, &b];
|
||||
let got = Providers(arr).lookup_disc_by_vid(&[0u8; 16]).unwrap();
|
||||
assert_eq!(got.disc_hash, "vid-a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn providers_empty_array_yields_nothing() {
|
||||
let arr: &[&dyn KeyProvider] = &[];
|
||||
let p = Providers(arr);
|
||||
assert!(p.device_keys().is_empty());
|
||||
assert!(p.processing_keys().is_empty());
|
||||
assert!(p.media_keys().is_empty());
|
||||
assert!(p.lookup_disc_by_hash(&[0u8; 20]).is_none());
|
||||
assert!(p.lookup_disc_by_vid(&[0u8; 16]).is_none());
|
||||
}
|
||||
|
||||
// ── SuppliedKey: each level exposes only its own material ──────────────
|
||||
|
||||
#[test]
|
||||
fn supplied_key_exposes_only_populated_fields() {
|
||||
// A SuppliedKey filled at the DK level exposes DKs and nothing else,
|
||||
// so the resolver runs the matching (DK→…) path and no other.
|
||||
let sk = SuppliedKey {
|
||||
device_keys: vec![dk(0x33, 9)],
|
||||
processing_keys: Vec::new(),
|
||||
media_keys: Vec::new(),
|
||||
disc_entry: None,
|
||||
};
|
||||
assert_eq!(sk.device_keys().len(), 1);
|
||||
assert!(sk.processing_keys().is_empty());
|
||||
assert!(sk.media_keys().is_empty());
|
||||
assert!(sk.lookup_disc_by_hash(&[0u8; 20]).is_none());
|
||||
assert!(sk.lookup_disc_by_vid(&[0u8; 16]).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn supplied_key_disc_entry_returned_for_any_hash_or_vid() {
|
||||
// decrypt_with already knows the disc, so a present disc_entry is
|
||||
// returned regardless of the hash/VID argument (the lookup args are
|
||||
// irrelevant in this bridge).
|
||||
let sk = SuppliedKey {
|
||||
device_keys: Vec::new(),
|
||||
processing_keys: Vec::new(),
|
||||
media_keys: Vec::new(),
|
||||
disc_entry: Some(entry("supplied", 0x44)),
|
||||
};
|
||||
// Two unrelated hashes both return the same entry.
|
||||
let h1 = sk.lookup_disc_by_hash(&[0x01u8; 20]).unwrap();
|
||||
let h2 = sk.lookup_disc_by_hash(&[0xFFu8; 20]).unwrap();
|
||||
assert_eq!(h1.disc_hash, "supplied");
|
||||
assert_eq!(h2.disc_hash, "supplied");
|
||||
// And by VID likewise.
|
||||
assert!(sk.lookup_disc_by_vid(&[0x00u8; 16]).is_some());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
//! Structured resolution trace — what the unlock + key-resolution attempt did.
|
||||
//!
|
||||
//! No user-facing English. Every step's STATE is a typed enum variant;
|
||||
//! applications RENDER these into localized text (the library never does). This
|
||||
//! module only DEFINES the shape and is wired through the resolve/handshake
|
||||
//! return path far enough to compile.
|
||||
//!
|
||||
//! The `who` of each step is the source's `label()` / unlocker's `name()` — a
|
||||
//! stable identifier string (a NAME, like a codec id, NOT user-facing prose),
|
||||
//! carried verbatim so an app renderer never has to match an enum back to a name
|
||||
//! it already has. Only the OUTCOME / path enums are structured states the app
|
||||
//! maps to i18n English.
|
||||
|
||||
/// The full trace of a resolution attempt: the unlock phase, then the
|
||||
/// key-resolution phase.
|
||||
#[derive(Debug, Clone, PartialEq, Default)]
|
||||
pub struct ResolutionTrace {
|
||||
/// One step per unlocker consulted, in consultation order.
|
||||
pub unlock: Vec<UnlockStep>,
|
||||
/// One step per key source consulted, in consultation order.
|
||||
pub keys: Vec<KeyStep>,
|
||||
}
|
||||
|
||||
impl ResolutionTrace {
|
||||
/// An empty trace (no steps recorded).
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
// ── Unlock phase ────────────────────────────────────────────────────────────
|
||||
|
||||
/// One unlocker's contribution to the unlock phase. `who` is the unlocker's
|
||||
/// `name()` (a stable identifier, e.g. `"LibreDrive"`), carried verbatim.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UnlockStep {
|
||||
pub who: String,
|
||||
pub outcome: UnlockOutcome,
|
||||
}
|
||||
|
||||
/// What an unlocker did.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum UnlockOutcome {
|
||||
/// The drive was unlocked (or already usable) and a VID is available.
|
||||
Unlocked,
|
||||
/// This unlocker cannot unlock this drive's firmware.
|
||||
FirmwareNotUnlockable,
|
||||
/// No non-revoked host cert was usable for the auth attempt. `mkb` is the
|
||||
/// disc MKB generation when known.
|
||||
NoUsableHostCert { mkb: Option<u32> },
|
||||
/// Every available host cert was revoked on this drive's HRL. `mkb` is the
|
||||
/// disc MKB generation when known.
|
||||
CertRevoked { mkb: Option<u32> },
|
||||
/// The drive rejected the auth handshake (non-revocation rejection / wedge).
|
||||
HandshakeRejected,
|
||||
/// Auth succeeded (or was skipped) but the Volume ID could not be read.
|
||||
VidUnavailable,
|
||||
}
|
||||
|
||||
// ── Key-resolution phase ────────────────────────────────────────────────────
|
||||
|
||||
/// One key source's contribution to the key-resolution phase, including the
|
||||
/// derivation path it walked. `who` is the source's `label()` (a stable
|
||||
/// identifier, e.g. `"keydb"` / `"online"`), carried verbatim.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct KeyStep {
|
||||
pub who: String,
|
||||
pub path: Vec<KeyNode>,
|
||||
pub outcome: KeyOutcome,
|
||||
}
|
||||
|
||||
/// A node on the derivation path a source walked. Ordered as encountered; not
|
||||
/// every path hits every node.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum KeyNode {
|
||||
/// The source matched this disc (by hash / VID).
|
||||
MatchedDisc,
|
||||
/// The source had no entry for this disc.
|
||||
NoEntry,
|
||||
/// Pre-decrypted unit keys were found.
|
||||
FoundUnitKeys,
|
||||
/// A VUK was found.
|
||||
FoundVuk,
|
||||
/// A Media Key was found.
|
||||
FoundMediaKey,
|
||||
/// A VID is required to proceed.
|
||||
NeedVid,
|
||||
/// The VID came from the unlock phase.
|
||||
VidFromUnlock,
|
||||
/// The VID came from the keydb entry.
|
||||
VidFromKeydb,
|
||||
/// No VID was available.
|
||||
NoVid,
|
||||
/// A VUK was derived (from MK + VID).
|
||||
DerivedVuk,
|
||||
/// Unit keys were derived (from VUK).
|
||||
DerivedUnitKeys,
|
||||
}
|
||||
|
||||
/// The terminal outcome of a source's resolution attempt.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum KeyOutcome {
|
||||
/// Usable unit keys were produced.
|
||||
Resolved,
|
||||
/// Derivation material existed but no VID was available to finish.
|
||||
MissingVid,
|
||||
/// No usable key from this source.
|
||||
NoKey,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The trace types are constructible, derive the required traits, and an
|
||||
/// empty trace round-trips. Pins the structural contract apps build against.
|
||||
#[test]
|
||||
fn trace_is_constructible_and_comparable() {
|
||||
let t = ResolutionTrace {
|
||||
unlock: vec![UnlockStep {
|
||||
who: "AACS cert".to_string(),
|
||||
outcome: UnlockOutcome::NoUsableHostCert { mkb: Some(68) },
|
||||
}],
|
||||
keys: vec![KeyStep {
|
||||
who: "keydb".to_string(),
|
||||
path: vec![
|
||||
KeyNode::MatchedDisc,
|
||||
KeyNode::FoundVuk,
|
||||
KeyNode::DerivedUnitKeys,
|
||||
],
|
||||
outcome: KeyOutcome::Resolved,
|
||||
}],
|
||||
};
|
||||
// Clone + PartialEq (derive contract the renderers rely on).
|
||||
assert_eq!(t.clone(), t);
|
||||
// `who` is the source's name carried verbatim.
|
||||
assert_eq!(t.keys[0].who, "keydb");
|
||||
assert_eq!(t.unlock[0].who, "AACS cert");
|
||||
// Default / new is empty.
|
||||
assert_eq!(ResolutionTrace::new(), ResolutionTrace::default());
|
||||
assert!(ResolutionTrace::new().unlock.is_empty());
|
||||
assert!(ResolutionTrace::new().keys.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
//! AACS primitive types shared across the resolve chain.
|
||||
//!
|
||||
//! These structs describe AACS key material (device keys, host
|
||||
//! certificates, per-disc entries). They carry no parsing logic — the
|
||||
//! keydb.cfg format lives in the `freemkv-keysources` crate. libfreemkv
|
||||
//! owns only the crypto and these value types that flow through it.
|
||||
|
||||
/// A device key for MKB subset-difference tree processing.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DeviceKey {
|
||||
pub key: [u8; 16],
|
||||
pub node: u16,
|
||||
pub uv: u32,
|
||||
pub u_mask_shift: u8,
|
||||
}
|
||||
|
||||
/// Host certificate + private key for AACS SCSI authentication.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HostCert {
|
||||
/// AACS 1.0: 20 bytes. AACS 2.0: 32 bytes.
|
||||
pub private_key: [u8; 20],
|
||||
/// AACS 1.0: 92 bytes. AACS 2.0: 132 bytes.
|
||||
pub certificate: Vec<u8>,
|
||||
/// AACS 2.0 host private key (P-256, 32 bytes). None for AACS 1.0 only.
|
||||
pub private_key_v2: Option<[u8; 32]>,
|
||||
/// AACS 2.0 host certificate (type 0x11). None for AACS 1.0 only.
|
||||
pub certificate_v2: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
/// A per-disc entry from the key database.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DiscEntry {
|
||||
/// Disc hash (20 bytes, hex)
|
||||
pub disc_hash: String,
|
||||
/// Disc title
|
||||
pub title: String,
|
||||
/// Media Key (16 bytes) — from MKB processing
|
||||
pub media_key: Option<[u8; 16]>,
|
||||
/// Disc ID (16 bytes)
|
||||
pub disc_id: Option<[u8; 16]>,
|
||||
/// Volume Unique Key (16 bytes) — decrypts title keys
|
||||
pub vuk: Option<[u8; 16]>,
|
||||
/// Unit keys (title keys) indexed by CPS unit number
|
||||
pub unit_keys: Vec<(u32, [u8; 16])>,
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+755
-28
@@ -6,52 +6,112 @@
|
||||
//!
|
||||
//! Reference: https://github.com/lw/BluRay/wiki/CLPI
|
||||
|
||||
use crate::consts::{BD_SOURCE_PACKET_BYTES, SECTOR_BYTES_U64};
|
||||
use crate::disc::Extent;
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
/// Parsed CLPI clip info.
|
||||
#[derive(Debug)]
|
||||
#[allow(dead_code)]
|
||||
pub struct ClipInfo {
|
||||
pub(crate) struct ClipInfo {
|
||||
/// CLPI version string. Parsed for completeness; not yet consumed.
|
||||
#[allow(dead_code)]
|
||||
pub version: String,
|
||||
/// Total source packets in the m2ts (each 192 bytes)
|
||||
pub source_packet_count: u32,
|
||||
/// Coarse EP entries for the primary video stream
|
||||
/// Coarse EP entries for the primary video stream. Populated for the
|
||||
/// EP-map → sector-extent lookup (`get_extents`), which is exercised by
|
||||
/// tests and reserved for the timestamp-range read path.
|
||||
#[allow(dead_code)]
|
||||
pub ep_coarse: Vec<EpCoarse>,
|
||||
/// Fine EP entries for the primary video stream
|
||||
/// Fine EP entries for the primary video stream (see `ep_coarse`).
|
||||
#[allow(dead_code)]
|
||||
pub ep_fine: Vec<EpFine>,
|
||||
/// Per-stream metadata from the ProgramInfo section (BD spec).
|
||||
/// Cross-validates the MPLS STN view — see `labels/clpi_audit.rs`.
|
||||
/// Empty when program_info is missing or malformed.
|
||||
pub streams: Vec<ClpiStream>,
|
||||
}
|
||||
|
||||
/// One stream descriptor from the CLPI ProgramInfo / stream_coding_info
|
||||
/// table. Mirrors the same fields the MPLS STN table carries — see
|
||||
/// `mpls::StreamEntry` for the playlist-side equivalent.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ClpiStream {
|
||||
/// PID of the stream in the MPEG-TS (matches MPLS).
|
||||
pub pid: u16,
|
||||
/// BD stream coding type byte (0x80 LPCM, 0x83 TrueHD, 0x86 DTS-HD MA,
|
||||
/// 0x90 PG, etc.). See `labels::mpls_universal::coding_type_to_codec_hint`.
|
||||
pub coding_type: u8,
|
||||
/// ISO 639-2 3-char language code. Empty for video streams.
|
||||
pub language: String,
|
||||
// The CLPI cross-validation consumer (labels/clpi_audit.rs) reads only
|
||||
// pid/coding_type/language. The codec sub-fields below are parsed from
|
||||
// the BD stream_coding_info for completeness but have no reader yet.
|
||||
/// Audio format byte (1=mono, 3=stereo, 6=5.1, 12=7.1).
|
||||
/// Zero for non-audio streams.
|
||||
#[allow(dead_code)]
|
||||
pub audio_format: u8,
|
||||
/// Audio sample rate (1=48kHz, 4=96kHz, 5=192kHz). Zero for non-audio.
|
||||
#[allow(dead_code)]
|
||||
pub audio_rate: u8,
|
||||
/// Video format byte (1=480i, 4=1080i, 5=720p, 6=1080p, 8=2160p).
|
||||
/// Zero for non-video.
|
||||
#[allow(dead_code)]
|
||||
pub video_format: u8,
|
||||
/// Video rate (1=23.976, 2=24, 3=25, 4=29.97, 6=50, 7=59.94).
|
||||
#[allow(dead_code)]
|
||||
pub video_rate: u8,
|
||||
}
|
||||
|
||||
/// Coarse EP-map entry. Fields feed the EP-map resolution used by
|
||||
/// `get_extents` (test-exercised; reserved for the timestamp-range path).
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct EpCoarse {
|
||||
pub(crate) struct EpCoarse {
|
||||
pub ref_to_fine_id: u32,
|
||||
pub pts_coarse: u32,
|
||||
pub spn_coarse: u32,
|
||||
}
|
||||
|
||||
/// Fine EP-map entry (see `EpCoarse`).
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct EpFine {
|
||||
pub(crate) struct EpFine {
|
||||
pub pts_fine: u32,
|
||||
pub spn_fine: u32,
|
||||
}
|
||||
|
||||
// EP-map → sector-extent resolution. Exercised by the unit tests and
|
||||
// reserved for the timestamp-range read path; no production caller yet.
|
||||
#[allow(dead_code)]
|
||||
impl ClipInfo {
|
||||
/// Reconstruct full PTS from coarse + fine entry.
|
||||
pub fn full_pts(coarse: &EpCoarse, fine: &EpFine) -> u32 {
|
||||
(coarse.pts_coarse << 19) + (fine.pts_fine << 8)
|
||||
///
|
||||
/// The BD spec PTS is 33-bit: `pts_coarse` is 14 bits (max 16383) and
|
||||
/// `16383 << 19` exceeds `u32::MAX`, so the result must be `u64` to
|
||||
/// avoid overflow (panic in debug, silent wrap in release).
|
||||
pub fn full_pts(coarse: &EpCoarse, fine: &EpFine) -> u64 {
|
||||
((coarse.pts_coarse as u64) << 19) + ((fine.pts_fine as u64) << 8)
|
||||
}
|
||||
|
||||
/// Reconstruct full SPN from coarse + fine entry.
|
||||
pub fn full_spn(coarse: &EpCoarse, fine: &EpFine) -> u32 {
|
||||
(coarse.spn_coarse & 0xFFFE_0000) + fine.spn_fine
|
||||
// The two operands occupy non-overlapping bit ranges (coarse holds
|
||||
// the high bits, fine the low 17), so OR expresses intent and is
|
||||
// robust to a hand-constructed EpFine.
|
||||
debug_assert!(fine.spn_fine <= 0x1_FFFF);
|
||||
(coarse.spn_coarse & 0xFFFE_0000) | fine.spn_fine
|
||||
}
|
||||
|
||||
/// Get all EP entries as (PTS, SPN) pairs, fully resolved.
|
||||
pub fn resolved_ep_map(&self) -> Vec<(u32, u32)> {
|
||||
let mut entries = Vec::new();
|
||||
///
|
||||
/// PTS resets at each coarse-group boundary on disc, so the raw
|
||||
/// concatenation is not globally monotonic. The returned vector is
|
||||
/// sorted by PTS so callers (e.g. [`get_extents`]) can binary-search it.
|
||||
///
|
||||
/// [`get_extents`]: ClipInfo::get_extents
|
||||
pub fn resolved_ep_map(&self) -> Vec<(u64, u32)> {
|
||||
let mut entries = Vec::with_capacity(self.ep_fine.len());
|
||||
|
||||
for (ci, coarse) in self.ep_coarse.iter().enumerate() {
|
||||
let fine_start = coarse.ref_to_fine_id as usize;
|
||||
@@ -69,6 +129,12 @@ impl ClipInfo {
|
||||
}
|
||||
}
|
||||
|
||||
// get_extents binary-searches by PTS, so the map must be ordered.
|
||||
// Real discs have globally increasing PTS in coarse order; sort by
|
||||
// (pts, spn) so a cross-group PTS collision can't leave the search
|
||||
// landing on the wrong group's SPN.
|
||||
entries.sort_by_key(|&(pts, spn)| (pts, spn));
|
||||
|
||||
entries
|
||||
}
|
||||
|
||||
@@ -76,7 +142,9 @@ impl ClipInfo {
|
||||
///
|
||||
/// Converts PTS timestamps to SPN ranges, then SPN to LBA
|
||||
/// using the file's starting LBA on disc.
|
||||
pub fn get_extents(&self, in_time: u32, out_time: u32) -> Vec<Extent> {
|
||||
pub fn get_extents(&self, in_time: u64, out_time: u64) -> Vec<Extent> {
|
||||
// resolved_ep_map() returns entries sorted by PTS, so binary search
|
||||
// is valid here.
|
||||
let ep_map = self.resolved_ep_map();
|
||||
if ep_map.is_empty() {
|
||||
return Vec::new();
|
||||
@@ -93,20 +161,22 @@ impl ClipInfo {
|
||||
let end_spn = match ep_map.binary_search_by_key(&out_time, |(pts, _)| *pts) {
|
||||
Ok(i) => ep_map[i].1,
|
||||
Err(i) if i < ep_map.len() => ep_map[i].1,
|
||||
_ => ep_map.last().unwrap().1 + 1,
|
||||
_ => ep_map.last().unwrap().1.saturating_add(1),
|
||||
};
|
||||
|
||||
if end_spn <= start_spn {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// SPN → byte offset: spn × 192
|
||||
// Byte offset → sectors: offset / 2048
|
||||
// Note: the caller needs to add the file's starting LBA from UDF
|
||||
let start_byte = start_spn as u64 * 192;
|
||||
let end_byte = end_spn as u64 * 192;
|
||||
let start_sector = (start_byte / 2048) as u32;
|
||||
let end_sector = end_byte.div_ceil(2048) as u32;
|
||||
// SPN → byte offset → sector range. Note: the caller adds the file's
|
||||
// starting LBA from UDF. The start sector FLOORS (the extent begins in
|
||||
// whichever sector contains its first byte) and the end sector CEILS
|
||||
// (the extent must cover through the sector holding its last byte), so
|
||||
// a sub-sector-aligned range still spans every sector it touches.
|
||||
let start_byte = start_spn as u64 * BD_SOURCE_PACKET_BYTES as u64;
|
||||
let end_byte = end_spn as u64 * BD_SOURCE_PACKET_BYTES as u64;
|
||||
let start_sector = (start_byte / SECTOR_BYTES_U64) as u32;
|
||||
let end_sector = end_byte.div_ceil(SECTOR_BYTES_U64) as u32;
|
||||
|
||||
vec![Extent {
|
||||
start_lba: start_sector, // relative to m2ts file start
|
||||
@@ -128,17 +198,27 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
|
||||
|
||||
// Header offsets
|
||||
let _seq_info_start = u32::from_be_bytes([data[8], data[9], data[10], data[11]]) as usize;
|
||||
let _prog_info_start = u32::from_be_bytes([data[12], data[13], data[14], data[15]]) as usize;
|
||||
let prog_info_start = u32::from_be_bytes([data[12], data[13], data[14], data[15]]) as usize;
|
||||
let cpi_start = u32::from_be_bytes([data[16], data[17], data[18], data[19]]) as usize;
|
||||
|
||||
// ClipInfo section at offset 40
|
||||
// source_packet_count at offset 40 + 4(len) + 2(reserved) + 1(stream_type) + 1(app_type) + 4(reserved) + 4(ts_rate)
|
||||
let source_packet_count = if data.len() > 56 {
|
||||
let source_packet_count = if data.len() >= 60 {
|
||||
u32::from_be_bytes([data[56], data[57], data[58], data[59]])
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
// Parse ProgramInfo (per-stream language + codec). Best-effort:
|
||||
// malformed program_info doesn't fail the parse, just gives an
|
||||
// empty streams list. EP map is unaffected — sector-range lookups
|
||||
// continue to work.
|
||||
let streams = if prog_info_start > 0 && prog_info_start + 6 < data.len() {
|
||||
parse_program_info(&data[prog_info_start..])
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
// Parse CPI / EP Map
|
||||
let (ep_coarse, ep_fine) = if cpi_start > 0 && cpi_start + 8 < data.len() {
|
||||
parse_cpi(&data[cpi_start..])?
|
||||
@@ -151,9 +231,127 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
|
||||
source_packet_count,
|
||||
ep_coarse,
|
||||
ep_fine,
|
||||
streams,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse the ProgramInfo section: per-stream (pid, coding_type,
|
||||
/// language, codec sub-fields). Layout per BD spec / libbluray
|
||||
/// clpi_parse.c:
|
||||
///
|
||||
/// ```text
|
||||
/// ProgramInfo:
|
||||
/// length: 4 bytes
|
||||
/// reserved: 1 byte
|
||||
/// num_programs: 1 byte
|
||||
/// for each program:
|
||||
/// spn_program_sequence_start: 4 bytes
|
||||
/// program_map_pid: 2 bytes
|
||||
/// num_streams: 1 byte
|
||||
/// num_groups: 1 byte
|
||||
/// for each stream:
|
||||
/// pid: 2 bytes
|
||||
/// stream_coding_info_length: 1 byte
|
||||
/// stream_coding_info: (varies by coding_type)
|
||||
/// coding_type: 1 byte
|
||||
/// per-type bytes (see match arms below)
|
||||
/// ```
|
||||
///
|
||||
/// Returns `Vec::new()` on any structural mismatch — we don't propagate
|
||||
/// errors because the EP map is the primary CLPI output, and a corrupt
|
||||
/// program_info shouldn't break sector-range lookups.
|
||||
fn parse_program_info(data: &[u8]) -> Vec<ClpiStream> {
|
||||
use crate::consts::coding_type as c;
|
||||
let mut out = Vec::new();
|
||||
if data.len() < 6 {
|
||||
return out;
|
||||
}
|
||||
// length: 4 bytes (skipped — we trust the section bounds in the
|
||||
// caller's slice and read the bytes that follow). Reserved 1 byte
|
||||
// at offset 4. num_programs at offset 5.
|
||||
let num_programs = data[5] as usize;
|
||||
let mut pos = 6usize;
|
||||
for _ in 0..num_programs {
|
||||
// Program header: 4 (spn) + 2 (pmt_pid) + 1 (num_streams) + 1 (num_groups) = 8 bytes
|
||||
if pos + 8 > data.len() {
|
||||
return out;
|
||||
}
|
||||
let num_streams = data[pos + 6] as usize;
|
||||
pos += 8;
|
||||
|
||||
for _ in 0..num_streams {
|
||||
// Stream header: 2 (pid) + 1 (sci_length) + sci bytes
|
||||
if pos + 3 > data.len() {
|
||||
return out;
|
||||
}
|
||||
let pid = u16::from_be_bytes([data[pos], data[pos + 1]]);
|
||||
let sci_len = data[pos + 2] as usize;
|
||||
let sci_end = pos + 3 + sci_len;
|
||||
if sci_end > data.len() || sci_len < 1 {
|
||||
return out;
|
||||
}
|
||||
let sci = &data[pos + 3..sci_end];
|
||||
let coding_type = sci[0];
|
||||
|
||||
let mut audio_format = 0u8;
|
||||
let mut audio_rate = 0u8;
|
||||
let mut video_format = 0u8;
|
||||
let mut video_rate = 0u8;
|
||||
let mut language = String::new();
|
||||
|
||||
match coding_type {
|
||||
// Video — MPEG-2, H.264, HEVC
|
||||
c::MPEG2_VIDEO | c::H264 | c::HEVC => {
|
||||
if sci.len() >= 2 {
|
||||
video_format = (sci[1] >> 4) & 0x0F;
|
||||
video_rate = sci[1] & 0x0F;
|
||||
}
|
||||
}
|
||||
// Primary audio — LPCM, AC-3, DTS, TrueHD, AC-3+, DTS-HD HR, DTS-HD MA
|
||||
c::LPCM..=c::DTS_HD_MA => {
|
||||
if sci.len() >= 2 {
|
||||
audio_format = (sci[1] >> 4) & 0x0F;
|
||||
audio_rate = sci[1] & 0x0F;
|
||||
}
|
||||
if sci.len() >= 5 {
|
||||
language = String::from_utf8_lossy(&sci[2..5]).to_string();
|
||||
}
|
||||
}
|
||||
// Secondary audio (AC-3+ secondary, DTS-HD secondary)
|
||||
c::AC3_PLUS_SECONDARY | c::DTS_HD_SECONDARY => {
|
||||
if sci.len() >= 2 {
|
||||
audio_format = (sci[1] >> 4) & 0x0F;
|
||||
audio_rate = sci[1] & 0x0F;
|
||||
}
|
||||
if sci.len() >= 5 {
|
||||
language = String::from_utf8_lossy(&sci[2..5]).to_string();
|
||||
}
|
||||
}
|
||||
// PG, IG: coding_type + 3-byte language [+ char_code for PG]
|
||||
c::PG | c::IG => {
|
||||
if sci.len() >= 4 {
|
||||
language = String::from_utf8_lossy(&sci[1..4]).to_string();
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
out.push(ClpiStream {
|
||||
pid,
|
||||
coding_type,
|
||||
language,
|
||||
audio_format,
|
||||
audio_rate,
|
||||
video_format,
|
||||
video_rate,
|
||||
});
|
||||
|
||||
pos = sci_end;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Parse the CPI section containing the EP map.
|
||||
fn parse_cpi(data: &[u8]) -> Result<(Vec<EpCoarse>, Vec<EpFine>)> {
|
||||
if data.len() < 8 {
|
||||
@@ -165,8 +363,17 @@ fn parse_cpi(data: &[u8]) -> Result<(Vec<EpCoarse>, Vec<EpFine>)> {
|
||||
return Ok((Vec::new(), Vec::new()));
|
||||
}
|
||||
|
||||
// Bound all EP-map reads to this CPI section. The length field counts
|
||||
// bytes after itself, so the section spans data[..cpi_length + 4]. A
|
||||
// bogus ep_map_offset within data.len() but past the CPI section would
|
||||
// otherwise read into an adjacent CLPI section; clamp first.
|
||||
let data = &data[..(cpi_length + 4).min(data.len())];
|
||||
|
||||
// CPI type at bits 44-47 (byte 5, lower 4 bits)
|
||||
// Skip to EP map: offset 4 (after length) + 2 (reserved/type)
|
||||
if data.len() < 6 {
|
||||
return Ok((Vec::new(), Vec::new()));
|
||||
}
|
||||
let ep_map = &data[6..];
|
||||
if ep_map.len() < 4 {
|
||||
return Ok((Vec::new(), Vec::new()));
|
||||
@@ -194,9 +401,6 @@ fn parse_cpi(data: &[u8]) -> Result<(Vec<EpCoarse>, Vec<EpFine>)> {
|
||||
// num_EP_coarse: 16 bits │ (10+4+16+18+32 = 80)
|
||||
// num_EP_fine: 18 bits │
|
||||
// EP_map_start_address: 32 bits ┘
|
||||
if ep_map.len() < 16 {
|
||||
return Ok((Vec::new(), Vec::new()));
|
||||
}
|
||||
let _stream_pid = u16::from_be_bytes([ep_map[2], ep_map[3]]);
|
||||
|
||||
// Read 10 bytes (80 bits) from ep_map[4..14] for bit extraction
|
||||
@@ -232,7 +436,10 @@ fn parse_cpi(data: &[u8]) -> Result<(Vec<EpCoarse>, Vec<EpFine>)> {
|
||||
|
||||
// Coarse entries start at offset 4, 8 bytes each
|
||||
let coarse_data = &stream_ep[4..];
|
||||
let mut ep_coarse = Vec::with_capacity(num_coarse);
|
||||
// Cap the pre-reservation by what the slice can actually hold:
|
||||
// num_coarse is a 16-bit disc field, so a hostile value would
|
||||
// otherwise reserve up to ~0.5 MB for an entry table that doesn't exist.
|
||||
let mut ep_coarse = Vec::with_capacity(num_coarse.min(coarse_data.len() / 8));
|
||||
for i in 0..num_coarse {
|
||||
let off = i * 8;
|
||||
if off + 8 > coarse_data.len() {
|
||||
@@ -262,7 +469,13 @@ fn parse_cpi(data: &[u8]) -> Result<(Vec<EpCoarse>, Vec<EpFine>)> {
|
||||
}
|
||||
|
||||
// Fine entries at fine_start, 4 bytes each
|
||||
let mut ep_fine = Vec::with_capacity(num_fine);
|
||||
// Cap the pre-reservation: num_fine is an 18-bit disc field (max
|
||||
// 262143), so reserve only what the slice can actually hold.
|
||||
let mut ep_fine = if fine_start < stream_ep.len() {
|
||||
Vec::with_capacity(num_fine.min((stream_ep.len() - fine_start) / 4))
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
if fine_start < stream_ep.len() {
|
||||
let fine_data = &stream_ep[fine_start..];
|
||||
for i in 0..num_fine {
|
||||
@@ -486,10 +699,49 @@ mod tests {
|
||||
};
|
||||
// full_pts = (100 << 19) + (50 << 8) = 52_428_800 + 12_800 = 52_441_600
|
||||
let pts = ClipInfo::full_pts(&coarse, &fine);
|
||||
assert_eq!(pts, (100 << 19) + (50 << 8));
|
||||
assert_eq!(pts, (100u64 << 19) + (50u64 << 8));
|
||||
assert_eq!(pts, 52_441_600);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_pts_no_u32_overflow() {
|
||||
// pts_coarse is a 14-bit field (max 0x3FFF = 16383); 16383 << 19
|
||||
// overflows u32, so full_pts must use u64.
|
||||
let coarse = EpCoarse {
|
||||
ref_to_fine_id: 0,
|
||||
pts_coarse: 0x3FFF,
|
||||
spn_coarse: 0,
|
||||
};
|
||||
let fine = EpFine {
|
||||
pts_fine: 0x7FF,
|
||||
spn_fine: 0,
|
||||
};
|
||||
let pts = ClipInfo::full_pts(&coarse, &fine);
|
||||
assert_eq!(pts, (0x3FFFu64 << 19) + (0x7FFu64 << 8));
|
||||
assert!(pts > u32::MAX as u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolved_ep_map_sorted_for_binary_search() {
|
||||
// Two coarse groups whose fine PTS reset across the boundary
|
||||
// (50,100 then 25,75) produce a non-monotonic raw concatenation.
|
||||
// resolved_ep_map must sort so get_extents' binary search is valid.
|
||||
let cpi = build_cpi(
|
||||
0x1011,
|
||||
&[(0, 0, 0x00020000), (2, 0, 0x00040000)],
|
||||
&[(50, 1024), (100, 2048), (25, 512), (75, 1536)],
|
||||
);
|
||||
let data = build_clpi(1_000_000, Some(&cpi));
|
||||
let clip = parse(&data).expect("should parse");
|
||||
|
||||
let resolved = clip.resolved_ep_map();
|
||||
assert_eq!(resolved.len(), 4);
|
||||
// Strictly sorted by PTS.
|
||||
for w in resolved.windows(2) {
|
||||
assert!(w[0].0 <= w[1].0, "ep_map not sorted: {resolved:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_spn_calculation() {
|
||||
let coarse = EpCoarse {
|
||||
@@ -517,6 +769,22 @@ mod tests {
|
||||
assert_eq!(spn2, 0x00FE0000 + 0x1234);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_truncated_clipinfo_no_panic() {
|
||||
// 57/58/59-byte CLPI with valid magic: passes the data.len() < 40
|
||||
// guard but data[56..60] needs 60 bytes. Must not panic.
|
||||
for len in 40..60usize {
|
||||
let mut data = vec![0u8; len];
|
||||
data[0..4].copy_from_slice(b"HDMV");
|
||||
if len >= 8 {
|
||||
data[4..8].copy_from_slice(b"0200");
|
||||
}
|
||||
let clip = parse(&data).expect("short CLPI should parse, not panic");
|
||||
// source_packet_count is unreadable below 60 bytes → 0.
|
||||
assert_eq!(clip.source_packet_count, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_invalid_magic() {
|
||||
let mut data = build_clpi(1000, None);
|
||||
@@ -550,4 +818,463 @@ mod tests {
|
||||
assert!(clip2.ep_coarse.is_empty());
|
||||
assert!(clip2.ep_fine.is_empty());
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// Added hardening tests. Grounded in the BD-ROM CLPI spec
|
||||
// (https://github.com/lw/BluRay/wiki/CLPI) and libbluray clpi_parse.c.
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Build a ProgramInfo section. `streams` = Vec<(pid, sci_bytes)>.
|
||||
/// Layout per source doc: length(4)+reserved(1)+num_programs(1)+
|
||||
/// per program [spn(4)+pmt_pid(2)+num_streams(1)+num_groups(1)] then
|
||||
/// per stream [pid(2)+sci_len(1)+sci].
|
||||
fn build_program_info(streams: &[(u16, Vec<u8>)]) -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.push(0); // reserved (offset 4)
|
||||
body.push(1); // num_programs = 1 (offset 5)
|
||||
// program 0 header (8 bytes)
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // spn_program_sequence_start
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // program_map_pid
|
||||
body.push(streams.len() as u8); // num_streams
|
||||
body.push(0); // num_groups
|
||||
for (pid, sci) in streams {
|
||||
body.extend_from_slice(&pid.to_be_bytes());
|
||||
body.push(sci.len() as u8);
|
||||
body.extend_from_slice(sci);
|
||||
}
|
||||
// Prepend length(4) = bytes after the length field.
|
||||
let mut out = Vec::new();
|
||||
out.extend_from_slice(&(body.len() as u32).to_be_bytes());
|
||||
out.extend_from_slice(&body);
|
||||
out
|
||||
}
|
||||
|
||||
/// Build a CLPI with a ProgramInfo section. prog_info_start is placed
|
||||
/// right after the 60-byte header; cpi (if any) follows program_info.
|
||||
fn build_clpi_with_proginfo(
|
||||
source_packet_count: u32,
|
||||
prog_info: &[u8],
|
||||
cpi_data: Option<&[u8]>,
|
||||
) -> Vec<u8> {
|
||||
let mut buf = vec![0u8; 60];
|
||||
buf[0..4].copy_from_slice(b"HDMV");
|
||||
buf[4..8].copy_from_slice(b"0200");
|
||||
let prog_info_start: u32 = 60;
|
||||
buf[12..16].copy_from_slice(&prog_info_start.to_be_bytes());
|
||||
let cpi_start: u32 = if cpi_data.is_some() {
|
||||
(60 + prog_info.len()) as u32
|
||||
} else {
|
||||
0
|
||||
};
|
||||
buf[16..20].copy_from_slice(&cpi_start.to_be_bytes());
|
||||
buf[56..60].copy_from_slice(&source_packet_count.to_be_bytes());
|
||||
buf.extend_from_slice(prog_info);
|
||||
if let Some(cpi) = cpi_data {
|
||||
buf.extend_from_slice(cpi);
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
/// source_packet_count is a big-endian u32 at offset [56..60]. Verify
|
||||
/// BE decode of a value with all four bytes distinct (not LE / wrong
|
||||
/// offset).
|
||||
#[test]
|
||||
fn source_packet_count_big_endian_offset_56() {
|
||||
let data = build_clpi(0x01020304, None);
|
||||
let clip = parse(&data).expect("should parse");
|
||||
assert_eq!(clip.source_packet_count, 0x01020304);
|
||||
}
|
||||
|
||||
/// Magic must be exactly "HDMV" at [0..4]. Anything else → ClpiParse.
|
||||
/// Spec: CLPI files begin with the type_indicator "HDMV".
|
||||
#[test]
|
||||
fn wrong_magic_rejected() {
|
||||
let mut data = build_clpi(1000, None);
|
||||
data[0..4].copy_from_slice(b"INDX");
|
||||
assert!(parse(&data).is_err());
|
||||
}
|
||||
|
||||
/// Under-40-byte input is rejected before any field read
|
||||
/// (`data.len() < 40` guard).
|
||||
#[test]
|
||||
fn under_40_bytes_rejected() {
|
||||
assert!(parse(&[0u8; 39]).is_err());
|
||||
assert!(parse(b"HDMV0200").is_err());
|
||||
assert!(parse(&[]).is_err());
|
||||
}
|
||||
|
||||
/// ProgramInfo: a video stream (coding 0x1B = H.264) carries
|
||||
/// format/rate in sci[1] nibbles and NO language. Verify the video
|
||||
/// arm: format hi-nibble, rate lo-nibble, language stays empty.
|
||||
#[test]
|
||||
fn program_info_video_stream() {
|
||||
// sci = coding_type(0x1B) + format_rate(0x61 → fmt 6, rate 1)
|
||||
let sci = vec![0x1Bu8, 0x61];
|
||||
let pi = build_program_info(&[(0x1011, sci)]);
|
||||
let data = build_clpi_with_proginfo(100, &pi, None);
|
||||
let clip = parse(&data).expect("should parse");
|
||||
assert_eq!(clip.streams.len(), 1);
|
||||
assert_eq!(clip.streams[0].pid, 0x1011);
|
||||
assert_eq!(clip.streams[0].coding_type, 0x1B);
|
||||
assert_eq!(clip.streams[0].video_format, 6);
|
||||
assert_eq!(clip.streams[0].video_rate, 1);
|
||||
assert_eq!(clip.streams[0].language, "");
|
||||
}
|
||||
|
||||
/// ProgramInfo primary-audio (coding 0x80..=0x86): sci[1] = format/rate
|
||||
/// nibbles, sci[2..5] = ISO 639 language. Verify TrueHD (0x83) at
|
||||
/// offset, 5.1 / 48kHz, language "eng".
|
||||
#[test]
|
||||
fn program_info_audio_stream_lang_offset() {
|
||||
// sci = 0x83 + 0x61 (fmt 6, rate 1) + "eng"
|
||||
let sci = vec![0x83u8, 0x61, b'e', b'n', b'g'];
|
||||
let pi = build_program_info(&[(0x1100, sci)]);
|
||||
let data = build_clpi_with_proginfo(100, &pi, None);
|
||||
let clip = parse(&data).expect("should parse");
|
||||
assert_eq!(clip.streams[0].coding_type, 0x83);
|
||||
assert_eq!(clip.streams[0].audio_format, 6);
|
||||
assert_eq!(clip.streams[0].audio_rate, 1);
|
||||
assert_eq!(clip.streams[0].language, "eng");
|
||||
}
|
||||
|
||||
/// ProgramInfo PG (0x90)/IG (0x91): layout is coding_type(1)+lang(3),
|
||||
/// so language is at sci[1..4] (NOT sci[2..5] like audio). Verify the
|
||||
/// PG arm reads from the right offset.
|
||||
#[test]
|
||||
fn program_info_pg_lang_offset() {
|
||||
// sci = 0x90 + "fra" (lang directly after coding_type)
|
||||
let sci = vec![0x90u8, b'f', b'r', b'a'];
|
||||
let pi = build_program_info(&[(0x1200, sci)]);
|
||||
let data = build_clpi_with_proginfo(100, &pi, None);
|
||||
let clip = parse(&data).expect("should parse");
|
||||
assert_eq!(clip.streams[0].coding_type, 0x90);
|
||||
assert_eq!(clip.streams[0].language, "fra");
|
||||
// Audio nibbles must NOT be populated for a PG stream.
|
||||
assert_eq!(clip.streams[0].audio_format, 0);
|
||||
}
|
||||
|
||||
/// ProgramInfo with multiple streams: PID and coding for each must be
|
||||
/// read from the correct per-stream offset (pid(2)+sci_len(1)+sci).
|
||||
/// Three mixed streams must all parse with distinct PIDs in order.
|
||||
#[test]
|
||||
fn program_info_multiple_streams_advance_correctly() {
|
||||
let v = (0x1011u16, vec![0x24u8, 0x81]); // HEVC video
|
||||
let a = (0x1100u16, vec![0x86u8, 0x61, b'e', b'n', b'g']); // DTS-HD MA
|
||||
let s = (0x1200u16, vec![0x90u8, b'j', b'p', b'n']); // PG
|
||||
let pi = build_program_info(&[v, a, s]);
|
||||
let data = build_clpi_with_proginfo(100, &pi, None);
|
||||
let clip = parse(&data).expect("should parse");
|
||||
assert_eq!(clip.streams.len(), 3);
|
||||
assert_eq!(clip.streams[0].pid, 0x1011);
|
||||
assert_eq!(clip.streams[0].coding_type, 0x24);
|
||||
assert_eq!(clip.streams[1].pid, 0x1100);
|
||||
assert_eq!(clip.streams[1].coding_type, 0x86);
|
||||
assert_eq!(clip.streams[1].language, "eng");
|
||||
assert_eq!(clip.streams[2].pid, 0x1200);
|
||||
assert_eq!(clip.streams[2].language, "jpn");
|
||||
}
|
||||
|
||||
/// parse_program_info is best-effort: a stream whose declared sci_len
|
||||
/// runs past the section (`sci_end > data.len()`) makes it return the
|
||||
/// streams collected so far (here: none), never panic. Source returns
|
||||
/// `out` early on the overflow.
|
||||
#[test]
|
||||
fn program_info_truncated_sci_no_panic() {
|
||||
// One stream claiming sci_len = 200 but with no body.
|
||||
let mut body = Vec::new();
|
||||
body.push(0); // reserved
|
||||
body.push(1); // num_programs
|
||||
body.extend_from_slice(&0u32.to_be_bytes());
|
||||
body.extend_from_slice(&0u16.to_be_bytes());
|
||||
body.push(1); // num_streams
|
||||
body.push(0); // num_groups
|
||||
body.extend_from_slice(&0x1011u16.to_be_bytes()); // pid
|
||||
body.push(200); // sci_len = 200, no body follows
|
||||
let mut pi = Vec::new();
|
||||
pi.extend_from_slice(&(body.len() as u32).to_be_bytes());
|
||||
pi.extend_from_slice(&body);
|
||||
let data = build_clpi_with_proginfo(100, &pi, None);
|
||||
let clip = parse(&data).expect("should not panic");
|
||||
assert!(clip.streams.is_empty());
|
||||
}
|
||||
|
||||
/// parse_program_info rejects sci_len == 0 (`sci_len < 1` → return).
|
||||
/// A zero-length stream_coding_info is unusable.
|
||||
#[test]
|
||||
fn program_info_zero_sci_len_yields_no_stream() {
|
||||
let mut body = Vec::new();
|
||||
body.push(0);
|
||||
body.push(1);
|
||||
body.extend_from_slice(&0u32.to_be_bytes());
|
||||
body.extend_from_slice(&0u16.to_be_bytes());
|
||||
body.push(1);
|
||||
body.push(0);
|
||||
body.extend_from_slice(&0x1011u16.to_be_bytes());
|
||||
body.push(0); // sci_len = 0
|
||||
let mut pi = Vec::new();
|
||||
pi.extend_from_slice(&(body.len() as u32).to_be_bytes());
|
||||
pi.extend_from_slice(&body);
|
||||
let data = build_clpi_with_proginfo(100, &pi, None);
|
||||
let clip = parse(&data).expect("should parse");
|
||||
assert!(clip.streams.is_empty());
|
||||
}
|
||||
|
||||
/// pts_coarse field is 14 bits: dword0 = ref_to_fine_id<<14 | pts_coarse.
|
||||
/// A pts_coarse of 0x3FFF (max) with ref_to_fine_id 5 must decode both
|
||||
/// without bleed. Verify the >>14 and &0x3FFF split.
|
||||
#[test]
|
||||
fn coarse_pts_14bit_split() {
|
||||
let cpi = build_cpi(0x1011, &[(5, 0x3FFF, 0x12340000)], &[(0, 0)]);
|
||||
let data = build_clpi(1000, Some(&cpi));
|
||||
let clip = parse(&data).expect("should parse");
|
||||
assert_eq!(clip.ep_coarse[0].ref_to_fine_id, 5);
|
||||
assert_eq!(clip.ep_coarse[0].pts_coarse, 0x3FFF);
|
||||
assert_eq!(clip.ep_coarse[0].spn_coarse, 0x12340000);
|
||||
}
|
||||
|
||||
/// Fine entry: dword = is_angle(1)+i_end_offset(3)+pts_fine(11)+
|
||||
/// spn_fine(17). pts_fine occupies bits 17..28 (>>17 & 0x7FF), spn_fine
|
||||
/// the low 17 bits (& 0x1FFFF). Set high bits (is_angle/i_end_offset)
|
||||
/// and verify they do NOT bleed into pts_fine.
|
||||
#[test]
|
||||
fn fine_entry_bit_layout_isolates_pts_and_spn() {
|
||||
// Construct a raw fine dword with is_angle=1, i_end_offset=0b111,
|
||||
// pts_fine=0x5AA, spn_fine=0x1AAAA, then verify decode.
|
||||
let is_angle: u32 = 1;
|
||||
let i_end: u32 = 0b111;
|
||||
let pts_f: u32 = 0x5AA; // 11-bit
|
||||
let spn_f: u32 = 0x1AAAA; // 17-bit
|
||||
let dword: u32 = (is_angle << 31) | (i_end << 28) | (pts_f << 17) | spn_f;
|
||||
|
||||
// Build the CPI by hand with this raw fine dword.
|
||||
let mut stream_ep = Vec::new();
|
||||
let fine_start: u32 = 4; // no coarse entries → fine right after header
|
||||
stream_ep.extend_from_slice(&fine_start.to_be_bytes());
|
||||
stream_ep.extend_from_slice(&dword.to_be_bytes());
|
||||
|
||||
let num_coarse: u32 = 0;
|
||||
let num_fine: u32 = 1;
|
||||
let ep_map_start: u32 = 14;
|
||||
let ep_stream_type: u32 = 1;
|
||||
let packed: u128 = ((ep_stream_type as u128) << 66)
|
||||
| ((num_coarse as u128) << 50)
|
||||
| ((num_fine as u128) << 32)
|
||||
| (ep_map_start as u128);
|
||||
let packed_bytes = packed.to_be_bytes();
|
||||
let stream_header_bits = &packed_bytes[6..16];
|
||||
|
||||
let mut ep_map = Vec::new();
|
||||
ep_map.push(0);
|
||||
ep_map.push(1);
|
||||
ep_map.extend_from_slice(&0x1011u16.to_be_bytes());
|
||||
ep_map.extend_from_slice(stream_header_bits);
|
||||
ep_map.extend_from_slice(&stream_ep);
|
||||
|
||||
let mut cpi = Vec::new();
|
||||
cpi.extend_from_slice(&((2 + ep_map.len()) as u32).to_be_bytes());
|
||||
cpi.extend_from_slice(&[0u8; 2]);
|
||||
cpi.extend_from_slice(&ep_map);
|
||||
|
||||
let data = build_clpi(1000, Some(&cpi));
|
||||
let clip = parse(&data).expect("should parse");
|
||||
assert_eq!(clip.ep_fine.len(), 1);
|
||||
assert_eq!(clip.ep_fine[0].pts_fine, 0x5AA); // high bits stripped
|
||||
assert_eq!(clip.ep_fine[0].spn_fine, 0x1AAAA);
|
||||
}
|
||||
|
||||
/// resolved_ep_map assigns fine entries to coarse groups via
|
||||
/// [ref_to_fine_id .. next coarse's ref_to_fine_id). full_pts combines
|
||||
/// coarse<<19 + fine<<8 and full_spn ORs masked coarse with fine.
|
||||
/// Verify the first resolved entry's (pts, spn) for a known fixture.
|
||||
#[test]
|
||||
fn resolved_ep_map_combines_coarse_and_fine() {
|
||||
// coarse 0: ref_to_fine_id=0, pts_coarse=10, spn_coarse=0x00020000
|
||||
// fine 0: pts_fine=3, spn_fine=0x100
|
||||
let cpi = build_cpi(0x1011, &[(0, 10, 0x00020000)], &[(3, 0x100)]);
|
||||
let data = build_clpi(1000, Some(&cpi));
|
||||
let clip = parse(&data).expect("should parse");
|
||||
let resolved = clip.resolved_ep_map();
|
||||
assert_eq!(resolved.len(), 1);
|
||||
let expected_pts = (10u64 << 19) + (3u64 << 8);
|
||||
let expected_spn = (0x00020000u32 & 0xFFFE_0000) | 0x100;
|
||||
assert_eq!(resolved[0].0, expected_pts);
|
||||
assert_eq!(resolved[0].1, expected_spn);
|
||||
}
|
||||
|
||||
/// get_extents converts an in/out PTS range to a single sector Extent.
|
||||
/// SPN→byte = spn×192, byte→sector = /2048 (start floored, end ceiled),
|
||||
/// relative to m2ts file start. Verify the math for a known fixture.
|
||||
#[test]
|
||||
fn get_extents_spn_to_sector_math() {
|
||||
// Two EP points: PTS p0 → SPN 0, PTS p1 → SPN big_spn.
|
||||
// full_spn ORs (spn_coarse & 0xFFFE0000) with spn_fine, so the SPN
|
||||
// must be coarse-aligned (low 17 bits clear) to survive intact.
|
||||
// 0x20000 (131072) is the smallest non-zero coarse-aligned SPN.
|
||||
let big_spn: u32 = 0x20000;
|
||||
let cpi = build_cpi(0x1011, &[(0, 0, 0), (1, 100, big_spn)], &[(0, 0), (0, 0)]);
|
||||
let data = build_clpi(1000, Some(&cpi));
|
||||
let clip = parse(&data).expect("should parse");
|
||||
|
||||
let p0 = 0u64; // PTS of first EP
|
||||
let p1 = 100u64 << 19; // PTS of second EP
|
||||
let extents = clip.get_extents(p0, p1);
|
||||
assert_eq!(extents.len(), 1);
|
||||
// Mirror production: SPN→byte ×packet, byte→sector with start FLOORed
|
||||
// and end CEILed (same constants as get_extents).
|
||||
let start_spn: u64 = 0;
|
||||
let end_spn = big_spn as u64;
|
||||
let start_byte = start_spn * BD_SOURCE_PACKET_BYTES as u64;
|
||||
let end_byte = end_spn * BD_SOURCE_PACKET_BYTES as u64;
|
||||
let start_sector = (start_byte / SECTOR_BYTES_U64) as u32;
|
||||
let end_sector = end_byte.div_ceil(SECTOR_BYTES_U64) as u32;
|
||||
assert_eq!(extents[0].start_lba, start_sector);
|
||||
assert_eq!(extents[0].sector_count, end_sector - start_sector);
|
||||
// Concretely: 0x20000 × 192 / 2048 = 12288 sectors.
|
||||
assert_eq!(extents[0].sector_count, 12288);
|
||||
}
|
||||
|
||||
/// get_extents returns an empty Vec when the EP map is empty (no CPI),
|
||||
/// since there is no SPN to resolve. Documented early return.
|
||||
#[test]
|
||||
fn get_extents_empty_when_no_ep_map() {
|
||||
let data = build_clpi(1000, None);
|
||||
let clip = parse(&data).expect("should parse");
|
||||
assert!(clip.get_extents(0, 1_000_000).is_empty());
|
||||
}
|
||||
|
||||
/// get_extents returns empty when end_spn <= start_spn (degenerate or
|
||||
/// inverted range). Source has an explicit `if end_spn <= start_spn`
|
||||
/// guard. Use in_time == out_time on a single-point map.
|
||||
#[test]
|
||||
fn get_extents_empty_on_degenerate_range() {
|
||||
let cpi = build_cpi(0x1011, &[(0, 50, 0x1000)], &[(0, 0)]);
|
||||
let data = build_clpi(1000, Some(&cpi));
|
||||
let clip = parse(&data).expect("should parse");
|
||||
let p = 50u64 << 19;
|
||||
// in == out → start_spn == end_spn → empty.
|
||||
assert!(clip.get_extents(p, p).is_empty());
|
||||
}
|
||||
|
||||
/// full_spn masks the LOW 17 bits of spn_coarse (& 0xFFFE0000) before
|
||||
/// OR-ing fine. A spn_coarse with low bits set must have them cleared,
|
||||
/// then replaced by spn_fine. Independent of parse, exercises the
|
||||
/// reconstruction directly with a hostile low-bit pattern.
|
||||
#[test]
|
||||
fn full_spn_clears_coarse_low_17_bits() {
|
||||
let coarse = EpCoarse {
|
||||
ref_to_fine_id: 0,
|
||||
pts_coarse: 0,
|
||||
spn_coarse: 0x0006_FFFF, // low 17 bits all set
|
||||
};
|
||||
let fine = EpFine {
|
||||
pts_fine: 0,
|
||||
spn_fine: 0x5,
|
||||
};
|
||||
// 0x0006_FFFF & 0xFFFE_0000 = 0x0006_0000; | 0x5 = 0x0006_0005.
|
||||
assert_eq!(ClipInfo::full_spn(&coarse, &fine), 0x0006_0005);
|
||||
}
|
||||
|
||||
/// CPI guard: cpi_length < 4 short-circuits to empty maps (the length
|
||||
/// field counts bytes after itself, and the EP map needs ≥4). A
|
||||
/// cpi_length of 0/1/2/3 must yield empty EP maps, not panic.
|
||||
#[test]
|
||||
fn cpi_length_below_4_yields_empty() {
|
||||
for bad_len in 0u32..4 {
|
||||
let mut cpi = Vec::new();
|
||||
cpi.extend_from_slice(&bad_len.to_be_bytes());
|
||||
cpi.extend_from_slice(&[0u8; 20]); // padding so the slice exists
|
||||
let data = build_clpi(1000, Some(&cpi));
|
||||
let clip = parse(&data).expect("should parse");
|
||||
assert!(clip.ep_coarse.is_empty(), "len={bad_len}");
|
||||
assert!(clip.ep_fine.is_empty(), "len={bad_len}");
|
||||
}
|
||||
}
|
||||
|
||||
/// ep_map_offset that points past the EP map (`ep_map_offset + 4 >
|
||||
/// ep_map.len()`) → empty maps (bounds guard), not panic. Patch the
|
||||
/// EP_map_start field to a huge value.
|
||||
#[test]
|
||||
fn ep_map_offset_out_of_bounds_yields_empty() {
|
||||
let cpi = build_cpi(0x1011, &[(0, 10, 0x20000)], &[(5, 100)]);
|
||||
let mut data = build_clpi(1000, Some(&cpi));
|
||||
// EP_map_start is the low 32 bits of the 80-bit stream header at
|
||||
// ep_map[4..14]. In the file: header(60) + cpi_length(4) +
|
||||
// reserved(2) + ep_map reserved(1) + num_streams(1) + pid(2) = 70,
|
||||
// then 10 header bytes [70..80]; EP_map_start is the last 4 [76..80].
|
||||
let off = 60 + 4 + 2 + 1 + 1 + 2 + 6; // = 76
|
||||
data[off..off + 4].copy_from_slice(&0xFFFF_FFFFu32.to_be_bytes());
|
||||
let clip = parse(&data).expect("should not panic");
|
||||
assert!(clip.ep_coarse.is_empty());
|
||||
assert!(clip.ep_fine.is_empty());
|
||||
}
|
||||
|
||||
/// num_coarse declares more entries than the CPI section holds. The
|
||||
/// loop must stop at `off + 8 > coarse_data.len()` (break), not read
|
||||
/// out of bounds. Patch num_coarse to a large value while supplying 1
|
||||
/// coarse entry's worth of bytes.
|
||||
#[test]
|
||||
fn coarse_count_overshoot_truncates_safely() {
|
||||
let cpi = build_cpi(0x1011, &[(0, 10, 0x20000)], &[(5, 100)]);
|
||||
let mut data = build_clpi(1000, Some(&cpi));
|
||||
// num_coarse is bits 14..30 of the 80-bit header. Rather than
|
||||
// bit-surgery, rebuild with a hand-set num_coarse=255 but only 1
|
||||
// coarse entry of bytes — done below directly.
|
||||
let _ = &mut data;
|
||||
|
||||
let num_coarse_decl: u32 = 255;
|
||||
let num_fine: u32 = 1;
|
||||
let ep_map_start: u32 = 14;
|
||||
let ep_stream_type: u32 = 1;
|
||||
let packed: u128 = ((ep_stream_type as u128) << 66)
|
||||
| ((num_coarse_decl as u128) << 50)
|
||||
| ((num_fine as u128) << 32)
|
||||
| (ep_map_start as u128);
|
||||
let packed_bytes = packed.to_be_bytes();
|
||||
let stream_header_bits = &packed_bytes[6..16];
|
||||
|
||||
// stream EP data: fine_start points past the 1 coarse entry.
|
||||
let fine_start: u32 = 4 + 8; // 4-byte header + 1 coarse entry x 8 bytes
|
||||
let mut stream_ep = Vec::new();
|
||||
stream_ep.extend_from_slice(&fine_start.to_be_bytes());
|
||||
// exactly ONE coarse entry (8 bytes), though header claims 255.
|
||||
stream_ep.extend_from_slice(&10u32.to_be_bytes());
|
||||
stream_ep.extend_from_slice(&0x20000u32.to_be_bytes());
|
||||
// one fine entry (4 bytes)
|
||||
stream_ep.extend_from_slice(&(((5u32 & 0x7FF) << 17) | 100).to_be_bytes());
|
||||
|
||||
let mut ep_map = Vec::new();
|
||||
ep_map.push(0);
|
||||
ep_map.push(1);
|
||||
ep_map.extend_from_slice(&0x1011u16.to_be_bytes());
|
||||
ep_map.extend_from_slice(stream_header_bits);
|
||||
ep_map.extend_from_slice(&stream_ep);
|
||||
let mut cpi2 = Vec::new();
|
||||
cpi2.extend_from_slice(&((2 + ep_map.len()) as u32).to_be_bytes());
|
||||
cpi2.extend_from_slice(&[0u8; 2]);
|
||||
cpi2.extend_from_slice(&ep_map);
|
||||
let data2 = build_clpi(1000, Some(&cpi2));
|
||||
let clip = parse(&data2).expect("should not panic on coarse overshoot");
|
||||
// Only the 1 real coarse entry was readable.
|
||||
assert_eq!(clip.ep_coarse.len(), 1);
|
||||
assert_eq!(clip.ep_coarse[0].pts_coarse, 10);
|
||||
}
|
||||
|
||||
/// resolved_ep_map: the LAST coarse group's fine range extends to
|
||||
/// ep_fine.len() (no "next coarse" bound). Verify all trailing fine
|
||||
/// entries are assigned to the final coarse group.
|
||||
#[test]
|
||||
fn resolved_ep_map_last_group_to_end() {
|
||||
// coarse 0 ref_to_fine_id=0, coarse 1 ref_to_fine_id=1.
|
||||
// 3 fine entries: fine 0 → coarse 0; fine 1,2 → coarse 1.
|
||||
let cpi = build_cpi(
|
||||
0x1011,
|
||||
&[(0, 0, 0), (1, 100, 0)],
|
||||
&[(0, 10), (0, 20), (0, 30)],
|
||||
);
|
||||
let data = build_clpi(1000, Some(&cpi));
|
||||
let clip = parse(&data).expect("should parse");
|
||||
let resolved = clip.resolved_ep_map();
|
||||
// All 3 fine entries resolved (last group picks up fine 1 and 2).
|
||||
assert_eq!(resolved.len(), 3);
|
||||
}
|
||||
}
|
||||
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
//! Physical media constants — the single source of truth.
|
||||
//!
|
||||
//! Naming convention: a constant is prefixed by the **narrowest scope where it
|
||||
//! is valid**. A value common to all optical media carries no prefix; a value
|
||||
//! specific to a container/format/disc-type is prefixed by it
|
||||
//! (`TS_`, `BD_`, …). Define each physical quantity here exactly once and import
|
||||
//! it — never re-declare a bare literal or a local copy.
|
||||
|
||||
/// Bytes per logical sector on every optical medium freemkv reads
|
||||
/// (Blu-ray, DVD-Video, CD-ROM Mode 1). Universal — hence unprefixed.
|
||||
///
|
||||
/// `usize` because its dominant use is buffer sizing and slice indexing, where
|
||||
/// Rust *requires* `usize` (`vec![0u8; SECTOR_BYTES]`, `buf.len() < SECTOR_BYTES`).
|
||||
/// For byte-offset / capacity arithmetic — which is `u64` because a disc can
|
||||
/// exceed 4 GiB — use [`SECTOR_BYTES_U64`] instead of casting at each site.
|
||||
pub const SECTOR_BYTES: usize = 2048;
|
||||
|
||||
/// [`SECTOR_BYTES`] as `u64`, for byte-offset and capacity arithmetic. The
|
||||
/// single `usize → u64` boundary cast lives here, once, so offset math across
|
||||
/// the workspace reads as `sectors * SECTOR_BYTES_U64` with no per-site cast.
|
||||
pub const SECTOR_BYTES_U64: u64 = SECTOR_BYTES as u64;
|
||||
|
||||
/// Bytes per MPEG-2 transport-stream packet. Common to all MPEG-TS, not just
|
||||
/// Blu-ray — prefixed by the format, not a disc type.
|
||||
pub const TS_PACKET_BYTES: usize = 188;
|
||||
|
||||
/// Bytes in an MPEG-2 transport-stream packet header: sync byte, the
|
||||
/// flags/PID word, and the adaptation/continuity byte.
|
||||
pub const TS_HEADER_BYTES: usize = 4;
|
||||
|
||||
/// Bytes in the arrival-timestamp prefix a Blu-ray M2TS prepends to each TS
|
||||
/// packet to form a source packet. Same width as a TS header but a distinct
|
||||
/// quantity ([`TS_HEADER_BYTES`]) — do not conflate.
|
||||
pub const BD_TIMESTAMP_PREFIX_BYTES: usize = 4;
|
||||
|
||||
/// Bytes of payload in an MPEG-2 transport-stream packet:
|
||||
/// [`TS_PACKET_BYTES`] minus the [`TS_HEADER_BYTES`] header.
|
||||
pub const TS_PAYLOAD_BYTES: usize = TS_PACKET_BYTES - TS_HEADER_BYTES;
|
||||
|
||||
/// Bytes per Blu-ray M2TS *source packet*: a TS packet ([`TS_PACKET_BYTES`])
|
||||
/// prefixed with the [`BD_TIMESTAMP_PREFIX_BYTES`] arrival-timestamp header.
|
||||
/// A BDAV/M2TS construct only — DVD VOBs have no source packets — hence `BD_`.
|
||||
pub const BD_SOURCE_PACKET_BYTES: usize = TS_PACKET_BYTES + BD_TIMESTAMP_PREFIX_BYTES;
|
||||
|
||||
/// Elementary-stream coding-type codes — the single source of truth for the
|
||||
/// byte that identifies a stream's codec.
|
||||
///
|
||||
/// This is one registry used in two places that share the same value space:
|
||||
/// the MPEG-TS PMT `stream_type` (ISO/IEC 13818-1 Table 2-34) and the Blu-ray
|
||||
/// STN/CLPI `stream_coding_type` (BD-ROM Part 3). The standardized video codes
|
||||
/// (`0x02`, `0x1B`, `0x24`) are ISO assignments (ISO/IEC 13818-1 Table 2-34);
|
||||
/// `0xEA` (VC-1) is a BD-ROM convention in the ISO user-private range. The
|
||||
/// `0x80..=0xA2` audio/graphics codes also sit in the user-private range and follow the
|
||||
/// Blu-ray Disc Association / ATSC A/52 convention. Because every consumer
|
||||
/// reads or writes this single byte, the family is unprefixed — the scope is
|
||||
/// "any elementary stream freemkv parses or muxes".
|
||||
///
|
||||
/// Each constant is `u8`: the spec defines an 8-bit field and the code compares
|
||||
/// it directly against a byte read from the buffer, so no casts are needed.
|
||||
pub mod coding_type {
|
||||
/// MPEG-2 video (ISO/IEC 13818-1 Table 2-34).
|
||||
pub const MPEG2_VIDEO: u8 = 0x02;
|
||||
/// H.264 / AVC video (ISO/IEC 13818-1 Table 2-34).
|
||||
pub const H264: u8 = 0x1B;
|
||||
/// HEVC / H.265 video (ISO/IEC 13818-1 Table 2-34, 2015 amendment).
|
||||
pub const HEVC: u8 = 0x24;
|
||||
/// SMPTE VC-1 video (BD-ROM convention, ISO user-private range).
|
||||
pub const VC1: u8 = 0xEA;
|
||||
|
||||
/// LPCM audio (BD-ROM convention).
|
||||
pub const LPCM: u8 = 0x80;
|
||||
/// Dolby Digital (AC-3) audio (BD-ROM / ATSC A/52 convention).
|
||||
pub const AC3: u8 = 0x81;
|
||||
/// DTS audio (BD-ROM convention).
|
||||
pub const DTS: u8 = 0x82;
|
||||
/// Dolby TrueHD audio (BD-ROM convention).
|
||||
pub const TRUEHD: u8 = 0x83;
|
||||
/// Dolby Digital Plus (E-AC-3 / AC-3+) audio (BD-ROM convention).
|
||||
pub const AC3_PLUS: u8 = 0x84;
|
||||
/// DTS-HD High Resolution audio (BD-ROM Part 3-1).
|
||||
pub const DTS_HD_HR: u8 = 0x85;
|
||||
/// DTS-HD Master Audio (lossless) (BD-ROM Part 3-1).
|
||||
pub const DTS_HD_MA: u8 = 0x86;
|
||||
|
||||
/// Presentation Graphics — PG subtitle stream (BD-ROM HDMV).
|
||||
pub const PG: u8 = 0x90;
|
||||
/// Interactive Graphics — IG / BD-J menu overlay, NOT a subtitle (BD-ROM HDMV).
|
||||
pub const IG: u8 = 0x91;
|
||||
/// Text subtitle stream (BD-ROM HDMV).
|
||||
pub const TEXT_SUBTITLE: u8 = 0x92;
|
||||
|
||||
/// Secondary Dolby Digital Plus audio (BD-ROM convention).
|
||||
pub const AC3_PLUS_SECONDARY: u8 = 0xA1;
|
||||
/// Secondary DTS-HD audio (lossless MA, not lossy HR) (BD-ROM convention).
|
||||
pub const DTS_HD_SECONDARY: u8 = 0xA2;
|
||||
}
|
||||
|
||||
/// MPEG PES `stream_id` codes — the byte after the `00 00 01` start-code prefix
|
||||
/// that identifies an elementary stream's role in a PES packet (ISO/IEC
|
||||
/// 13818-1 Table 2-22). Shared by the program-stream demuxer and the TS/M2TS
|
||||
/// muxers, so defined here once. Each is `u8` (matches the byte on the wire).
|
||||
pub mod pes_stream_id {
|
||||
/// Video stream (`110x xxxx`; freemkv emits the base id `0xE0`).
|
||||
pub const VIDEO: u8 = 0xE0;
|
||||
/// private_stream_1 — AC-3 / DTS / LPCM / PGS subtitle payloads.
|
||||
pub const PRIVATE_STREAM_1: u8 = 0xBD;
|
||||
/// padding_stream — stuffing bytes only, no payload to demux.
|
||||
pub const PADDING_STREAM: u8 = 0xBE;
|
||||
/// private_stream_2 — DVD navigation (PCI/DSI); carries no muxable ES.
|
||||
pub const PRIVATE_STREAM_2: u8 = 0xBF;
|
||||
|
||||
/// Highest video stream_id — the `110x xxxx` video range tops out at 0xEF.
|
||||
pub const VIDEO_MAX: u8 = 0xEF;
|
||||
|
||||
/// Inclusive range of every PES `stream_id` that carries demuxable payload:
|
||||
/// [`PRIVATE_STREAM_1`] (0xBD) through [`VIDEO_MAX`] (0xEF) — i.e. private
|
||||
/// stream 1/2, padding, MPEG audio (0xC0-0xDF) and video (0xE0-0xEF). The
|
||||
/// pack (0xBA), system-header (0xBB) and program-end (0xB9) codes sit below
|
||||
/// this range and are deliberately excluded: they're structural, not ES.
|
||||
pub const PAYLOAD_RANGE: core::ops::RangeInclusive<u8> = PRIVATE_STREAM_1..=VIDEO_MAX;
|
||||
}
|
||||
+403
-203
@@ -1,51 +1,16 @@
|
||||
//! CSS drive authentication — full key hierarchy.
|
||||
//! CSS drive bus-authentication — read-unlock primitive.
|
||||
//!
|
||||
//! Protocol:
|
||||
//! 1. Bus authentication (challenge-response) → bus key
|
||||
//! 2. Read disc key block (READ DVD STRUCTURE) → XOR with bus key → decrypt with player keys → disc key
|
||||
//! 3. Read title key (REPORT KEY format 0x04) → XOR with bus key → decrypt with disc key → title key
|
||||
//!
|
||||
//! Based on libdvdcss (VideoLAN) and Stevenson 1999 analysis.
|
||||
//! A CSS-enforcing DVD drive refuses to return scrambled sectors until a
|
||||
//! CSS bus-auth handshake has set its Authentication Success Flag (ASF=1).
|
||||
//! [`unlock_css_reads`] runs that bus-auth challenge-response (which is what
|
||||
//! actually opens scrambled-sector reads), then a best-effort, non-fatal
|
||||
//! disc-key REPORT KEY. The bytes are NOT used as keys: the descramble title
|
||||
//! key is recovered keylessly by the Stevenson known-plaintext attack (see
|
||||
//! [`super::crack_key`]).
|
||||
|
||||
use crate::drive::Drive;
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
// ── Player keys (from libdvdcss, Stevenson's PlayerKey cracker) ───────────
|
||||
|
||||
const PLAYER_KEYS: [[u8; 5]; 31] = [
|
||||
[0x01, 0xaf, 0xe3, 0x12, 0x80],
|
||||
[0x12, 0x11, 0xca, 0x04, 0x3b],
|
||||
[0x14, 0x0c, 0x9e, 0xd0, 0x09],
|
||||
[0x14, 0x71, 0x35, 0xba, 0xe2],
|
||||
[0x1a, 0xa4, 0x33, 0x21, 0xa6],
|
||||
[0x26, 0xec, 0xc4, 0xa7, 0x4e],
|
||||
[0x2c, 0xb2, 0xc1, 0x09, 0xee],
|
||||
[0x2f, 0x25, 0x9e, 0x96, 0xdd],
|
||||
[0x33, 0x2f, 0x49, 0x6c, 0xe0],
|
||||
[0x35, 0x5b, 0xc1, 0x31, 0x0f],
|
||||
[0x36, 0x67, 0xb2, 0xe3, 0x85],
|
||||
[0x39, 0x3d, 0xf1, 0xf1, 0xbd],
|
||||
[0x3b, 0x31, 0x34, 0x0d, 0x91],
|
||||
[0x45, 0xed, 0x28, 0xeb, 0xd3],
|
||||
[0x48, 0xb7, 0x6c, 0xce, 0x69],
|
||||
[0x4b, 0x65, 0x0d, 0xc1, 0xee],
|
||||
[0x4c, 0xbb, 0xf5, 0x5b, 0x23],
|
||||
[0x51, 0x67, 0x67, 0xc5, 0xe0],
|
||||
[0x53, 0x94, 0xe1, 0x75, 0xbf],
|
||||
[0x57, 0x2c, 0x8b, 0x31, 0xae],
|
||||
[0x63, 0xdb, 0x4c, 0x5b, 0x4a],
|
||||
[0x7b, 0x1e, 0x5e, 0x2b, 0x57],
|
||||
[0x85, 0xf3, 0x85, 0xa0, 0xe0],
|
||||
[0xab, 0x1e, 0xe7, 0x7b, 0x72],
|
||||
[0xab, 0x36, 0xe3, 0xeb, 0x76],
|
||||
[0xb1, 0xb8, 0xf9, 0x38, 0x03],
|
||||
[0xb8, 0x5d, 0xd8, 0x53, 0xbd],
|
||||
[0xbf, 0x92, 0xc3, 0xb0, 0xe2],
|
||||
[0xcf, 0x1a, 0xb2, 0xf8, 0x0a],
|
||||
[0xec, 0xa0, 0xcf, 0xb3, 0xff],
|
||||
[0xfc, 0x95, 0xa9, 0x87, 0x35],
|
||||
];
|
||||
|
||||
// ── CryptKey tables ───────────────────────────────────────────────────────
|
||||
|
||||
const CRYPT_TAB0: [u8; 256] = [
|
||||
@@ -105,7 +70,7 @@ const CRYPT_TAB2: [u8; 256] = [
|
||||
0x45, 0x78, 0xA9, 0xA8, 0xEA, 0xC9, 0x6A, 0xF7, 0x29, 0x91, 0xF0, 0x02, 0x18, 0x3A, 0x4E, 0x7C,
|
||||
];
|
||||
|
||||
const CRYPT_TAB3: [u8; 288] = [
|
||||
const CRYPT_TAB3: [u8; 256] = [
|
||||
0x73, 0x51, 0x95, 0xE1, 0x12, 0xE4, 0xC0, 0x58, 0xEE, 0xF2, 0x08, 0x1B, 0xA9, 0xFA, 0x98, 0x4C,
|
||||
0xA7, 0x33, 0xE2, 0x1B, 0xA7, 0x6D, 0xF5, 0x30, 0x97, 0x1D, 0xF3, 0x02, 0x60, 0x5A, 0x82, 0x0F,
|
||||
0x91, 0xD0, 0x9C, 0x10, 0x39, 0x7A, 0x83, 0x85, 0x3B, 0xB2, 0xB8, 0xAE, 0x0C, 0x09, 0x52, 0xEA,
|
||||
@@ -122,8 +87,6 @@ const CRYPT_TAB3: [u8; 288] = [
|
||||
0xBD, 0xC1, 0x0E, 0x56, 0x54, 0x3E, 0x14, 0x5F, 0x8C, 0x8F, 0x6E, 0x75, 0x1C, 0x07, 0x39, 0x7B,
|
||||
0x4B, 0xDB, 0xD3, 0x4B, 0x1E, 0xC8, 0x7E, 0xFE, 0x3E, 0x72, 0x16, 0x83, 0x7D, 0xEE, 0xF5, 0xCA,
|
||||
0xC5, 0x18, 0xF9, 0xD8, 0x68, 0xAB, 0x38, 0x85, 0xA8, 0xF0, 0xA1, 0x73, 0x9F, 0x5D, 0x19, 0x0B,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, 0x72, 0x39, 0x25, 0x67, 0x26, 0x6D, 0x71,
|
||||
0x36, 0x77, 0x3C, 0x20, 0x62, 0x23, 0x68, 0x74, 0xC3, 0x82, 0xC9, 0x15, 0x57, 0x16, 0x5D, 0x81,
|
||||
];
|
||||
|
||||
const VARIANTS: [u8; 32] = [
|
||||
@@ -152,40 +115,51 @@ const PERM_VARIANT: [[u8; 32]; 2] = [
|
||||
],
|
||||
];
|
||||
|
||||
// ── SCSI constants ────────────────────────────────────────────────────────
|
||||
|
||||
const SCSI_READ_DVD_STRUCTURE: u8 = 0xAD;
|
||||
|
||||
// ── Public API ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Perform CSS bus authentication only.
|
||||
pub fn authenticate(drive: &mut Drive) -> Result<()> {
|
||||
let (_, _) = bus_auth(drive)?;
|
||||
Ok(())
|
||||
/// CSS bus-auth **unlock** primitive.
|
||||
///
|
||||
/// Runs the bus-auth challenge-response (which sets the drive's ASF=1 and is
|
||||
/// what actually unlocks scrambled-sector reads), then a best-effort,
|
||||
/// non-fatal disc-key REPORT KEY. The title-key REPORT KEY is NOT issued: it
|
||||
/// is unnecessary (the descramble key is recovered keylessly by the Stevenson
|
||||
/// attack in [`super::crack_key`]) and its hard failure on some USB bridges
|
||||
/// used to abort the whole unlock (the 7014 bug). The bytes are discarded.
|
||||
pub fn unlock_css_reads(drive: &mut Drive, lba: u32) -> Result<()> {
|
||||
let t0 = std::time::Instant::now();
|
||||
tracing::info!(target: "freemkv::css", phase = "unlock_css_reads", lba, "begin");
|
||||
let r = unlock_css_reads_inner(drive, lba);
|
||||
tracing::info!(
|
||||
target: "freemkv::css",
|
||||
phase = "unlock_css_reads",
|
||||
lba,
|
||||
ok = r.is_ok(),
|
||||
elapsed_ms = t0.elapsed().as_millis() as u64,
|
||||
"end"
|
||||
);
|
||||
r
|
||||
}
|
||||
|
||||
/// Full CSS key extraction: bus auth → disc key → title key.
|
||||
pub fn authenticate_and_read_title_key(drive: &mut Drive, lba: u32) -> Result<[u8; 5]> {
|
||||
// Session 1: bus auth → disc key (AGID consumed by READ_DVD_STRUCTURE)
|
||||
let (agid, bus_key) = bus_auth(drive)?;
|
||||
let disc_key = read_disc_key(drive, agid, &bus_key)?;
|
||||
|
||||
// Session 2: fresh bus auth → title key (needs separate AGID)
|
||||
let (agid2, bus_key2) = bus_auth(drive)?;
|
||||
let encrypted_title = read_raw_title_key(drive, agid2, lba)?;
|
||||
|
||||
// Decrypt title key: XOR with bus key, then decrypt with disc key
|
||||
let mut title_key = [0u8; 5];
|
||||
for i in 0..5 {
|
||||
title_key[i] = encrypted_title[i] ^ bus_key2[i];
|
||||
fn unlock_css_reads_inner(drive: &mut Drive, _lba: u32) -> Result<()> {
|
||||
tracing::debug!(target: "freemkv::css", "css unlock: begin");
|
||||
// The bus-auth challenge-response sets the drive's Authentication Success
|
||||
// Flag (ASF=1), which is what opens scrambled-sector reads. This is the
|
||||
// ONLY step required to unlock reads; a failure here is fatal — we
|
||||
// genuinely cannot read scrambled sectors.
|
||||
let (agid, _bus_key) = bus_auth(drive).inspect_err(|e| {
|
||||
tracing::warn!(target: "freemkv::css", error_code = e.code(), "css unlock: bus_auth failed");
|
||||
})?;
|
||||
tracing::debug!(target: "freemkv::css", agid, "css unlock: bus_auth ok");
|
||||
// Disc-key REPORT KEY: issued BEST-EFFORT for any firmware that ties part
|
||||
// of its read-unlock to it. The bytes are unused (the descramble key is
|
||||
// recovered keylessly) and a failure is NON-FATAL — the gate is already
|
||||
// open from bus-auth. This replaces the title-key REPORT KEY, whose hard
|
||||
// failure used to abort the whole unlock (the 7014 bug on USB bridges).
|
||||
if let Err(e) = read_disc_key(drive, agid) {
|
||||
tracing::debug!(target: "freemkv::css", error_code = e.code(), "css unlock: disc-key REPORT KEY skipped (non-fatal)");
|
||||
}
|
||||
|
||||
if title_key == [0u8; 5] {
|
||||
return Ok(title_key);
|
||||
}
|
||||
|
||||
let title_key = super::lfsr::decrypt_key(0xFF, &disc_key, &title_key);
|
||||
Ok(title_key)
|
||||
tracing::debug!(target: "freemkv::css", "css unlock: ok");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Step 1: Bus Authentication ────────────────────────────────────────────
|
||||
@@ -219,8 +193,14 @@ fn bus_auth(drive: &mut Drive) -> Result<(u8, [u8; 5])> {
|
||||
.map_err(|_| Error::CssAuthFailed)?;
|
||||
let agid = (buf[7] >> 6) & 0x03;
|
||||
|
||||
// Host sends challenge
|
||||
let host_challenge: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
// Host sends challenge. The spec wants a fresh per-session random nonce,
|
||||
// not a fixed constant — a predictable challenge weakens the bus-auth
|
||||
// handshake.
|
||||
let mut host_challenge = [0u8; 10];
|
||||
{
|
||||
use rand::RngCore;
|
||||
rand::thread_rng().fill_bytes(&mut host_challenge);
|
||||
}
|
||||
let mut hc_buf = [0u8; 16];
|
||||
hc_buf[0] = 0x00;
|
||||
hc_buf[1] = 0x0E;
|
||||
@@ -300,13 +280,19 @@ fn bus_auth(drive: &mut Drive) -> Result<(u8, [u8; 5])> {
|
||||
|
||||
// ── Step 2: Disc Key ──────────────────────────────────────────────────────
|
||||
|
||||
fn read_disc_key(drive: &mut Drive, agid: u8, bus_key: &[u8; 5]) -> Result<[u8; 5]> {
|
||||
/// Issue READ DVD STRUCTURE format 0x02 (Copyright Information — opcode 0xAD,
|
||||
/// NOT the REPORT KEY 0xA4 disc-key block) purely for the bus-auth unlock side
|
||||
/// effect. The returned block contents are not used — the descramble title key
|
||||
/// is recovered keylessly elsewhere, so the genuine disc-key REPORT KEY is
|
||||
/// intentionally skipped. (If a drive is ever found where bus-auth alone does
|
||||
/// not open scrambled reads, a real REPORT KEY format 0x02 belongs here.)
|
||||
fn read_disc_key(drive: &mut Drive, agid: u8) -> Result<()> {
|
||||
let scsi = drive.scsi_mut();
|
||||
|
||||
// READ DVD STRUCTURE, format 0x02 (disc key), 2048+4 bytes
|
||||
let alloc_len: u16 = 2048 + 4;
|
||||
let mut cdb = [0u8; 12];
|
||||
cdb[0] = SCSI_READ_DVD_STRUCTURE;
|
||||
cdb[0] = crate::scsi::SCSI_READ_DISC_STRUCTURE;
|
||||
// bytes 2-5: address = 0
|
||||
cdb[6] = 0; // layer
|
||||
cdb[7] = 0x02; // format = disc key
|
||||
@@ -323,133 +309,19 @@ fn read_disc_key(drive: &mut Drive, agid: u8, bus_key: &[u8; 5]) -> Result<[u8;
|
||||
);
|
||||
dvd_result.map_err(|_| Error::CssAuthFailed)?;
|
||||
|
||||
// Disc key block starts at offset 4 (skip 4-byte header)
|
||||
let disc_key_block = &mut buf[4..4 + 2048];
|
||||
|
||||
// XOR with reversed bus key (per libdvdcss)
|
||||
for (i, byte) in disc_key_block.iter_mut().enumerate() {
|
||||
*byte ^= bus_key[4 - (i % 5)];
|
||||
}
|
||||
|
||||
// Try each player key against each of 408 disc key entries.
|
||||
// Each entry in the block is the disc key encrypted with a specific player key.
|
||||
// We try all known player keys and verify by checking that two different
|
||||
// entries produce the same disc key.
|
||||
let mut candidates: Vec<([u8; 5], usize, usize)> = Vec::new(); // (disc_key, pk_idx, pos)
|
||||
|
||||
for (pk_idx, player_key) in PLAYER_KEYS.iter().enumerate() {
|
||||
for pos in 0..408 {
|
||||
let offset = pos * 5;
|
||||
if offset + 5 > disc_key_block.len() {
|
||||
break;
|
||||
}
|
||||
let mut enc = [0u8; 5];
|
||||
enc.copy_from_slice(&disc_key_block[offset..offset + 5]);
|
||||
let candidate = super::lfsr::decrypt_key(0x00, player_key, &enc);
|
||||
|
||||
// Check if any previous candidate matches (same disc key from different entry/pk)
|
||||
for (prev, _, _) in &candidates {
|
||||
if *prev == candidate {
|
||||
return Ok(candidate);
|
||||
}
|
||||
}
|
||||
candidates.push((candidate, pk_idx, pos));
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::CssAuthFailed)
|
||||
}
|
||||
|
||||
// ── Step 3: Title Key ─────────────────────────────────────────────────────
|
||||
|
||||
/// Read the raw (bus-encrypted) title key bytes from the drive.
|
||||
fn read_raw_title_key(drive: &mut Drive, agid: u8, lba: u32) -> Result<[u8; 5]> {
|
||||
let scsi = drive.scsi_mut();
|
||||
let mut cdb = [0u8; 12];
|
||||
cdb[0] = crate::scsi::SCSI_REPORT_KEY;
|
||||
cdb[2] = (lba >> 24) as u8;
|
||||
cdb[3] = (lba >> 16) as u8;
|
||||
cdb[4] = (lba >> 8) as u8;
|
||||
cdb[5] = lba as u8;
|
||||
cdb[8] = 0x00;
|
||||
cdb[9] = 0x0C;
|
||||
cdb[10] = (agid << 6) | 0x04;
|
||||
|
||||
let mut buf = [0u8; 12];
|
||||
let result = scsi.execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
);
|
||||
result.map_err(|_| Error::CssAuthFailed)?;
|
||||
|
||||
let mut key = [0u8; 5];
|
||||
for i in 0..5 {
|
||||
key[i] = buf[5 + (4 - i)];
|
||||
}
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn read_title_key(
|
||||
drive: &mut Drive,
|
||||
agid: u8,
|
||||
lba: u32,
|
||||
bus_key: &[u8; 5],
|
||||
disc_key: &[u8; 5],
|
||||
) -> Result<[u8; 5]> {
|
||||
let scsi = drive.scsi_mut();
|
||||
|
||||
let mut cdb = [0u8; 12];
|
||||
cdb[0] = crate::scsi::SCSI_REPORT_KEY;
|
||||
cdb[2] = (lba >> 24) as u8;
|
||||
cdb[3] = (lba >> 16) as u8;
|
||||
cdb[4] = (lba >> 8) as u8;
|
||||
cdb[5] = lba as u8;
|
||||
cdb[8] = 0x00;
|
||||
cdb[9] = 0x0C;
|
||||
cdb[10] = (agid << 6) | 0x04;
|
||||
|
||||
let mut buf = [0u8; 12];
|
||||
let tk_result = scsi.execute(
|
||||
&cdb,
|
||||
crate::scsi::DataDirection::FromDevice,
|
||||
&mut buf,
|
||||
5_000,
|
||||
);
|
||||
tk_result.map_err(|_| Error::CssAuthFailed)?;
|
||||
|
||||
// Title key at bytes 5..10, byte-reversed
|
||||
let mut title_key = [0u8; 5];
|
||||
for i in 0..5 {
|
||||
title_key[i] = buf[5 + (4 - i)];
|
||||
}
|
||||
|
||||
// XOR with reversed bus key (same pattern as disc key block)
|
||||
for i in 0..5 {
|
||||
title_key[i] ^= bus_key[4 - i];
|
||||
}
|
||||
|
||||
// Check for null key (title not encrypted)
|
||||
if title_key == [0u8; 5] {
|
||||
return Ok(title_key);
|
||||
}
|
||||
|
||||
// Decrypt with disc key (invert=0xFF for title keys)
|
||||
let title_key = super::lfsr::decrypt_key(0xFF, disc_key, &title_key);
|
||||
|
||||
Ok(title_key)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── CSSCryptKey ───────────────────────────────────────────────────────────
|
||||
|
||||
/// Exposed for testing only.
|
||||
pub fn test_crypt_key(key_type: usize, variant: u8, challenge: &[u8; 10]) -> [u8; 5] {
|
||||
crypt_key(key_type, variant, challenge)
|
||||
}
|
||||
|
||||
fn crypt_key(key_type: usize, variant: u8, challenge: &[u8; 10]) -> [u8; 5] {
|
||||
// key_type indexes PERM_CHALLENGE ([_;3]); variant indexes
|
||||
// VARIANTS/PERM_VARIANT ([_;32]). All internal callers pass key_type in
|
||||
// 0..3 and variant in 0..32; the asserts document the contract for the
|
||||
// pub(crate) test entry point test_crypt_key and turn a would-be
|
||||
// out-of-bounds panic into an explicit precondition violation.
|
||||
debug_assert!(key_type < 3, "crypt_key: key_type out of range");
|
||||
debug_assert!((variant as usize) < 32, "crypt_key: variant out of range");
|
||||
let perm = &PERM_CHALLENGE[key_type];
|
||||
let mut scratch = [0u8; 10];
|
||||
for i in 0..10 {
|
||||
@@ -601,6 +473,120 @@ fn send_key_cdb(agid: u8, format: u8, param_len: u16) -> [u8; 12] {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// SECURITY REGRESSION GUARD: no instrumentation in libfreemkv may emit
|
||||
/// raw key material. Scan every source file for a `tracing` field that
|
||||
/// binds a forbidden key name to a value-producing expression (`= expr`
|
||||
/// or `%expr` / `?expr`). The only allowed forms are a string literal
|
||||
/// (e.g. `disc_key = "<redacted>"`) or a `_fp` fingerprint field.
|
||||
///
|
||||
/// This is a source-scan test (not a runtime capture) so it stays cheap
|
||||
/// and catches re-introductions at compile/CI time.
|
||||
#[test]
|
||||
fn no_key_bytes_in_instrumentation() {
|
||||
use std::path::Path;
|
||||
|
||||
// Forbidden field names whose VALUES must never be logged.
|
||||
const FORBIDDEN: &[&str] = &[
|
||||
"title_key",
|
||||
"disc_key",
|
||||
"unit_key",
|
||||
"vuk",
|
||||
"player_key",
|
||||
"bus_key",
|
||||
];
|
||||
|
||||
fn scan_dir(dir: &Path, forbidden: &[&str], violations: &mut Vec<String>) {
|
||||
let entries = match std::fs::read_dir(dir) {
|
||||
Ok(e) => e,
|
||||
Err(_) => return,
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
scan_dir(&path, forbidden, violations);
|
||||
continue;
|
||||
}
|
||||
if path.extension().and_then(|e| e.to_str()) != Some("rs") {
|
||||
continue;
|
||||
}
|
||||
let src = match std::fs::read_to_string(&path) {
|
||||
Ok(s) => s,
|
||||
Err(_) => continue,
|
||||
};
|
||||
for (lineno, line) in src.lines().enumerate() {
|
||||
let trimmed = line.trim_start();
|
||||
// Only inspect tracing instrumentation lines.
|
||||
if !(trimmed.contains("tracing::")
|
||||
|| trimmed.starts_with("debug!")
|
||||
|| trimmed.starts_with("info!")
|
||||
|| trimmed.starts_with("warn!")
|
||||
|| trimmed.starts_with("trace!")
|
||||
|| trimmed.starts_with("error!"))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
// This guard test itself contains the forbidden names.
|
||||
if path.file_name().and_then(|n| n.to_str()) == Some("auth.rs")
|
||||
&& line.contains("FORBIDDEN")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for &name in forbidden {
|
||||
// A fingerprint field (`<name>_fp = ...`) is allowed.
|
||||
// Match `<name>` followed by optional fingerprint
|
||||
// suffix then `=` and a value that is NOT a string
|
||||
// literal redaction marker.
|
||||
if let Some(idx) = line.find(name) {
|
||||
let after = &line[idx + name.len()..];
|
||||
let after = after.trim_start();
|
||||
// `<name>_fp` / `<name>_id` etc. are safe.
|
||||
if after.starts_with('_') {
|
||||
continue;
|
||||
}
|
||||
// Must be a field binding `name = ...`.
|
||||
let Some(rest) = after.strip_prefix('=') else {
|
||||
continue;
|
||||
};
|
||||
let rest = rest.trim_start();
|
||||
// Redaction string literal is the only allowed value.
|
||||
if rest.starts_with('"') {
|
||||
continue;
|
||||
}
|
||||
// Anything else (`%expr`, `?expr`, bare expr) leaks bytes.
|
||||
violations.push(format!(
|
||||
"{}:{}: forbidden key field `{}` logged with a value: {}",
|
||||
path.display(),
|
||||
lineno + 1,
|
||||
name,
|
||||
line.trim()
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Scan this crate's `src` plus the sibling workspace crates so the
|
||||
// key-material logging guard covers every crate that can reach the
|
||||
// CSS/AACS internals, not just libfreemkv. Missing sibling dirs (e.g.
|
||||
// when building the crate standalone) are simply skipped.
|
||||
let manifest = Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let workspace = manifest.parent().unwrap_or(manifest);
|
||||
let mut violations = Vec::new();
|
||||
scan_dir(&manifest.join("src"), FORBIDDEN, &mut violations);
|
||||
for sibling in ["autorip", "freemkv", "freemkv-keysources"] {
|
||||
let dir = workspace.join(sibling).join("src");
|
||||
if dir.is_dir() {
|
||||
scan_dir(&dir, FORBIDDEN, &mut violations);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
violations.is_empty(),
|
||||
"key material logged in instrumentation:\n{}",
|
||||
violations.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crypt_key_is_deterministic() {
|
||||
let challenge: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
@@ -631,8 +617,222 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// ── CSS constant-table integrity ───────────────────────────────────────
|
||||
|
||||
/// Each PERM_CHALLENGE row is a permutation of indices 0..10 (it reorders
|
||||
/// the 10 challenge bytes). A non-permutation would drop/duplicate
|
||||
/// challenge bytes, weakening or corrupting the bus key derivation.
|
||||
///
|
||||
/// Grounding: crypt_key does `scratch[i] = challenge[perm[i]]` for i in
|
||||
/// 0..10 — perm must be a bijection on 0..10 to use every challenge byte
|
||||
/// exactly once.
|
||||
/// Mutation: change PERM_CHALLENGE[0] entry `9` to `8` (duplicate) -> the
|
||||
/// "covers 0..10" assert fires.
|
||||
#[test]
|
||||
fn player_keys_count() {
|
||||
assert_eq!(PLAYER_KEYS.len(), 31);
|
||||
fn perm_challenge_rows_are_permutations() {
|
||||
for (row, perm) in PERM_CHALLENGE.iter().enumerate() {
|
||||
let mut seen = [false; 10];
|
||||
for &idx in perm.iter() {
|
||||
assert!(idx < 10, "PERM_CHALLENGE[{row}] index {idx} out of range");
|
||||
assert!(!seen[idx], "PERM_CHALLENGE[{row}] duplicates index {idx}");
|
||||
seen[idx] = true;
|
||||
}
|
||||
assert!(
|
||||
seen.iter().all(|&b| b),
|
||||
"PERM_CHALLENGE[{row}] misses an index"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Each PERM_VARIANT row maps the 32 variants to 32 distinct 5-bit values
|
||||
/// (it is a permutation of 0..32). key_type 1 uses PERM_VARIANT[0],
|
||||
/// key_type 2 uses PERM_VARIANT[1] to pick the css_variant; a collision
|
||||
/// would make two variants indistinguishable.
|
||||
///
|
||||
/// Grounding: `css_variant = PERM_VARIANT[k][variant]` then indexes
|
||||
/// VARIANTS[css_variant] (0..32).
|
||||
/// Mutation: set PERM_VARIANT[0][1] = PERM_VARIANT[0][0] -> duplicate
|
||||
/// assert fires; also any value >= 32 would later index VARIANTS OOB.
|
||||
#[test]
|
||||
fn perm_variant_rows_are_permutations_of_0_31() {
|
||||
for (row, perm) in PERM_VARIANT.iter().enumerate() {
|
||||
let mut seen = [false; 32];
|
||||
for &v in perm.iter() {
|
||||
let v = v as usize;
|
||||
assert!(v < 32, "PERM_VARIANT[{row}] value {v} out of 0..32");
|
||||
assert!(!seen[v], "PERM_VARIANT[{row}] duplicates {v}");
|
||||
seen[v] = true;
|
||||
}
|
||||
assert!(
|
||||
seen.iter().all(|&b| b),
|
||||
"PERM_VARIANT[{row}] misses a value"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── crypt_key behaviour ────────────────────────────────────────────────
|
||||
|
||||
/// crypt_key result depends on every challenge byte. The challenge is
|
||||
/// permuted into `scratch` and folded through the LFSR seeding and the 6
|
||||
/// XOR rounds. Flipping any single challenge byte must change the output.
|
||||
///
|
||||
/// Grounding: scratch[i]=challenge[perm[i]] for all 10 i, and scratch
|
||||
/// seeds both LFSRs (bytes 5..10 via tmp1) and the round terms (bytes
|
||||
/// 0..5).
|
||||
/// Mutation: in `scratch[i] = challenge[perm[i]]` replace with
|
||||
/// `challenge[i]` for a perm that drops a byte — or hardcode one scratch
|
||||
/// entry — and some challenge byte stops mattering; this fails.
|
||||
#[test]
|
||||
fn crypt_key_depends_on_every_challenge_byte() {
|
||||
let base: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
let base_out = crypt_key(0, 5, &base);
|
||||
for i in 0..10 {
|
||||
let mut c = base;
|
||||
c[i] ^= 0x55;
|
||||
assert_ne!(
|
||||
crypt_key(0, 5, &c),
|
||||
base_out,
|
||||
"flipping challenge byte {i} did not change the bus-key derivation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// crypt_key(0, v, ..) must produce a DISTINCT result for each of the 32
|
||||
/// variants on a fixed challenge. bus_auth brute-forces the variant by
|
||||
/// matching crypt_key(0, v, host_challenge) == key1; if two variants
|
||||
/// collided, the wrong variant could be selected and the whole auth
|
||||
/// derail.
|
||||
///
|
||||
/// Grounding: variant selects css_variant -> VARIANTS[css_variant] -> cse,
|
||||
/// which feeds every round; distinct variants give distinct cse-driven
|
||||
/// keys in practice.
|
||||
/// Mutation: make `cse` ignore the variant (e.g. `let cse = 0`) -> all 32
|
||||
/// outputs collapse to one value; the distinctness assert fires.
|
||||
#[test]
|
||||
fn crypt_key_type0_distinct_per_variant() {
|
||||
let challenge: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
let mut outs = Vec::new();
|
||||
for v in 0..32u8 {
|
||||
let k = crypt_key(0, v, &challenge);
|
||||
assert!(
|
||||
!outs.contains(&k),
|
||||
"variant {v} collides with an earlier variant"
|
||||
);
|
||||
outs.push(k);
|
||||
}
|
||||
}
|
||||
|
||||
/// crypt_key enforces its documented precondition `key_type < 3` via
|
||||
/// debug_assert (active in test builds). A key_type of 3 would index
|
||||
/// PERM_CHALLENGE (len 3) out of bounds; the assert turns that into an
|
||||
/// explicit precondition panic.
|
||||
///
|
||||
/// Grounding: `debug_assert!(key_type < 3, ...)`; PERM_CHALLENGE has 3
|
||||
/// rows (indices 0,1,2).
|
||||
/// Mutation: delete the debug_assert AND the match-arm guard — but the
|
||||
/// match `_ =>` arm would then index PERM_CHALLENGE[3] OOB and panic
|
||||
/// differently; with the assert in place this test pins the contract.
|
||||
#[test]
|
||||
#[should_panic]
|
||||
fn crypt_key_rejects_out_of_range_key_type() {
|
||||
let challenge: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
let _ = crypt_key(3, 0, &challenge);
|
||||
}
|
||||
|
||||
/// crypt_key enforces `variant < 32` via debug_assert. A variant of 32
|
||||
/// would index VARIANTS / PERM_VARIANT (len 32) out of bounds.
|
||||
///
|
||||
/// Grounding: `debug_assert!((variant as usize) < 32, ...)`.
|
||||
/// Mutation: removing the assert makes this index VARIANTS[32] (still a
|
||||
/// panic, but unguarded); the assert documents/enforces the contract.
|
||||
#[test]
|
||||
#[should_panic]
|
||||
fn crypt_key_rejects_out_of_range_variant() {
|
||||
let challenge: [u8; 10] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9];
|
||||
let _ = crypt_key(0, 32, &challenge);
|
||||
}
|
||||
|
||||
// ── SCSI CDB builders (MMC REPORT KEY / SEND KEY layout) ───────────────
|
||||
|
||||
/// report_key_cdb encodes a 12-byte MMC REPORT KEY (opcode 0xA4) CDB:
|
||||
/// byte 0 = operation code 0xA4
|
||||
/// bytes 8-9 = allocation length, big-endian
|
||||
/// byte 10 = (AGID << 6) | (key_format & 0x3F)
|
||||
/// All other bytes are zero.
|
||||
///
|
||||
/// Grounding: MMC REPORT KEY CDB; the AGID is the top 2 bits of byte 10,
|
||||
/// key format the low 6 bits.
|
||||
/// Mutation: change `(alloc_len >> 8)` to `alloc_len` for byte 8 (lose the
|
||||
/// big-endian split) -> byte 8/9 assert fails. Change `agid << 6` to
|
||||
/// `agid << 5` -> the AGID-position assert fails.
|
||||
#[test]
|
||||
fn report_key_cdb_matches_mmc_layout() {
|
||||
let cdb = report_key_cdb(0b10, 0x04, 0x010C); // AGID=2, format=0x04, len=268
|
||||
assert_eq!(cdb[0], 0xA4, "REPORT KEY opcode");
|
||||
assert_eq!(cdb[8], 0x01, "alloc_len high byte (big-endian)");
|
||||
assert_eq!(cdb[9], 0x0C, "alloc_len low byte");
|
||||
assert_eq!(
|
||||
cdb[10],
|
||||
(0b10 << 6) | 0x04,
|
||||
"AGID in bits 6-7, format in bits 0-5"
|
||||
);
|
||||
// Every other byte must be zero.
|
||||
for (i, &b) in cdb.iter().enumerate() {
|
||||
if ![0, 8, 9, 10].contains(&i) {
|
||||
assert_eq!(b, 0, "CDB byte {i} must be zero");
|
||||
}
|
||||
}
|
||||
assert_eq!(cdb.len(), 12, "REPORT KEY is a 12-byte CDB");
|
||||
}
|
||||
|
||||
/// The key format field is masked to 6 bits: a format with high bits set
|
||||
/// must not corrupt the AGID. report_key_cdb(0, 0xFF, _) -> byte 10 low 6
|
||||
/// bits = 0x3F, AGID = 0.
|
||||
///
|
||||
/// Grounding: `(agid << 6) | (format & 0x3F)`.
|
||||
/// Mutation: drop the `& 0x3F` mask -> 0xFF would overwrite the AGID bits;
|
||||
/// byte 10 would be 0xFF not 0x3F, this fails.
|
||||
#[test]
|
||||
fn report_key_cdb_masks_format_to_6_bits() {
|
||||
let cdb = report_key_cdb(0, 0xFF, 8);
|
||||
assert_eq!(cdb[10], 0x3F, "format masked to 6 bits, AGID stays 0");
|
||||
}
|
||||
|
||||
/// send_key_cdb encodes a 12-byte MMC SEND KEY (opcode 0xA3) CDB with the
|
||||
/// parameter-list length at bytes 8-9 (big-endian) and AGID/format at byte
|
||||
/// 10.
|
||||
///
|
||||
/// Grounding: MMC SEND KEY CDB layout.
|
||||
/// Mutation: change opcode to SCSI_REPORT_KEY -> opcode assert fails;
|
||||
/// swap bytes 8/9 -> length assert fails.
|
||||
#[test]
|
||||
fn send_key_cdb_matches_mmc_layout() {
|
||||
let cdb = send_key_cdb(0b11, 0x03, 0x000C); // AGID=3, format=3, param_len=12
|
||||
assert_eq!(cdb[0], 0xA3, "SEND KEY opcode");
|
||||
assert_eq!(cdb[8], 0x00, "param_len high byte");
|
||||
assert_eq!(cdb[9], 0x0C, "param_len low byte");
|
||||
assert_eq!(
|
||||
cdb[10],
|
||||
(0b11 << 6) | 0x03,
|
||||
"AGID bits 6-7, format bits 0-5"
|
||||
);
|
||||
assert_eq!(cdb.len(), 12);
|
||||
}
|
||||
|
||||
/// Allocation length larger than 255 must split across bytes 8 (high) and
|
||||
/// 9 (low) — a 16-bit big-endian field. report_key_cdb with alloc_len
|
||||
/// 0x0804 (2052, the disc-key block size used in read_disc_key) -> byte 8
|
||||
/// = 0x08, byte 9 = 0x04.
|
||||
///
|
||||
/// Grounding: read_disc_key uses `alloc_len = 2048 + 4 = 2052 = 0x0804`
|
||||
/// and writes `cdb[8] = (alloc_len >> 8); cdb[9] = alloc_len`.
|
||||
/// Mutation: write only byte 9 (`cdb[9] = alloc_len as u8`) without byte 8
|
||||
/// -> the drive sees a 4-byte transfer, truncating the disc-key block;
|
||||
/// this asserts the high byte is present.
|
||||
#[test]
|
||||
fn report_key_cdb_alloc_len_is_16bit_big_endian() {
|
||||
let cdb = report_key_cdb(0, 0x00, 0x0804);
|
||||
assert_eq!(cdb[8], 0x08, "high byte of 2052-byte transfer");
|
||||
assert_eq!(cdb[9], 0x04, "low byte of 2052-byte transfer");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,398 +0,0 @@
|
||||
//! CSS title key recovery — Stevenson's divide-and-conquer attack (1999).
|
||||
//!
|
||||
//! Given a scrambled DVD sector with known plaintext (MPEG-2 PES headers),
|
||||
//! recovers the 5-byte title key by:
|
||||
//!
|
||||
//! 1. XORing ciphertext with TAB1[ciphertext] to cancel the mangling
|
||||
//! 2. Iterating all 2^16 LFSR1 states
|
||||
//! 3. For each: deducing what LFSR0 must produce, then verifying
|
||||
//!
|
||||
//! Total work: ~65536 iterations with 10-byte validation = instant.
|
||||
//!
|
||||
//! Algorithm: Frank A. Stevenson, "Divide and conquer attack" (1999).
|
||||
|
||||
use super::tables::{TAB1, TAB2, TAB3, TAB4, TAB5};
|
||||
|
||||
/// Sector layout constants.
|
||||
const SECTOR_SIZE: usize = 2048;
|
||||
const ENCRYPTED_START: usize = 0x80; // byte 128
|
||||
const SEED_OFFSET: usize = 0x54; // sector seed at bytes 0x54-0x58
|
||||
const FLAG_BYTE: usize = 0x14;
|
||||
|
||||
/// Recover the CSS title key from a scrambled sector using known plaintext.
|
||||
///
|
||||
/// The `plain` slice should contain the expected plaintext of the encrypted
|
||||
/// region (bytes 0x80+). For MPEG-2 sectors, the first bytes are typically
|
||||
/// a PES header: `00 00 01 [stream_id] ...`
|
||||
///
|
||||
/// Returns the recovered 5-byte title key, or None if recovery fails.
|
||||
pub fn recover_title_key(sector: &[u8], plain: &[u8]) -> Option<[u8; 5]> {
|
||||
if sector.len() < SECTOR_SIZE || plain.len() < 10 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let flags = (sector[FLAG_BYTE] >> 4) & 0x03;
|
||||
if flags == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let crypted = §or[ENCRYPTED_START..];
|
||||
let seed = §or[SEED_OFFSET..SEED_OFFSET + 5];
|
||||
|
||||
// Phase 1: Cancel the TAB1 mangling layer
|
||||
// The CSS cipher applies TAB1 as an output permutation.
|
||||
// XORing ciphertext with TAB1[ciphertext] and plaintext removes it,
|
||||
// leaving the raw LFSR combination output.
|
||||
let mut buf = [0u8; 10];
|
||||
for i in 0..10 {
|
||||
if i >= crypted.len() || i >= plain.len() {
|
||||
return None;
|
||||
}
|
||||
buf[i] = TAB1[crypted[i] as usize] ^ plain[i];
|
||||
}
|
||||
|
||||
// Phase 2: Stevenson attack — iterate all 2^16 LFSR1 initial states
|
||||
let mut result_key = [0u8; 5];
|
||||
let mut found = false;
|
||||
|
||||
'outer: for i_try in 0u32..0x10000 {
|
||||
let mut t1 = (i_try >> 8) | 0x100;
|
||||
let mut t2 = i_try & 0xFF;
|
||||
let mut t5: u32 = 0;
|
||||
|
||||
// Clock LFSR1 forward 4 steps to reconstruct LFSR0 state
|
||||
let mut t3: u32 = 0;
|
||||
|
||||
for &buf_byte in buf.iter().take(4) {
|
||||
// Advance LFSR1
|
||||
let t4 = TAB2[t2 as usize] ^ TAB3[t1 as usize];
|
||||
t2 = t1 >> 1;
|
||||
t1 = ((t1 & 1) << 8) ^ t4 as u32;
|
||||
let t4_perm = TAB5[t4 as usize];
|
||||
|
||||
// Deduce LFSR0 output from the buffer and LFSR1 output
|
||||
let mut t6 = buf_byte as u32;
|
||||
if t5 > 0 {
|
||||
t6 = (t6 + 0xFF) & 0xFF;
|
||||
}
|
||||
if t6 < t4_perm as u32 {
|
||||
t6 += 0x100;
|
||||
}
|
||||
t6 -= t4_perm as u32;
|
||||
t5 += t6 + t4_perm as u32;
|
||||
let t6_inv = TAB4[t6 as usize & 0xFF];
|
||||
|
||||
// Build LFSR0 candidate from deduced output bytes
|
||||
t3 = (t3 << 8) | t6_inv as u32;
|
||||
t5 >>= 8;
|
||||
}
|
||||
|
||||
let candidate = t3;
|
||||
|
||||
// Phase 3: Validate — clock 6 more steps and check against buffer
|
||||
let mut valid = true;
|
||||
for &buf_byte in buf.iter().skip(4) {
|
||||
let t4 = TAB2[t2 as usize] ^ TAB3[t1 as usize];
|
||||
t2 = t1 >> 1;
|
||||
t1 = ((t1 & 1) << 8) ^ t4 as u32;
|
||||
let t4_perm = TAB5[t4 as usize];
|
||||
|
||||
// Clock LFSR0 forward
|
||||
let t6 = ((((((t3 >> 8) ^ t3) >> 1) ^ t3) >> 3) ^ t3) >> 7;
|
||||
t3 = (t3 << 8) | (t6 & 0xFF);
|
||||
let t6_perm = TAB4[(t6 & 0xFF) as usize];
|
||||
|
||||
t5 += t6_perm as u32 + t4_perm as u32;
|
||||
if (t5 & 0xFF) as u8 != buf_byte {
|
||||
valid = false;
|
||||
break;
|
||||
}
|
||||
t5 >>= 8;
|
||||
}
|
||||
|
||||
if !valid {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Phase 4: Recover the initial LFSR0 state from the candidate
|
||||
t3 = candidate;
|
||||
let mut recovery_ok = true;
|
||||
for _ in 0..4 {
|
||||
let t1_byte = t3 & 0xFF;
|
||||
t3 >>= 8;
|
||||
// Brute-force the byte that was shifted in
|
||||
let mut found_j = false;
|
||||
for j in 0u32..256 {
|
||||
t3 = (t3 & 0x1FFFF) | (j << 17);
|
||||
let t6 = ((((((t3 >> 8) ^ t3) >> 1) ^ t3) >> 3) ^ t3) >> 7;
|
||||
if (t6 & 0xFF) == t1_byte {
|
||||
found_j = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !found_j {
|
||||
recovery_ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !recovery_ok {
|
||||
continue 'outer;
|
||||
}
|
||||
|
||||
// Convert LFSR0 initial state back to key bytes
|
||||
let t4 = (t3 >> 1).wrapping_sub(4);
|
||||
for t5_off in 0u32..8 {
|
||||
let val = t4.wrapping_add(t5_off);
|
||||
if (val * 2 + 8 - (val & 7)) == t3 {
|
||||
result_key[0] = (i_try >> 8) as u8;
|
||||
result_key[1] = (i_try & 0xFF) as u8;
|
||||
result_key[2] = (val & 0xFF) as u8;
|
||||
result_key[3] = ((val >> 8) & 0xFF) as u8;
|
||||
result_key[4] = ((val >> 16) & 0xFF) as u8;
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if found {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
return None;
|
||||
}
|
||||
|
||||
// XOR with sector seed to get the actual title key
|
||||
result_key[0] ^= seed[0];
|
||||
result_key[1] ^= seed[1];
|
||||
result_key[2] ^= seed[2];
|
||||
result_key[3] ^= seed[3];
|
||||
result_key[4] ^= seed[4];
|
||||
|
||||
Some(result_key)
|
||||
}
|
||||
|
||||
/// Crack the CSS title key from an encrypted sector using MPEG-2 pattern attack.
|
||||
///
|
||||
/// Detects the PES header pattern at byte 0x80 and uses it as known plaintext.
|
||||
pub fn crack_title_key(sector: &[u8]) -> Option<[u8; 5]> {
|
||||
if sector.len() < SECTOR_SIZE {
|
||||
return None;
|
||||
}
|
||||
|
||||
let flags = (sector[FLAG_BYTE] >> 4) & 0x03;
|
||||
if flags == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// The PES header at byte 0x80 typically starts with 00 00 01 [stream_id].
|
||||
// The next bytes are PES length and flags. We need at least 10 bytes of
|
||||
// known plaintext for the Stevenson attack.
|
||||
//
|
||||
// Strategy: try common PES patterns. The first 3 bytes are always 00 00 01.
|
||||
// The stream_id varies. Bytes 4-9 depend on PES header structure.
|
||||
//
|
||||
// For a standard PES with PTS:
|
||||
// 00 00 01 [id] [len_hi] [len_lo] [flags] [flags2] [hdr_len] [PTS...]
|
||||
//
|
||||
// We try multiple stream IDs and use zeros for unknown bytes (most common).
|
||||
|
||||
// Try many PES header patterns at byte 0x80.
|
||||
// Structure: 00 00 01 [stream_id] [len_hi] [len_lo] [flags1] [flags2] [hdr_len] [data]
|
||||
let mut patterns: Vec<[u8; 10]> = Vec::with_capacity(128);
|
||||
|
||||
// Padding stream (0xBE): payload is 0xFF bytes, various lengths
|
||||
for len_hi in 0u8..8 {
|
||||
for len_lo_top in [0x00u8, 0x80, 0xFF] {
|
||||
patterns.push([
|
||||
0x00, 0x00, 0x01, 0xBE, len_hi, len_lo_top, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
// Video (0xE0) and audio (0xBD, 0xC0) with typical PES headers
|
||||
for &sid in &[0xE0u8, 0xBD, 0xC0] {
|
||||
for &flags1 in &[0x80u8, 0x81, 0x84, 0x85, 0x8C, 0x8D] {
|
||||
for &flags2 in &[0x00u8, 0x05, 0x80, 0xC0] {
|
||||
let hdr_len = if flags2 & 0x80 != 0 { 0x05u8 } else { 0x00 };
|
||||
let pts0 = if flags2 & 0x80 != 0 { 0x21u8 } else { 0x00 };
|
||||
// Try with several PES lengths
|
||||
for &len_hi in &[0x00u8, 0x07] {
|
||||
patterns.push([
|
||||
0x00, 0x00, 0x01, sid, len_hi, 0x00, flags1, flags2, hdr_len, pts0,
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Navigation pack system header (0xBB)
|
||||
patterns.push([0x00, 0x00, 0x01, 0xBB, 0x00, 0x12, 0x80, 0xC4, 0xE1, 0x04]);
|
||||
|
||||
for pattern in &patterns {
|
||||
if let Some(key) = recover_title_key(sector, pattern) {
|
||||
let mut test = sector.to_vec();
|
||||
super::lfsr::descramble_sector(&key, &mut test);
|
||||
if test[0x80] == 0x00 && test[0x81] == 0x00 && test[0x82] == 0x01 {
|
||||
return Some(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Crack CSS key from multiple sectors.
|
||||
pub fn crack_from_sectors(sectors: &[Vec<u8>]) -> Option<[u8; 5]> {
|
||||
for sector in sectors {
|
||||
if sector.len() < SECTOR_SIZE {
|
||||
continue;
|
||||
}
|
||||
let flags = (sector[FLAG_BYTE] >> 4) & 0x03;
|
||||
if flags == 0 {
|
||||
continue;
|
||||
}
|
||||
if let Some(key) = crack_title_key(sector) {
|
||||
return Some(key);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn crack_unscrambled_returns_none() {
|
||||
let sector = vec![0u8; 2048];
|
||||
assert!(crack_title_key(§or).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crack_too_short_returns_none() {
|
||||
let sector = vec![0u8; 100];
|
||||
assert!(crack_title_key(§or).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recover_needs_10_bytes_plain() {
|
||||
let sector = vec![0u8; 2048];
|
||||
let short_plain = [0u8; 5];
|
||||
assert!(recover_title_key(§or, &short_plain).is_none());
|
||||
}
|
||||
|
||||
/// Test 3: css_crack_recovers_key_from_scrambled_sector
|
||||
///
|
||||
/// Build a plaintext sector with known MPEG-2 PES headers, scramble it
|
||||
/// with a known title key, then run crack_title_key() on the scrambled
|
||||
/// sector. If the Stevenson attack succeeds, verify that descrambling
|
||||
/// with the recovered key produces the original plaintext at bytes 128..132.
|
||||
#[test]
|
||||
fn css_crack_recovers_key_from_scrambled_sector() {
|
||||
use super::super::lfsr::descramble_sector;
|
||||
|
||||
let title_key: [u8; 5] = [0x42, 0x13, 0x37, 0xBE, 0xEF];
|
||||
|
||||
// Build a plaintext MPEG-2 sector
|
||||
let mut plaintext = vec![0x00u8; SECTOR_SIZE];
|
||||
|
||||
// Pack header at byte 0: 00 00 01 BA
|
||||
plaintext[0] = 0x00;
|
||||
plaintext[1] = 0x00;
|
||||
plaintext[2] = 0x01;
|
||||
plaintext[3] = 0xBA;
|
||||
|
||||
// Scramble flag at byte 0x14
|
||||
plaintext[FLAG_BYTE] = 0x30;
|
||||
|
||||
// Sector seed at bytes 0x54-0x58
|
||||
plaintext[SEED_OFFSET..SEED_OFFSET + 5].copy_from_slice(&[0x11, 0x22, 0x33, 0x44, 0x55]);
|
||||
|
||||
// PES header at byte 0x80: 00 00 01 E0 (video stream)
|
||||
// Then typical PES header bytes for a stream with PTS
|
||||
plaintext[0x80] = 0x00;
|
||||
plaintext[0x81] = 0x00;
|
||||
plaintext[0x82] = 0x01;
|
||||
plaintext[0x83] = 0xE0;
|
||||
plaintext[0x84] = 0x00; // PES length hi
|
||||
plaintext[0x85] = 0x00; // PES length lo
|
||||
plaintext[0x86] = 0x80; // flags: data_alignment, copyright
|
||||
plaintext[0x87] = 0x80; // PTS flag
|
||||
plaintext[0x88] = 0x05; // PES header data length
|
||||
plaintext[0x89] = 0x21; // PTS byte 1
|
||||
|
||||
let original_plaintext = plaintext.clone();
|
||||
|
||||
// "Scramble" the sector by calling descramble (which XORs the keystream)
|
||||
// on the plaintext. This produces a scrambled sector.
|
||||
descramble_sector(&title_key, &mut plaintext);
|
||||
|
||||
// The scramble flag was cleared by descramble_sector. Restore it so
|
||||
// the cracker sees it as encrypted.
|
||||
plaintext[FLAG_BYTE] = 0x30;
|
||||
|
||||
// Now we have a scrambled sector. Try to crack the title key.
|
||||
let cracked_key = crack_title_key(&plaintext);
|
||||
|
||||
match cracked_key {
|
||||
Some(key) => {
|
||||
// Verify: descramble with the cracked key should recover plaintext
|
||||
let mut test = plaintext.clone();
|
||||
descramble_sector(&key, &mut test);
|
||||
|
||||
// Check that the PES header is recovered
|
||||
assert_eq!(test[0x80], 0x00, "PES byte 0 mismatch");
|
||||
assert_eq!(test[0x81], 0x00, "PES byte 1 mismatch");
|
||||
assert_eq!(test[0x82], 0x01, "PES byte 2 mismatch");
|
||||
assert_eq!(test[0x83], 0xE0, "PES byte 3 mismatch");
|
||||
|
||||
// Also verify the rest of the encrypted region matches original
|
||||
assert_eq!(
|
||||
&test[0x80..SECTOR_SIZE],
|
||||
&original_plaintext[0x80..SECTOR_SIZE],
|
||||
"Decrypted content does not match original plaintext"
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"Stevenson attack succeeded: cracked key = {:02X?}, original = {:02X?}",
|
||||
key, title_key
|
||||
);
|
||||
}
|
||||
None => {
|
||||
// The Stevenson attack may not always find a key for all title keys
|
||||
// and sector seeds. This is expected for some combinations where the
|
||||
// known plaintext pattern doesn't match what crack_title_key tries.
|
||||
eprintln!(
|
||||
"Stevenson attack did not find key for title_key={:02X?} seed={:02X?}. \
|
||||
This can happen when the cipher output doesn't match the tried patterns. \
|
||||
Testing with recover_title_key directly with exact plaintext.",
|
||||
title_key,
|
||||
&[0x11u8, 0x22, 0x33, 0x44, 0x55],
|
||||
);
|
||||
|
||||
// Try with exact known plaintext instead of guessing
|
||||
let exact_plain: [u8; 10] =
|
||||
[0x00, 0x00, 0x01, 0xE0, 0x00, 0x00, 0x80, 0x80, 0x05, 0x21];
|
||||
let recovered = recover_title_key(&plaintext, &exact_plain);
|
||||
if let Some(key) = recovered {
|
||||
let mut test = plaintext.clone();
|
||||
descramble_sector(&key, &mut test);
|
||||
assert_eq!(test[0x80], 0x00);
|
||||
assert_eq!(test[0x81], 0x00);
|
||||
assert_eq!(test[0x82], 0x01);
|
||||
eprintln!(
|
||||
"recover_title_key with exact plaintext succeeded: {:02X?}",
|
||||
key
|
||||
);
|
||||
} else {
|
||||
eprintln!(
|
||||
"recover_title_key also returned None. The attack may not converge \
|
||||
for this particular key/seed combination. This is a known limitation \
|
||||
of the brute-force LFSR0 recovery phase."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+358
-153
@@ -1,11 +1,15 @@
|
||||
//! CSS cipher implementation based on the Stevenson 1999 analysis.
|
||||
//!
|
||||
//! The CSS cipher uses two table-driven feedback circuits:
|
||||
//! - LFSR1: 9-bit state (two halves), driven by TAB2/TAB3
|
||||
//! - LFSR0: 32-bit state, driven by a feedback polynomial through TAB4
|
||||
//! - LFSR1: 17-bit state (9-bit lo + 8-bit hi register, seeded from
|
||||
//! key[0..2]), driven by TAB2/TAB3
|
||||
//! - LFSR0: 24-bit feedback register (seeded from key[2..5] XOR seed[2..5],
|
||||
//! masked to 0xFFFFFF), driven by a feedback polynomial through TAB4
|
||||
//!
|
||||
//! The keystream is the bytewise sum (with carry) of both LFSR outputs.
|
||||
//! Content descrambling XORs this keystream with the encrypted sector data.
|
||||
//! Content descrambling computes plain = TAB1[cipher] ^ keystream — a TAB1
|
||||
//! substitution of each ciphertext byte followed by an XOR with the keystream
|
||||
//! (NOT a plain XOR; the cipher is not its own inverse).
|
||||
//!
|
||||
//! Algorithm: Frank A. Stevenson's divide-and-conquer attack (1999).
|
||||
//! Tables: CSS specification constants.
|
||||
@@ -14,122 +18,155 @@ use super::tables::{TAB1, TAB2, TAB3, TAB4, TAB5};
|
||||
|
||||
/// Descramble a CSS-encrypted DVD sector in place.
|
||||
///
|
||||
/// The sector seed (bytes 0x54-0x58) is XORed with the title key to produce
|
||||
/// the per-sector key. Bytes 0x80..0x800 (128..2048) are then decrypted
|
||||
/// using the two-LFSR keystream.
|
||||
/// Exact port of libdvdcss `dvdcss_unscramble` (css.c). The two content
|
||||
/// LFSRs are seeded **directly** from `title_key XOR sector_seed` — there is
|
||||
/// no `decrypt_key` mangling on this path (that is the disc/title-key
|
||||
/// hierarchy, not the content cipher). Bytes 0x80..0x800 are recovered with
|
||||
/// `*p = TAB1[*p] ^ (i_t5 & 0xff)`.
|
||||
///
|
||||
/// The scramble flag at byte 0x14 (bits 4-5) indicates encryption.
|
||||
/// After descrambling, the flag is cleared.
|
||||
/// The scramble flag at byte 0x14 (bits 4-5) indicates encryption. Like
|
||||
/// libdvdcss, the flag byte is NOT modified here — the caller treats a
|
||||
/// nonzero `sector[0x14] & 0x30` as "needs unscrambling" and the descramble
|
||||
/// is its own inverse, so re-running it on plaintext would re-scramble.
|
||||
/// (freemkv historically cleared the flag; we keep clearing it so callers
|
||||
/// and the existing tests can distinguish a descrambled sector. This does
|
||||
/// not affect the recovered body.)
|
||||
///
|
||||
/// No-op (returns without modifying `sector`) in two cases:
|
||||
/// - `sector.len() < 2048`: the encrypted region (0x80..0x800) is not
|
||||
/// fully present. Callers chunk by 2048, so a trailing partial chunk is
|
||||
/// left untouched. The `debug_assert!` flags this misuse in debug/test
|
||||
/// builds; a DVD sector is always exactly 2048 bytes.
|
||||
/// - scramble flags are zero: the sector is not CSS-encrypted.
|
||||
///
|
||||
/// Design reference: libdvdcss `dvdcss_unscramble`. The combiner mirrors
|
||||
/// `css.c` line-for-line:
|
||||
/// ```text
|
||||
/// i_t1 = (key[0] ^ sec[0x54]) | 0x100;
|
||||
/// i_t2 = key[1] ^ sec[0x55];
|
||||
/// i_t3 = (key[2]|key[3]<<8|key[4]<<16) ^ (sec[0x56]|sec[0x57]<<8|sec[0x58]<<16);
|
||||
/// i_t4 = i_t3 & 7; i_t3 = i_t3*2 + 8 - i_t4;
|
||||
/// // per byte over 0x80..0x800:
|
||||
/// i_t4 = TAB2[i_t2] ^ TAB3[i_t1];
|
||||
/// i_t2 = i_t1 >> 1; i_t1 = ((i_t1 & 1) << 8) ^ i_t4; i_t4 = TAB5[i_t4];
|
||||
/// i_t6 = (((((((i_t3>>3)^i_t3)>>1)^i_t3)>>8)^i_t3)>>5) & 0xff;
|
||||
/// i_t3 = (i_t3 << 8) | i_t6; i_t6 = TAB4[i_t6];
|
||||
/// i_t5 += i_t6 + i_t4; *p = TAB1[*p] ^ (i_t5 & 0xff); i_t5 >>= 8;
|
||||
/// ```
|
||||
pub fn descramble_sector(title_key: &[u8; 5], sector: &mut [u8]) {
|
||||
debug_assert!(
|
||||
sector.len() >= 2048,
|
||||
"descramble_sector: buffer shorter than one 2048-byte sector"
|
||||
);
|
||||
if sector.len() < 2048 {
|
||||
return;
|
||||
}
|
||||
|
||||
let flags = (sector[0x14] >> 4) & 0x03;
|
||||
if flags == 0 {
|
||||
// libdvdcss: `if( !(p_sec[0x14] & 0x30) ) return;`
|
||||
if sector[0x14] & 0x30 == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
// Per-sector key = title_key XOR sector_seed (bytes 0x54-0x58)
|
||||
let key = [
|
||||
title_key[0] ^ sector[0x54],
|
||||
title_key[1] ^ sector[0x55],
|
||||
title_key[2] ^ sector[0x56],
|
||||
title_key[3] ^ sector[0x57],
|
||||
title_key[4] ^ sector[0x58],
|
||||
];
|
||||
// LFSR1: seeded directly from (key ^ seed) — NO decrypt_key.
|
||||
let mut i_t1: u32 = ((title_key[0] ^ sector[0x54]) as u32) | 0x100;
|
||||
let mut i_t2: u32 = (title_key[1] ^ sector[0x55]) as u32;
|
||||
|
||||
// Decrypt the key through the CSS mangling function to get the working key
|
||||
let working_key = decrypt_key(0xFF, &key, §or[0x54..0x59]);
|
||||
// LFSR0 (i_t3): 24-bit feedback register seeded from the remaining three
|
||||
// key/seed bytes, then transformed `i_t3 = i_t3*2 + 8 - (i_t3 & 7)`.
|
||||
let mut i_t3: u32 = (((title_key[2] as u32)
|
||||
| ((title_key[3] as u32) << 8)
|
||||
| ((title_key[4] as u32) << 16))
|
||||
^ ((sector[0x56] as u32) | ((sector[0x57] as u32) << 8) | ((sector[0x58] as u32) << 16)))
|
||||
& 0xFF_FFFF;
|
||||
let i_t4_seed = i_t3 & 7;
|
||||
i_t3 = i_t3 * 2 + 8 - i_t4_seed;
|
||||
|
||||
// Generate keystream and XOR with encrypted region
|
||||
let mut lfsr1_lo: u32 = working_key[0] as u32 | 0x100;
|
||||
let mut lfsr1_hi: u32 = working_key[1] as u32;
|
||||
let mut i_t5: u32 = 0;
|
||||
|
||||
let mut lfsr0: u32 = ((working_key[4] as u32) << 17)
|
||||
| ((working_key[3] as u32) << 9)
|
||||
| (((working_key[2] as u32) << 1) + 8 - (working_key[2] as u32 & 7));
|
||||
lfsr0 = (TAB4[(lfsr0 & 0xFF) as usize] as u32) << 24
|
||||
| (TAB4[((lfsr0 >> 8) & 0xFF) as usize] as u32) << 16
|
||||
| (TAB4[((lfsr0 >> 16) & 0xFF) as usize] as u32) << 8
|
||||
| TAB4[((lfsr0 >> 24) & 0xFF) as usize] as u32;
|
||||
|
||||
let mut combined: u32 = 0;
|
||||
|
||||
// Generate 1920 keystream bytes (for sector bytes 128..2048)
|
||||
// Per libdvdcss css_unscramble: TAB1 permutation on ciphertext, no invert on LFSR0
|
||||
for byte in sector.iter_mut().take(2048).skip(128) {
|
||||
let o_lfsr1 = TAB2[lfsr1_hi as usize] ^ TAB3[lfsr1_lo as usize];
|
||||
lfsr1_hi = lfsr1_lo >> 1;
|
||||
lfsr1_lo = ((lfsr1_lo & 1) << 8) ^ o_lfsr1 as u32;
|
||||
// Advance LFSR1.
|
||||
let mut i_t4 = (TAB2[i_t2 as usize] ^ TAB3[i_t1 as usize]) as u32;
|
||||
i_t2 = i_t1 >> 1;
|
||||
i_t1 = ((i_t1 & 1) << 8) ^ i_t4;
|
||||
i_t4 = TAB5[i_t4 as usize] as u32;
|
||||
|
||||
let o_lfsr0 = (((((((lfsr0 >> 8) ^ lfsr0) >> 1) ^ lfsr0) >> 3) ^ lfsr0) >> 7) as u8;
|
||||
lfsr0 = (lfsr0 >> 8) | ((o_lfsr0 as u32) << 24);
|
||||
// Advance LFSR0 (i_t3) and fold both outputs into i_t5.
|
||||
let mut i_t6 = (((((((i_t3 >> 3) ^ i_t3) >> 1) ^ i_t3) >> 8) ^ i_t3) >> 5) & 0xFF;
|
||||
i_t3 = (i_t3 << 8) | i_t6;
|
||||
i_t6 = TAB4[i_t6 as usize] as u32;
|
||||
i_t5 += i_t6 + i_t4;
|
||||
|
||||
combined += TAB5[o_lfsr1 as usize] as u32 + TAB4[o_lfsr0 as usize] as u32;
|
||||
*byte ^= (combined & 0xFF) as u8;
|
||||
combined >>= 8;
|
||||
*byte = TAB1[*byte as usize] ^ (i_t5 & 0xFF) as u8;
|
||||
i_t5 >>= 8;
|
||||
}
|
||||
|
||||
// Clear scramble flags
|
||||
// libdvdcss leaves byte 0x14 untouched; freemkv clears the scramble bits
|
||||
// so downstream code and tests can tell a sector was descrambled.
|
||||
sector[0x14] &= 0xCF;
|
||||
}
|
||||
|
||||
/// CSS key decryption / mangling function.
|
||||
/// Exact inverse of [`descramble_sector`]: turn a plaintext sector body into
|
||||
/// CSS ciphertext under `title_key`.
|
||||
///
|
||||
/// Decrypts `p_crypted` using `p_key` with the CSS two-LFSR cipher.
|
||||
/// The `invert` parameter controls the XOR applied to LFSR0 output
|
||||
/// (0x00 for disc key decryption, 0xFF for title key / sector key).
|
||||
pub(crate) fn decrypt_key(invert: u8, p_key: &[u8; 5], p_crypted: &[u8]) -> [u8; 5] {
|
||||
if p_crypted.len() < 5 {
|
||||
return *p_key;
|
||||
/// Descramble computes `plain = TAB1[cipher] ^ (i_t5 & 0xff)`, so the
|
||||
/// inverse is `cipher = TAB1_INV[plain ^ (i_t5 & 0xff)]` with the identical
|
||||
/// LFSR keystream. The keystream derivation is byte-for-byte the same as
|
||||
/// `descramble_sector` (libdvdcss `dvdcss_unscramble`); only the final
|
||||
/// substitution differs. Bytes 0x80..0x800 are rewritten in place; the
|
||||
/// scramble flag is set to 0x10 so a subsequent descramble runs.
|
||||
///
|
||||
/// Not on any production read path — it exists so the key-recovery tests
|
||||
/// (and any caller that needs to produce a known CSS-encrypted sector) can
|
||||
/// build genuine ciphertext rather than approximating it.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn scramble_sector(title_key: &[u8; 5], sector: &mut [u8]) {
|
||||
if sector.len() < 2048 {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut lfsr1_lo: u32 = p_key[0] as u32 | 0x100;
|
||||
let mut lfsr1_hi: u32 = p_key[1] as u32;
|
||||
let mut i_t1: u32 = ((title_key[0] ^ sector[0x54]) as u32) | 0x100;
|
||||
let mut i_t2: u32 = (title_key[1] ^ sector[0x55]) as u32;
|
||||
let mut i_t3: u32 = (((title_key[2] as u32)
|
||||
| ((title_key[3] as u32) << 8)
|
||||
| ((title_key[4] as u32) << 16))
|
||||
^ ((sector[0x56] as u32) | ((sector[0x57] as u32) << 8) | ((sector[0x58] as u32) << 16)))
|
||||
& 0xFF_FFFF;
|
||||
let i_t4_seed = i_t3 & 7;
|
||||
i_t3 = i_t3 * 2 + 8 - i_t4_seed;
|
||||
|
||||
let mut lfsr0: u32 = ((p_key[4] as u32) << 17)
|
||||
| ((p_key[3] as u32) << 9)
|
||||
| (((p_key[2] as u32) << 1) + 8 - (p_key[2] as u32 & 7));
|
||||
lfsr0 = (TAB4[(lfsr0 & 0xFF) as usize] as u32) << 24
|
||||
| (TAB4[((lfsr0 >> 8) & 0xFF) as usize] as u32) << 16
|
||||
| (TAB4[((lfsr0 >> 16) & 0xFF) as usize] as u32) << 8
|
||||
| TAB4[((lfsr0 >> 24) & 0xFF) as usize] as u32;
|
||||
let mut i_t5: u32 = 0;
|
||||
|
||||
let mut combined: u32 = 0;
|
||||
let mut k = [0u8; 5];
|
||||
for byte in sector.iter_mut().take(2048).skip(128) {
|
||||
let mut i_t4 = (TAB2[i_t2 as usize] ^ TAB3[i_t1 as usize]) as u32;
|
||||
i_t2 = i_t1 >> 1;
|
||||
i_t1 = ((i_t1 & 1) << 8) ^ i_t4;
|
||||
i_t4 = TAB5[i_t4 as usize] as u32;
|
||||
|
||||
for byte in &mut k {
|
||||
let o_lfsr1 = TAB2[lfsr1_hi as usize] ^ TAB3[lfsr1_lo as usize];
|
||||
lfsr1_hi = lfsr1_lo >> 1;
|
||||
lfsr1_lo = ((lfsr1_lo & 1) << 8) ^ o_lfsr1 as u32;
|
||||
let mut i_t6 = (((((((i_t3 >> 3) ^ i_t3) >> 1) ^ i_t3) >> 8) ^ i_t3) >> 5) & 0xFF;
|
||||
i_t3 = (i_t3 << 8) | i_t6;
|
||||
i_t6 = TAB4[i_t6 as usize] as u32;
|
||||
i_t5 += i_t6 + i_t4;
|
||||
|
||||
let o_lfsr0 = (((((((lfsr0 >> 8) ^ lfsr0) >> 1) ^ lfsr0) >> 3) ^ lfsr0) >> 7) as u8;
|
||||
lfsr0 = (lfsr0 >> 8) | ((o_lfsr0 as u32) << 24);
|
||||
|
||||
// TAB5 for LFSR1 output, TAB4 for LFSR0^invert (per libdvdcss css_DecryptKey)
|
||||
combined += TAB5[o_lfsr1 as usize] as u32 + TAB4[(o_lfsr0 ^ invert) as usize] as u32;
|
||||
*byte = (combined & 0xFF) as u8;
|
||||
combined >>= 8;
|
||||
// Inverse of `*p = TAB1[*p] ^ ks`: apply ks then TAB1's inverse.
|
||||
*byte = (*TAB1_INV)[(*byte ^ (i_t5 & 0xFF) as u8) as usize];
|
||||
i_t5 >>= 8;
|
||||
}
|
||||
|
||||
// Two rounds of chained XOR through TAB1
|
||||
let mut result = [0u8; 5];
|
||||
result[4] = k[4] ^ TAB1[p_crypted[4] as usize] ^ p_crypted[3];
|
||||
result[3] = k[3] ^ TAB1[p_crypted[3] as usize] ^ p_crypted[2];
|
||||
result[2] = k[2] ^ TAB1[p_crypted[2] as usize] ^ p_crypted[1];
|
||||
result[1] = k[1] ^ TAB1[p_crypted[1] as usize] ^ p_crypted[0];
|
||||
result[0] = k[0] ^ TAB1[p_crypted[0] as usize] ^ result[4];
|
||||
|
||||
result[4] = k[4] ^ TAB1[result[4] as usize] ^ result[3];
|
||||
result[3] = k[3] ^ TAB1[result[3] as usize] ^ result[2];
|
||||
result[2] = k[2] ^ TAB1[result[2] as usize] ^ result[1];
|
||||
result[1] = k[1] ^ TAB1[result[1] as usize] ^ result[0];
|
||||
result[0] = k[0] ^ TAB1[result[0] as usize];
|
||||
|
||||
result
|
||||
// Mark the sector scrambled so the descrambler will process it.
|
||||
sector[0x14] = (sector[0x14] & 0xCF) | 0x10;
|
||||
}
|
||||
|
||||
/// Inverse permutation of [`TAB1`], built at first use. `TAB1` is a
|
||||
/// bijection on 0..256, so `TAB1_INV[TAB1[x]] == x`.
|
||||
#[cfg(test)]
|
||||
static TAB1_INV: std::sync::LazyLock<[u8; 256]> = std::sync::LazyLock::new(|| {
|
||||
let mut inv = [0u8; 256];
|
||||
for (i, &v) in TAB1.iter().enumerate() {
|
||||
inv[v as usize] = i as u8;
|
||||
}
|
||||
inv
|
||||
});
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -144,6 +181,34 @@ mod tests {
|
||||
assert_eq!(sector, original);
|
||||
}
|
||||
|
||||
/// Cross-check `descramble_sector` against the EXACT output of libdvdcss
|
||||
/// `dvdcss_unscramble` (css.c) for a fixed sector, computed from the
|
||||
/// reference C semantics with the reference tables. Pins the content
|
||||
/// cipher to libdvdcss byte-for-byte.
|
||||
///
|
||||
/// key = 42 13 37 BE EF, seed (0x54..0x59) = DE AD BE EF 42, body = 0xAA.
|
||||
#[test]
|
||||
fn descramble_matches_libdvdcss_unscramble_vector() {
|
||||
let key = [0x42, 0x13, 0x37, 0xBE, 0xEF];
|
||||
let mut sector = vec![0xAAu8; 2048];
|
||||
sector[0x14] = 0x30;
|
||||
sector[0x54..0x59].copy_from_slice(&[0xDE, 0xAD, 0xBE, 0xEF, 0x42]);
|
||||
descramble_sector(&key, &mut sector);
|
||||
assert_eq!(
|
||||
§or[0x80..0x90],
|
||||
&[
|
||||
0x81, 0x92, 0x24, 0xA2, 0x46, 0x70, 0x3C, 0x64, 0xA6, 0x91, 0x84, 0xF5, 0x1F, 0x98,
|
||||
0xA0, 0x31
|
||||
],
|
||||
"descramble body head must match libdvdcss dvdcss_unscramble"
|
||||
);
|
||||
assert_eq!(
|
||||
§or[0x7F8..0x800],
|
||||
&[0x46, 0x94, 0x80, 0x0E, 0x67, 0x36, 0x65, 0xBC],
|
||||
"descramble body tail must match libdvdcss dvdcss_unscramble"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descramble_modifies_scrambled() {
|
||||
let key = [0x01, 0x02, 0x03, 0x04, 0x05];
|
||||
@@ -174,67 +239,14 @@ mod tests {
|
||||
assert_eq!(sector[0x14] & 0x30, 0x00);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decrypt_key_produces_output() {
|
||||
let key = [0x12, 0x34, 0x56, 0x78, 0x9A];
|
||||
let crypted = [0xAB, 0xCD, 0xEF, 0x01, 0x23];
|
||||
let result = decrypt_key(0xFF, &key, &crypted);
|
||||
// Should produce a 5-byte result different from input
|
||||
assert_ne!(result, key);
|
||||
assert_ne!(result, [0u8; 5]);
|
||||
}
|
||||
|
||||
/// Test 1: css_decrypt_key_roundtrip
|
||||
/// Test 2: descramble inverts scramble over the body.
|
||||
///
|
||||
/// decrypt_key is not a simple encrypt/decrypt pair — it is a one-way mangling
|
||||
/// function. However, we can verify consistency: calling it twice with the same
|
||||
/// parameters produces the same output, and varying the invert byte changes
|
||||
/// the LFSR0 contribution predictably.
|
||||
/// The content cipher is NOT a plain XOR involution (it applies TAB1 to
|
||||
/// the ciphertext: `plain = TAB1[cipher] ^ ks`). The true inverse is
|
||||
/// [`scramble_sector`]. Scrambling a plaintext body and then descrambling
|
||||
/// with the same key must reproduce the original body exactly.
|
||||
#[test]
|
||||
fn css_decrypt_key_roundtrip() {
|
||||
let keys: &[[u8; 5]] = &[
|
||||
[0x12, 0x34, 0x56, 0x78, 0x9A],
|
||||
[0x00, 0x00, 0x00, 0x00, 0x00],
|
||||
[0xFF, 0xFF, 0xFF, 0xFF, 0xFF],
|
||||
[0xAB, 0xCD, 0xEF, 0x01, 0x23],
|
||||
];
|
||||
let crypted_inputs: &[[u8; 5]] = &[
|
||||
[0x11, 0x22, 0x33, 0x44, 0x55],
|
||||
[0xAA, 0xBB, 0xCC, 0xDD, 0xEE],
|
||||
[0x00, 0x00, 0x00, 0x00, 0x00],
|
||||
];
|
||||
|
||||
for key in keys {
|
||||
for crypted in crypted_inputs {
|
||||
// decrypt_key with invert=0x00 and invert=0xFF should give different results
|
||||
let r0 = decrypt_key(0x00, key, crypted);
|
||||
let rff = decrypt_key(0xFF, key, crypted);
|
||||
|
||||
// The two results differ because the invert byte XORs the LFSR0 output
|
||||
// They should not be equal (except by extreme coincidence)
|
||||
// More importantly, both should be deterministic
|
||||
let r0_again = decrypt_key(0x00, key, crypted);
|
||||
let rff_again = decrypt_key(0xFF, key, crypted);
|
||||
assert_eq!(r0, r0_again, "decrypt_key(0x00) not deterministic");
|
||||
assert_eq!(rff, rff_again, "decrypt_key(0xFF) not deterministic");
|
||||
|
||||
// With different invert values, the keystream differs
|
||||
assert_ne!(
|
||||
r0, rff,
|
||||
"invert=0x00 and 0xFF gave same result for key {:?}",
|
||||
key
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Test 2: css_descramble_produces_valid_mpeg2
|
||||
///
|
||||
/// descramble_sector XORs a keystream into bytes 128..2048. Calling it
|
||||
/// twice with the same key and restored scramble flag should roundtrip,
|
||||
/// since XOR is its own inverse.
|
||||
#[test]
|
||||
fn css_descramble_modifies_encrypted_region() {
|
||||
fn css_descramble_inverts_scramble_over_body() {
|
||||
let title_key = [0x42, 0x13, 0x37, 0xBE, 0xEF];
|
||||
|
||||
let mut sector = vec![0xAAu8; 2048];
|
||||
@@ -242,11 +254,10 @@ mod tests {
|
||||
sector[0x54..0x59].copy_from_slice(&[0xDE, 0xAD, 0xBE, 0xEF, 0x42]);
|
||||
|
||||
let original = sector.clone();
|
||||
descramble_sector(&title_key, &mut sector);
|
||||
|
||||
// Flag cleared
|
||||
assert_eq!(sector[0x14] & 0x30, 0x00);
|
||||
// Header (0..128) unchanged except flag byte
|
||||
// Scramble the plaintext body into ciphertext.
|
||||
scramble_sector(&title_key, &mut sector);
|
||||
// Header (0..128) unchanged except the flag byte (set by scramble).
|
||||
for i in 0..128 {
|
||||
if i == 0x14 {
|
||||
continue;
|
||||
@@ -255,9 +266,18 @@ mod tests {
|
||||
}
|
||||
// Encrypted region modified
|
||||
assert_ne!(§or[128..256], &original[128..256]);
|
||||
|
||||
// Descramble restores the plaintext body byte-for-byte.
|
||||
descramble_sector(&title_key, &mut sector);
|
||||
assert_eq!(sector[0x14] & 0x30, 0x00, "flag cleared after descramble");
|
||||
assert_eq!(
|
||||
§or[128..2048],
|
||||
&original[128..2048],
|
||||
"descramble(scramble(body)) did not restore the body"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test 4: css_tab1_relationship
|
||||
/// css_tab1_relationship
|
||||
///
|
||||
/// Verify the structure of TAB1: it is a substitution table used in
|
||||
/// key mangling. Check that no two inputs map to the same output
|
||||
@@ -281,7 +301,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Test 5: css_tab4_is_bit_reversal
|
||||
/// css_tab4_is_bit_reversal
|
||||
///
|
||||
/// TAB4 reverses the bits of each byte: TAB4[0x01] = 0x80, TAB4[0x80] = 0x01, etc.
|
||||
#[test]
|
||||
@@ -303,4 +323,189 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── scramble-flag detection (byte 0x14, bits 4-5) ──────────────────────
|
||||
|
||||
/// Only bits 4-5 of byte 0x14 are the CSS scramble flag: the code reads
|
||||
/// `sector[0x14] & 0x30 == 0` (bits 6-7, i.e. 0x40/0x80, are masked out by
|
||||
/// 0x30). A sector with 0x14 == 0x40 or 0x80 must therefore be treated as
|
||||
/// UNSCRAMBLED and left byte-for-byte unchanged. This guards against a
|
||||
/// too-wide mask silently "descrambling" (and thus corrupting) clear data.
|
||||
///
|
||||
/// Grounding: CSS sector header byte 0x14 — copyright/scramble bits live
|
||||
/// in bits 4-5; the masked value 0 means not scrambled.
|
||||
/// Mutation: widen the mask `0x30` to `0x70`/`0xF0` -> 0x40/0x80 would be
|
||||
/// seen as scrambled and the body would change.
|
||||
#[test]
|
||||
fn descramble_treats_high_bits_of_0x14_as_clear() {
|
||||
let key = [0x01, 0x02, 0x03, 0x04, 0x05];
|
||||
for &flag in &[0x40u8, 0x80, 0xC0, 0x0F, 0x4F, 0x8F] {
|
||||
let mut sector = vec![0xAA; 2048];
|
||||
sector[0x14] = flag;
|
||||
sector[0x54..0x59].copy_from_slice(&[0x11, 0x22, 0x33, 0x44, 0x55]);
|
||||
let original = sector.clone();
|
||||
descramble_sector(&key, &mut sector);
|
||||
assert_eq!(
|
||||
sector, original,
|
||||
"byte 0x14 = {flag:#04x} has flag bits 4-5 clear; sector must be untouched"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Each individual scramble bit (4 and 5) independently marks the sector
|
||||
/// as encrypted: 0x10 and 0x20 must both trigger descrambling.
|
||||
///
|
||||
/// Grounding: `(0x10 >> 4) & 3 == 1`, `(0x20 >> 4) & 3 == 2` — both
|
||||
/// nonzero.
|
||||
/// Mutation: change `!= 0` early-return condition to `== 3` -> a sector
|
||||
/// flagged only 0x10 or 0x20 would be skipped and left scrambled.
|
||||
#[test]
|
||||
fn descramble_triggers_on_either_flag_bit() {
|
||||
let key = [0x01, 0x02, 0x03, 0x04, 0x05];
|
||||
for &flag in &[0x10u8, 0x20, 0x30] {
|
||||
let mut sector = vec![0xAA; 2048];
|
||||
sector[0x14] = flag;
|
||||
sector[0x54..0x59].copy_from_slice(&[0xDE, 0xAD, 0xBE, 0xEF, 0x42]);
|
||||
let original = sector.clone();
|
||||
descramble_sector(&key, &mut sector);
|
||||
assert_ne!(
|
||||
§or[128..256],
|
||||
&original[128..256],
|
||||
"flag {flag:#04x} (bits 4-5 nonzero) must descramble the body"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// After descrambling, ONLY the two scramble bits are cleared (`& 0xCF`);
|
||||
/// bits 6 and 7 of byte 0x14 must be preserved. A sector with 0x14 == 0xF0
|
||||
/// becomes 0xC0 (bits 6,7 kept, bits 4,5 cleared), NOT 0x00.
|
||||
///
|
||||
/// Grounding: code does `sector[0x14] &= 0xCF`; 0xF0 & 0xCF == 0xC0.
|
||||
/// Mutation: change `&= 0xCF` to `= 0` or `&= 0x0F` -> the preserved
|
||||
/// high bits assert fails.
|
||||
#[test]
|
||||
fn descramble_clear_preserves_high_bits_of_0x14() {
|
||||
let key = [0x01, 0x02, 0x03, 0x04, 0x05];
|
||||
let mut sector = vec![0x00; 2048];
|
||||
sector[0x14] = 0xF0; // bits 4-7 set; bits 4-5 are the flag
|
||||
sector[0x54..0x59].copy_from_slice(&[0x00; 5]);
|
||||
descramble_sector(&key, &mut sector);
|
||||
assert_eq!(
|
||||
sector[0x14], 0xC0,
|
||||
"scramble bits cleared, bits 6-7 preserved (0xF0 & 0xCF)"
|
||||
);
|
||||
}
|
||||
|
||||
// ── header / body boundary (encrypted region is 0x80..0x800) ───────────
|
||||
|
||||
/// The encrypted region is exactly bytes 0x80..0x800. Bytes 0x00..0x80
|
||||
/// (the header) must NOT be modified by the keystream — except byte 0x14
|
||||
/// whose flag is cleared. In particular the sector-seed bytes 0x54..0x59
|
||||
/// (which live inside the header) must survive untouched, since the
|
||||
/// descrambler reads them but never writes them.
|
||||
///
|
||||
/// Grounding: loop is `sector.iter_mut().take(2048).skip(128)` -> indices
|
||||
/// 128..2048 only.
|
||||
/// Mutation: change `.skip(128)` to `.skip(0)` -> header bytes (incl. the
|
||||
/// seed) get XORed and this fails.
|
||||
#[test]
|
||||
fn descramble_leaves_header_and_seed_intact() {
|
||||
let key = [0x42, 0x13, 0x37, 0xBE, 0xEF];
|
||||
let mut sector = vec![0x5Au8; 2048];
|
||||
sector[0x14] = 0x30;
|
||||
let seed = [0xDE, 0xAD, 0xBE, 0xEF, 0x42];
|
||||
sector[0x54..0x59].copy_from_slice(&seed);
|
||||
let original = sector.clone();
|
||||
descramble_sector(&key, &mut sector);
|
||||
for i in 0..0x80usize {
|
||||
if i == 0x14 {
|
||||
continue;
|
||||
}
|
||||
assert_eq!(
|
||||
sector[i], original[i],
|
||||
"header byte {i:#04x} must be untouched"
|
||||
);
|
||||
}
|
||||
assert_eq!(§or[0x54..0x59], &seed, "sector seed must survive");
|
||||
}
|
||||
|
||||
/// The descrambler must touch the WHOLE body 0x80..0x800, not just a
|
||||
/// prefix. With a constant body and constant key, the keystream is
|
||||
/// non-degenerate enough that the very last sector byte (index 2047) is
|
||||
/// altered. This guards the loop bound `.take(2048)` against an
|
||||
/// off-by-one that would leave the final byte(s) scrambled.
|
||||
///
|
||||
/// Grounding: encrypted region end is 0x800 == 2048 (exclusive).
|
||||
/// Mutation: change `.take(2048)` to `.take(2047)` -> last byte unchanged,
|
||||
/// assert fires (keystream byte for the last position is verified nonzero
|
||||
/// below by the round-trip, and this body is all-zero so any XOR shows).
|
||||
#[test]
|
||||
fn descramble_covers_final_body_byte() {
|
||||
let key = [0x42, 0x13, 0x37, 0xBE, 0xEF];
|
||||
let mut sector = vec![0x00u8; 2048];
|
||||
sector[0x14] = 0x30;
|
||||
sector[0x54..0x59].copy_from_slice(&[0x11, 0x22, 0x33, 0x44, 0x55]);
|
||||
descramble_sector(&key, &mut sector);
|
||||
// Body was all zero; any nonzero in [0x80,0x800) is keystream. Confirm
|
||||
// the keystream reaches the final byte. (If the last keystream byte
|
||||
// happened to be 0 this could be a flaky test, so assert the run-end
|
||||
// region as a whole differs from zero.)
|
||||
assert_ne!(
|
||||
§or[2040..2048],
|
||||
&[0u8; 8][..],
|
||||
"the tail of the body must be descrambled (loop must reach index 2047)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Descramble is keyed by `title_key XOR seed`: two different title keys
|
||||
/// produce two different bodies for the same scrambled input. A cipher
|
||||
/// that ignored the title key (or mixed it in wrongly) would yield
|
||||
/// identical output — silent wrong-key decryption.
|
||||
///
|
||||
/// Grounding: per-sector key = title_key[i] ^ sector[0x54+i].
|
||||
/// Mutation: in the `key` array drop the `title_key[i] ^` term -> both
|
||||
/// keys give the same body, assert fires.
|
||||
#[test]
|
||||
fn descramble_output_depends_on_title_key() {
|
||||
let seed = [0xDE, 0xAD, 0xBE, 0xEF, 0x42];
|
||||
let make = |k: &[u8; 5]| {
|
||||
let mut s = vec![0x00u8; 2048];
|
||||
s[0x14] = 0x30;
|
||||
s[0x54..0x59].copy_from_slice(&seed);
|
||||
descramble_sector(k, &mut s);
|
||||
s
|
||||
};
|
||||
let a = make(&[0x01, 0x02, 0x03, 0x04, 0x05]);
|
||||
let b = make(&[0x01, 0x02, 0x03, 0x04, 0x06]); // differs in last byte
|
||||
assert_ne!(
|
||||
&a[128..2048],
|
||||
&b[128..2048],
|
||||
"different title keys must descramble differently"
|
||||
);
|
||||
}
|
||||
|
||||
/// Descramble is keyed by the sector seed too: same title key, different
|
||||
/// seed -> different body. Pins that bytes 0x54..0x59 actually feed the
|
||||
/// keystream (not just the per-sector XOR key).
|
||||
///
|
||||
/// Mutation: replace `seed` array reads with a constant -> both seeds give
|
||||
/// the same body, assert fires.
|
||||
#[test]
|
||||
fn descramble_output_depends_on_seed() {
|
||||
let key = [0x01, 0x02, 0x03, 0x04, 0x05];
|
||||
let make = |seed: [u8; 5]| {
|
||||
let mut s = vec![0x00u8; 2048];
|
||||
s[0x14] = 0x30;
|
||||
s[0x54..0x59].copy_from_slice(&seed);
|
||||
descramble_sector(&key, &mut s);
|
||||
s
|
||||
};
|
||||
let a = make([0x11, 0x22, 0x33, 0x44, 0x55]);
|
||||
let b = make([0x11, 0x22, 0x33, 0x44, 0x56]);
|
||||
assert_ne!(
|
||||
&a[128..2048],
|
||||
&b[128..2048],
|
||||
"different seeds must descramble differently"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+852
-37
@@ -1,65 +1,270 @@
|
||||
//! CSS (Content Scramble System) — DVD disc encryption.
|
||||
//!
|
||||
//! CSS uses a weak 40-bit LFSR stream cipher (broken since 1999).
|
||||
//! No keys needed — the title key is cracked from encrypted content
|
||||
//! using a known-plaintext attack on MPEG-2 PES headers.
|
||||
//!
|
||||
//! The title key is recovered keylessly: [`crack_key`] runs the Stevenson
|
||||
//! known-plaintext attack (see the [`stevenson`] module) on the scrambled
|
||||
//! data, needing no player keys, disc-key crack, or external key file.
|
||||
//! Sectors are then decrypted with [`descramble_sector`].
|
||||
//!
|
||||
//! Usage:
|
||||
//! ```rust,ignore
|
||||
//! let key = css::crack_key(reader, &extents)?;
|
||||
//! css::descramble_sector(&key, &mut sector);
|
||||
//! if let Some(state) = css::crack_key(reader, extents, batch) {
|
||||
//! css::descramble_sector(&state, &mut sector);
|
||||
//! }
|
||||
//! ```
|
||||
|
||||
pub mod auth;
|
||||
pub mod crack;
|
||||
pub mod lfsr;
|
||||
pub mod stevenson;
|
||||
pub(crate) mod tables;
|
||||
|
||||
use crate::disc::Extent;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
|
||||
/// Consecutive CSS-locked (`05/6F/03`) reads before the crack scan early-bails.
|
||||
/// The bus-auth read gate is global (all-or-nothing), so a run this long means
|
||||
/// it is shut and nothing here is crackable — bail instead of grinding the full
|
||||
/// 50_000-sector budget (which is what made rc5 appear to hang on a wedged USB
|
||||
/// bridge). The counter resets to 0 on any readable batch.
|
||||
const CSS_LOCKED_BAIL: u32 = 64;
|
||||
|
||||
/// CSS decryption state for a DVD title.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CssState {
|
||||
/// Cracked 5-byte title key
|
||||
/// 5-byte CSS title key (from SCSI auth or the crack fallback).
|
||||
pub title_key: [u8; 5],
|
||||
/// LBA half-open span `[start, end)` of the extent set this key was
|
||||
/// cracked from. CSS title keys are per-VTS: a key cracked from one
|
||||
/// VTS does NOT descramble a title living in a different VTS. The mux
|
||||
/// path checks whether the title being opened overlaps this span; if
|
||||
/// not, it re-cracks from that title's own extents. `None` for keys
|
||||
/// of unknown provenance (e.g. test fixtures) — treated as "applies
|
||||
/// everywhere" for backward compatibility.
|
||||
pub crack_span: Option<(u32, u32)>,
|
||||
}
|
||||
|
||||
/// Crack the CSS title key by reading encrypted sectors and applying
|
||||
/// a known-plaintext attack on MPEG-2 headers.
|
||||
/// Recover the CSS title key with no keys, by scanning scrambled sectors and
|
||||
/// running the Stevenson known-plaintext attack (see the [`stevenson`] module).
|
||||
///
|
||||
/// Crack the CSS title key by scanning scrambled sectors across extents.
|
||||
///
|
||||
/// The Stevenson attack needs a sector where a PES header starts at byte
|
||||
/// 0x80 (start of the encrypted region). This only happens when a new PES
|
||||
/// packet begins at exactly sector offset 128. We scan up to 50000
|
||||
/// scrambled sectors sequentially across all extents.
|
||||
pub fn crack_key(reader: &mut dyn SectorReader, extents: &[Extent]) -> Option<CssState> {
|
||||
let mut tried = 0u32;
|
||||
let max_tries = 50_000;
|
||||
/// The crib comes from `AttackPattern`: a scrambled sector's cleartext region
|
||||
/// (bytes 0x00..0x80) often ends in a short-period repeating run (stuffing /
|
||||
/// constant fill); the attack assumes that run continues across the 0x80
|
||||
/// boundary into the encrypted region, giving the known plaintext the 2^16
|
||||
/// LFSR recovery needs. We scan up to 50000 sectors across the
|
||||
/// extents and return the first sector that yields a key — no player keys, no
|
||||
/// disc-key crack. Works on a live drive (after bus-auth unlocks reads) and on
|
||||
/// disc images alike.
|
||||
pub fn crack_key(
|
||||
reader: &mut dyn SectorSource,
|
||||
extents: &[Extent],
|
||||
batch_sectors: u16,
|
||||
) -> Option<CssState> {
|
||||
crack_key_halt(reader, extents, batch_sectors, None)
|
||||
}
|
||||
|
||||
for ext in extents {
|
||||
let mut i = 0;
|
||||
while i < ext.sector_count && tried < max_tries {
|
||||
let mut buf = vec![0u8; 2048];
|
||||
if reader
|
||||
.read_sectors(ext.start_lba + i, 1, &mut buf, true)
|
||||
.is_ok()
|
||||
&& is_scrambled(&buf)
|
||||
{
|
||||
if let Some(key) = crack::crack_title_key(&buf) {
|
||||
return Some(CssState { title_key: key });
|
||||
}
|
||||
tried += 1;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
if tried >= max_tries {
|
||||
break;
|
||||
/// Outcome of a CSS crack scan that distinguishes the THREE cases the bare
|
||||
/// `Option<CssState>` conflated (and which caused a silent-failure bug:
|
||||
/// scrambled-but-uncracked content was treated as "unencrypted" and muxed as
|
||||
/// plaintext garbage at exit 0):
|
||||
///
|
||||
/// - [`CrackOutcome::Cracked`] — a scrambled sector yielded a title key.
|
||||
/// - [`CrackOutcome::Unencrypted`] — NO scrambled sector was seen across the
|
||||
/// scanned extents (`is_scrambled` never true): the content is genuinely
|
||||
/// plaintext, so proceeding without a key is correct.
|
||||
/// - [`CrackOutcome::ScrambledUncracked`] — scrambled sectors WERE seen but no
|
||||
/// key could be recovered (the Stevenson attack found no crackable crib, or
|
||||
/// the scrambled region was unreadable). The content is encrypted; muxing it
|
||||
/// as plaintext would emit garbage, so callers MUST surface a hard error
|
||||
/// ([`crate::error::Error::CssKeyMissing`]) instead of falling through to
|
||||
/// "unencrypted".
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum CrackOutcome {
|
||||
Cracked(CssState),
|
||||
Unencrypted,
|
||||
ScrambledUncracked,
|
||||
}
|
||||
|
||||
impl CrackOutcome {
|
||||
/// The cracked `CssState`, if any. `None` for `Unencrypted` /
|
||||
/// `ScrambledUncracked`. Lets the `Option`-returning wrappers stay thin.
|
||||
pub fn into_state(self) -> Option<CssState> {
|
||||
match self {
|
||||
CrackOutcome::Cracked(s) => Some(s),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
/// True when scrambled sectors were seen but no key was recovered — the
|
||||
/// case callers must surface as a hard error instead of "unencrypted".
|
||||
pub fn is_scrambled_uncracked(&self) -> bool {
|
||||
matches!(self, CrackOutcome::ScrambledUncracked)
|
||||
}
|
||||
}
|
||||
|
||||
/// [`crack_key`] returning the full [`CrackOutcome`] (Cracked / Unencrypted /
|
||||
/// ScrambledUncracked) so callers can distinguish "genuinely unencrypted" from
|
||||
/// "encrypted but uncrackable" — the latter must become a hard error, never a
|
||||
/// silent fall-through to plaintext.
|
||||
pub fn crack_key_outcome(
|
||||
reader: &mut dyn SectorSource,
|
||||
extents: &[Extent],
|
||||
batch_sectors: u16,
|
||||
halt: Option<&crate::halt::Halt>,
|
||||
) -> CrackOutcome {
|
||||
crack_key_scan(reader, extents, batch_sectors, halt, true)
|
||||
}
|
||||
|
||||
/// [`crack_key`] with an optional cooperative-cancellation token.
|
||||
///
|
||||
/// "No silent hangs": the crack scans up to 50_000 sectors, which on a live
|
||||
/// drive hitting bad sectors can take a long time. This variant polls `halt`
|
||||
/// once per batch (the same cadence sweep/patch use) so an operator Stop or a
|
||||
/// scan-level watchdog can interrupt the scan, and emits a
|
||||
/// `freemkv::heartbeat` beat ("css_crack") each batch so a stuck scan is
|
||||
/// visible in the log.
|
||||
pub fn crack_key_halt(
|
||||
reader: &mut dyn SectorSource,
|
||||
extents: &[Extent],
|
||||
batch_sectors: u16,
|
||||
halt: Option<&crate::halt::Halt>,
|
||||
) -> Option<CssState> {
|
||||
crack_key_scan(reader, extents, batch_sectors, halt, false).into_state()
|
||||
}
|
||||
|
||||
/// The crack scan, returning the full [`CrackOutcome`]. Tracks a
|
||||
/// `saw_scrambled` flag so a scrambled-but-uncracked disc is distinguished
|
||||
/// from a genuinely-unencrypted one (the [`crack_key`] / [`crack_key_halt`]
|
||||
/// `Option` wrappers collapse both to `None`).
|
||||
fn crack_key_scan(
|
||||
reader: &mut dyn SectorSource,
|
||||
extents: &[Extent],
|
||||
batch_sectors: u16,
|
||||
halt: Option<&crate::halt::Halt>,
|
||||
// True only on the INITIAL scan: a fully CSS-locked (`05/6F/03`) result is a
|
||||
// hard `ScrambledUncracked`. False on the per-VTS re-crack so a lapsed-AGID
|
||||
// locked read returns None instead of killing a genuinely crackable title.
|
||||
fail_on_locked: bool,
|
||||
) -> CrackOutcome {
|
||||
// Batch the reads: a live optical drive at 1 sector/read is glacial, and the
|
||||
// crack only needs to FIND one scrambled sector whose 0x80 plaintext matches
|
||||
// a known PES header. `batch_sectors` MUST be sized to the source — a drive
|
||||
// rejects a READ(10) larger than its per-command max (DVD = 16) and
|
||||
// `Drive::read` does not chunk, so an over-large batch fails every read and
|
||||
// scans nothing. Callers pass `detect_max_batch_sectors(device_path)` for a
|
||||
// live drive, a file-safe value for an image, or 1 to force per-sector.
|
||||
let batch = (batch_sectors.max(1)) as u32;
|
||||
// Record the LBA span the key is being cracked from so the per-title mux
|
||||
// path can tell whether a later title lives in the same VTS (overlaps the
|
||||
// span → key applies) or a different one (→ re-crack). Half-open [min,max).
|
||||
let crack_span = extents
|
||||
.iter()
|
||||
.filter(|e| e.sector_count > 0)
|
||||
.map(|e| (e.start_lba, e.start_lba.saturating_add(e.sector_count)))
|
||||
.reduce(|(amin, amax), (bmin, bmax)| (amin.min(bmin), amax.max(bmax)));
|
||||
let mut tried = 0u32;
|
||||
let max_tries = 50_000u32;
|
||||
let mut buf = vec![0u8; batch as usize * 2048];
|
||||
let mut hb = crate::progress::Heartbeat::new("css_crack");
|
||||
// Track whether ANY scrambled sector was observed. If we exhaust the scan
|
||||
// budget having seen scrambled data but never recovered a key, the content
|
||||
// is encrypted-but-uncrackable — a HARD failure the caller must surface,
|
||||
// NOT silently treat as unencrypted (which would mux scrambled MPEG as
|
||||
// plaintext → garbage at exit 0). See `CrackOutcome::ScrambledUncracked`.
|
||||
let mut saw_scrambled = false;
|
||||
// A read rejected with sense `05/6F/03` ("scrambled sector without
|
||||
// authentication") is positive proof of CSS encryption — never collapse it
|
||||
// to "unencrypted". A run of consecutive locked reads means the bus-auth
|
||||
// gate is shut (it is global, so reads are all-or-nothing), so the scan
|
||||
// early-bails. `consecutive_locked` resets on any readable batch, so a
|
||||
// crackable title (gate open) never trips it.
|
||||
let mut saw_locked = false;
|
||||
let mut consecutive_locked = 0u32;
|
||||
|
||||
'outer: for (extent_idx, ext) in extents.iter().enumerate() {
|
||||
let mut i = 0u32;
|
||||
while i < ext.sector_count && tried < max_tries {
|
||||
// Cooperative cancellation — poll once per batch, the same cadence
|
||||
// sweep/patch use, so a Stop / watchdog can interrupt the scan.
|
||||
if let Some(h) = halt {
|
||||
if h.is_cancelled() {
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
// Liveness beacon: a long scan over a damaged disc stays visible.
|
||||
// The heartbeat is time-throttled; only when it actually beats do
|
||||
// we emit the crack-specific context (tried/lba/extent_idx).
|
||||
if hb.tick(tried as u64, max_tries as u64) {
|
||||
tracing::debug!(
|
||||
target: "freemkv::heartbeat",
|
||||
phase = "css_crack",
|
||||
tried,
|
||||
lba = ext.start_lba + i,
|
||||
extent_idx,
|
||||
"scanning"
|
||||
);
|
||||
}
|
||||
let n = (ext.sector_count - i).min(batch);
|
||||
let want = n as usize * 2048;
|
||||
match reader.read_sectors(ext.start_lba + i, n as u16, &mut buf[..want], true) {
|
||||
Ok(_) => {
|
||||
// A readable batch: the gate is open — reset the locked run.
|
||||
consecutive_locked = 0;
|
||||
for s in 0..n as usize {
|
||||
tried += 1;
|
||||
let sect = &buf[s * 2048..(s + 1) * 2048];
|
||||
// Use the HARDENED pack-gated check (Fix 3): a clear stub
|
||||
// sector with stray bits at 0x14 must NOT count as
|
||||
// scramble evidence, or a genuinely-unencrypted title
|
||||
// would falsely report ScrambledUncracked (a false E7023).
|
||||
if is_scrambled_pack(sect) {
|
||||
saw_scrambled = true;
|
||||
if let Some(key) = stevenson::crack_title_key(sect) {
|
||||
return CrackOutcome::Cracked(CssState {
|
||||
title_key: key,
|
||||
crack_span,
|
||||
});
|
||||
}
|
||||
}
|
||||
if tried >= max_tries {
|
||||
break 'outer;
|
||||
}
|
||||
}
|
||||
}
|
||||
// A failed batch still counts toward the budget so a damaged
|
||||
// region can't loop forever. A CSS-locked failure (`05/6F/03`)
|
||||
// proves encryption and, in a long enough run, means the read
|
||||
// gate is shut — track it and early-bail rather than grind.
|
||||
Err(e) => {
|
||||
tried += n;
|
||||
if e.scsi_sense().is_some_and(|s| s.is_css_locked()) {
|
||||
saw_locked = true;
|
||||
consecutive_locked += 1;
|
||||
if consecutive_locked >= CSS_LOCKED_BAIL {
|
||||
break 'outer;
|
||||
}
|
||||
} else {
|
||||
consecutive_locked = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
i += n;
|
||||
}
|
||||
}
|
||||
|
||||
// Budget exhausted / extents walked / early-bailed with no key recovered.
|
||||
// The disc is ENCRYPTED-but-uncracked (a hard failure on the initial scan)
|
||||
// when EITHER a scrambled sector was actually seen, OR — on the initial scan
|
||||
// only (`fail_on_locked`) — every read was CSS-locked (`05/6F/03`), itself
|
||||
// proof of scrambling. A re-crack (`fail_on_locked` false) stays soft: a
|
||||
// lapsed-AGID locked read yields None, not a hard fail, so a crackable title
|
||||
// in another VTS isn't killed. Only a scan that saw neither a scrambled
|
||||
// sector nor a CSS-lock is genuinely unencrypted.
|
||||
if saw_scrambled || (saw_locked && fail_on_locked) {
|
||||
CrackOutcome::ScrambledUncracked
|
||||
} else {
|
||||
CrackOutcome::Unencrypted
|
||||
}
|
||||
}
|
||||
|
||||
/// Descramble a single CSS-encrypted sector in place.
|
||||
@@ -68,6 +273,616 @@ pub fn descramble_sector(state: &CssState, sector: &mut [u8]) {
|
||||
}
|
||||
|
||||
/// Check if a sector has the CSS scramble flag set.
|
||||
///
|
||||
/// This is the RAW flag test — bits 4-5 of the sub-header byte 0x14 — used by
|
||||
/// the descramble loop (`decrypt::decrypt_sectors`), which has already committed
|
||||
/// to descrambling a known title's VOB data and only needs to skip the clear
|
||||
/// NAV packs interleaved in it. For the CRACK SCAN's "did this disc actually
|
||||
/// contain scrambled content?" decision (which must not false-positive on a
|
||||
/// clear stub), use [`is_scrambled_pack`] instead.
|
||||
pub fn is_scrambled(sector: &[u8]) -> bool {
|
||||
sector.len() >= 2048 && (sector[0x14] >> 4) & 0x03 != 0
|
||||
}
|
||||
|
||||
/// The 4-byte MPEG-2 Program Stream pack-start code (`00 00 01 BA`) every DVD
|
||||
/// video sector opens with. CSS leaves the clear header (`0x00..0x80`)
|
||||
/// untouched, so this signature survives scrambling.
|
||||
pub(crate) const PACK_START: [u8; 4] = [0x00, 0x00, 0x01, 0xBA];
|
||||
|
||||
/// Check if a sector is a CSS-scrambled DVD **video pack** — the HARDENED test
|
||||
/// the crack scan uses to set its `saw_scrambled` evidence flag (Fix 3).
|
||||
///
|
||||
/// [`is_scrambled`] keys solely on bits 4-5 of byte 0x14. That single byte is
|
||||
/// only meaningful inside a real DVD sector — an MPEG-2 Program Stream pack,
|
||||
/// which ALWAYS begins with the 32-bit pack-start code `00 00 01 BA` at offset
|
||||
/// 0x00. A tiny clear / nav-only stub (a 0.5 s menu loop, an FBI-warning title)
|
||||
/// can carry arbitrary bytes that happen to set bits 4-5 of byte 0x14; trusting
|
||||
/// byte 0x14 alone there would flip the scan's `saw_scrambled` gate and make a
|
||||
/// genuinely-UNENCRYPTED title report `ScrambledUncracked` — a false E7023.
|
||||
///
|
||||
/// Requiring the pack-start signature FIRST means only a sector that is
|
||||
/// structurally a DVD video pack can be counted as scramble evidence. This does
|
||||
/// NOT weaken the genuine "encrypted but uncrackable" hard-fail: a real
|
||||
/// scrambled feature is made of valid PS packs, so its scrambled sectors still
|
||||
/// pass this check and still drive `ScrambledUncracked` when no key cracks. (The
|
||||
/// descramble loop keeps the looser [`is_scrambled`]: by the time it runs we
|
||||
/// already know the title is CSS, and it only needs to skip interleaved clear
|
||||
/// NAV packs — a wrongly-skipped or wrongly-included sector there is recoverable
|
||||
/// per-sector, whereas a false scramble verdict in the scan poisons the whole
|
||||
/// title's outcome.)
|
||||
pub fn is_scrambled_pack(sector: &[u8]) -> bool {
|
||||
sector.len() >= 2048 && sector[0x00..0x04] == PACK_START && (sector[0x14] >> 4) & 0x03 != 0
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
// ── is_scrambled ───────────────────────────────────────────────────────
|
||||
|
||||
/// is_scrambled returns false for any buffer shorter than one sector,
|
||||
/// WITHOUT indexing byte 0x14 (which would panic on a tiny buffer). The
|
||||
/// length guard is short-circuited before the flag read.
|
||||
///
|
||||
/// Grounding: `sector.len() >= 2048 && (sector[0x14] >> 4) & 0x03 != 0` —
|
||||
/// `&&` short-circuits so a 20-byte buffer never reads index 0x14.
|
||||
/// Mutation: swap the operands so the flag is read first
|
||||
/// (`(sector[0x14]...) && sector.len() >= 2048`) -> panics indexing a
|
||||
/// 20-byte slice; this test catches it.
|
||||
#[test]
|
||||
fn is_scrambled_short_buffer_is_false_no_panic() {
|
||||
assert!(!is_scrambled(&[]));
|
||||
assert!(!is_scrambled(&[0u8; 20])); // shorter than 0x14+1 even
|
||||
assert!(!is_scrambled(&[0xFFu8; 2047])); // one byte short of a sector
|
||||
}
|
||||
|
||||
/// is_scrambled keys on bits 4-5 of byte 0x14 (the CSS scramble field).
|
||||
/// A full sector flagged 0x10/0x20/0x30 is scrambled; 0x00 and the
|
||||
/// high-bit-only values 0x40/0x80 are clear.
|
||||
///
|
||||
/// Grounding: `(sector[0x14] >> 4) & 0x03`.
|
||||
/// Mutation: widen mask to `& 0x0F` -> 0x40 reports scrambled, the 0x40
|
||||
/// assert fails.
|
||||
#[test]
|
||||
fn is_scrambled_uses_bits_4_5_only() {
|
||||
let mut s = vec![0u8; 2048];
|
||||
for (flag, expected) in [
|
||||
(0x00u8, false),
|
||||
(0x10, true),
|
||||
(0x20, true),
|
||||
(0x30, true),
|
||||
(0x40, false),
|
||||
(0x80, false),
|
||||
(0xC0, false),
|
||||
(0xFF, true), // bits 4-5 set within 0xFF
|
||||
] {
|
||||
s[0x14] = flag;
|
||||
assert_eq!(
|
||||
is_scrambled(&s),
|
||||
expected,
|
||||
"flag byte {flag:#04x} scramble detection"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// is_scrambled accepts exactly 2048 bytes as the minimum (boundary at the
|
||||
/// inclusive value 2048).
|
||||
///
|
||||
/// Grounding: `sector.len() >= 2048`.
|
||||
/// Mutation: change `>= 2048` to `> 2048` -> an exact 2048-byte scrambled
|
||||
/// sector reports false; this fails.
|
||||
#[test]
|
||||
fn is_scrambled_exact_sector_length_accepted() {
|
||||
let mut s = vec![0u8; 2048];
|
||||
s[0x14] = 0x30;
|
||||
assert!(is_scrambled(&s), "exactly 2048 bytes must be eligible");
|
||||
}
|
||||
|
||||
/// Fix 3 hardening: `is_scrambled_pack` (the crack-scan evidence gate)
|
||||
/// requires BOTH the MPEG-PS pack-start code at 0x00 AND the 0x14 scramble
|
||||
/// bits. A clear / nav-only stub whose bytes happen to set bits 4-5 of 0x14
|
||||
/// but lacks the pack-start is NOT counted as scramble evidence — without
|
||||
/// this the scan flips `saw_scrambled` and a genuinely unencrypted title
|
||||
/// reports `ScrambledUncracked` (the false E7023). The looser `is_scrambled`
|
||||
/// (descramble gate) still reads the same sector as flagged.
|
||||
///
|
||||
/// Grounding: `sector[0x00..0x04] == 00 00 01 BA && (sector[0x14] >> 4)...`.
|
||||
/// Mutation: drop the pack-start clause -> the 0x14-only sector counts as a
|
||||
/// scrambled pack; the first assert fails.
|
||||
#[test]
|
||||
fn is_scrambled_pack_requires_pack_start_signature() {
|
||||
let mut s = vec![0u8; 2048];
|
||||
s[0x14] = 0x30; // scramble bits set, but no pack-start at 0x00
|
||||
assert!(
|
||||
!is_scrambled_pack(&s),
|
||||
"0x14 bits without the MPEG-PS pack-start must NOT count as a scrambled pack"
|
||||
);
|
||||
// The looser descramble-gate check still sees the raw flag.
|
||||
assert!(is_scrambled(&s), "is_scrambled keys on the 0x14 flag alone");
|
||||
// A near-miss pack-start (wrong final byte) is still rejected.
|
||||
s[0x00..0x04].copy_from_slice(&[0x00, 0x00, 0x01, 0xBB]);
|
||||
assert!(
|
||||
!is_scrambled_pack(&s),
|
||||
"a wrong pack-start byte must not qualify"
|
||||
);
|
||||
// The real signature flips it to a scrambled pack.
|
||||
s[0x00..0x04].copy_from_slice(&PACK_START);
|
||||
assert!(
|
||||
is_scrambled_pack(&s),
|
||||
"valid pack-start + 0x14 bits → scrambled pack"
|
||||
);
|
||||
}
|
||||
|
||||
// ── crack_key scanning over a mock SectorSource ────────────────────────
|
||||
|
||||
/// Records every (lba, count) read; returns a caller-supplied flag byte at
|
||||
/// 0x14 so we can drive scrambled/clear sectors, or an injected error.
|
||||
struct MockSource {
|
||||
reads: std::cell::RefCell<Vec<u32>>,
|
||||
flag_byte: u8,
|
||||
fail_all: bool,
|
||||
/// Every read fails with CSS-locked sense `05/6F/03` (drive refusing
|
||||
/// scrambled reads because the bus-auth gate isn't open).
|
||||
lock_all: bool,
|
||||
/// When set, the sector at `crackable.0` is served as a full
|
||||
/// Stevenson-crackable scrambled sector (`crackable.1`, 2048 bytes)
|
||||
/// instead of the uniform `flag_byte` fill. Lets the scan actually
|
||||
/// reach `CrackOutcome::Cracked` from a synthetic ISO.
|
||||
crackable: Option<(u32, Vec<u8>)>,
|
||||
}
|
||||
|
||||
impl MockSource {
|
||||
fn new(flag_byte: u8) -> Self {
|
||||
Self {
|
||||
reads: std::cell::RefCell::new(Vec::new()),
|
||||
flag_byte,
|
||||
fail_all: false,
|
||||
lock_all: false,
|
||||
crackable: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a Stevenson-crackable scrambled sector for `(title_key, seed)`:
|
||||
/// the cleartext header (0x59..0x80) carries a periodic run that continues
|
||||
/// across the 0x80 boundary into the encrypted region — the crib
|
||||
/// `stevenson::crack_title_key` recovers a key from. Mirrors the
|
||||
/// `synth_periodic_sector` fixture in the stevenson tests but built here
|
||||
/// from the crate-internal `scramble_sector`.
|
||||
fn crackable_sector(title_key: &[u8; 5], seed: &[u8; 5], period: usize) -> Vec<u8> {
|
||||
const RUN_START: usize = 0x59;
|
||||
const SEED_OFFSET: usize = 0x54;
|
||||
let mut plaintext = vec![0u8; 2048];
|
||||
plaintext[0x00..0x04].copy_from_slice(&PACK_START); // valid DVD pack header
|
||||
plaintext[0x14] = 0x10; // scramble flag
|
||||
let pat: Vec<u8> = (0..period)
|
||||
.map(|k| (0xA0u8.wrapping_add(k as u8)) ^ 0x5A)
|
||||
.collect();
|
||||
for (i, b) in plaintext.iter_mut().enumerate().skip(RUN_START) {
|
||||
*b = pat[i % period];
|
||||
}
|
||||
plaintext[SEED_OFFSET..SEED_OFFSET + 5].copy_from_slice(seed);
|
||||
lfsr::scramble_sector(title_key, &mut plaintext);
|
||||
plaintext
|
||||
}
|
||||
|
||||
impl SectorSource for MockSource {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
self.reads.borrow_mut().push(lba);
|
||||
if self.lock_all {
|
||||
return Err(Error::DiscRead {
|
||||
sector: lba as u64,
|
||||
status: Some(2),
|
||||
sense: Some(crate::scsi::ScsiSense {
|
||||
sense_key: 0x05,
|
||||
asc: 0x6F,
|
||||
ascq: 0x03,
|
||||
}),
|
||||
});
|
||||
}
|
||||
if self.fail_all {
|
||||
return Err(Error::DecryptFailed);
|
||||
}
|
||||
let n = count as usize * 2048;
|
||||
let end = n.min(buf.len());
|
||||
for b in buf[..end].iter_mut() {
|
||||
*b = 0;
|
||||
}
|
||||
// Fill each sector in the batch with the uniform flag byte, EXCEPT a
|
||||
// designated crackable LBA which gets the full synthetic sector.
|
||||
for s in 0..count as u32 {
|
||||
let sect_lba = lba + s;
|
||||
let base = s as usize * 2048;
|
||||
if base + 2048 > end {
|
||||
break;
|
||||
}
|
||||
match &self.crackable {
|
||||
Some((clba, sector)) if *clba == sect_lba => {
|
||||
buf[base..base + 2048].copy_from_slice(sector);
|
||||
}
|
||||
_ => {
|
||||
// Real DVD video sectors always open with the MPEG-PS
|
||||
// pack-start code; `is_scrambled` (Fix 3) requires it
|
||||
// before trusting the 0x14 scramble bits, so the fixture
|
||||
// must include it for a `flag_byte` of 0x30 to register
|
||||
// as scrambled.
|
||||
buf[base..base + 4].copy_from_slice(&PACK_START);
|
||||
buf[base + 0x14] = self.flag_byte;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
/// crack_key caps total scanned sectors at 50_000 even when extents are
|
||||
/// far larger, and counts EVERY scanned sector (clear ones included)
|
||||
/// toward the budget. With one 200_000-sector extent of clear sectors, it
|
||||
/// must read exactly 50_000 sectors and return None — never run away.
|
||||
///
|
||||
/// Grounding: `let max_tries = 50_000; ... tried += 1` before the read,
|
||||
/// loop guard `tried < max_tries`.
|
||||
/// Mutation: change `50_000` to `500_000` -> read count exceeds 50_000;
|
||||
/// the exact-count assert fails. Removing the `tried += 1` increment ->
|
||||
/// would read all 200_000; also fails.
|
||||
#[test]
|
||||
fn crack_key_caps_total_tries_at_50000() {
|
||||
let mut src = MockSource::new(0x00); // clear sectors, never a hit
|
||||
let extents = [Extent {
|
||||
start_lba: 0,
|
||||
sector_count: 200_000,
|
||||
}];
|
||||
let res = crack_key(&mut src, &extents, 1);
|
||||
assert!(res.is_none(), "clear sectors yield no key");
|
||||
assert_eq!(
|
||||
src.reads.borrow().len(),
|
||||
50_000,
|
||||
"scan must stop at the 50_000-sector budget"
|
||||
);
|
||||
}
|
||||
|
||||
// ── CrackOutcome: scrambled-but-uncracked vs genuinely unencrypted (Fix 6) ─
|
||||
|
||||
/// A scan over CLEAR sectors (scramble flag never set) returns
|
||||
/// `Unencrypted` — the content is genuinely plaintext, so proceeding
|
||||
/// without a key is correct.
|
||||
#[test]
|
||||
fn crack_outcome_clear_sectors_is_unencrypted() {
|
||||
let mut src = MockSource::new(0x00); // never scrambled
|
||||
let extents = [Extent {
|
||||
start_lba: 0,
|
||||
sector_count: 100,
|
||||
}];
|
||||
let outcome = crack_key_outcome(&mut src, &extents, 1, None);
|
||||
assert!(
|
||||
matches!(outcome, CrackOutcome::Unencrypted),
|
||||
"no scrambled sector seen → Unencrypted, got {outcome:?}"
|
||||
);
|
||||
// The Option wrapper collapses Unencrypted → None.
|
||||
assert!(crack_key(&mut MockSource::new(0x00), &extents, 1).is_none());
|
||||
}
|
||||
|
||||
/// THE Fix 6 regression: a scan that SEES scrambled sectors (flag set) but
|
||||
/// recovers no key (the mock's zeroed data has no Stevenson crib) must
|
||||
/// return `ScrambledUncracked` — a HARD failure — NOT `Unencrypted`. The
|
||||
/// old code conflated this with "unencrypted" and muxed scrambled MPEG as
|
||||
/// plaintext (garbage at exit 0).
|
||||
#[test]
|
||||
fn crack_outcome_scrambled_uncracked_is_hard_failure() {
|
||||
let mut src = MockSource::new(0x30); // scrambled flag set, no crackable crib
|
||||
let extents = [Extent {
|
||||
start_lba: 0,
|
||||
sector_count: 100,
|
||||
}];
|
||||
let outcome = crack_key_outcome(&mut src, &extents, 1, None);
|
||||
assert!(
|
||||
outcome.is_scrambled_uncracked(),
|
||||
"scrambled sectors seen but no key → ScrambledUncracked, got {outcome:?}"
|
||||
);
|
||||
// The legacy Option wrapper still collapses this to None (the callers
|
||||
// that need the distinction now use crack_key_outcome instead).
|
||||
assert!(crack_key(&mut MockSource::new(0x30), &extents, 1).is_none());
|
||||
}
|
||||
|
||||
/// Even when every read FAILS, a scan that never managed to observe a
|
||||
/// scrambled sector reports `Unencrypted` (we cannot prove encryption from
|
||||
/// unreadable data alone — the AACS/keydb paths and the disc-level
|
||||
/// `css_error` plumbing cover genuinely unreadable encrypted discs).
|
||||
#[test]
|
||||
fn crack_outcome_all_reads_fail_is_unencrypted() {
|
||||
let mut src = MockSource::new(0x30);
|
||||
src.fail_all = true; // no sector is ever inspected
|
||||
let extents = [Extent {
|
||||
start_lba: 0,
|
||||
sector_count: 10,
|
||||
}];
|
||||
let outcome = crack_key_outcome(&mut src, &extents, 1, None);
|
||||
assert!(
|
||||
matches!(outcome, CrackOutcome::Unencrypted),
|
||||
"no readable scrambled sector → Unencrypted, got {outcome:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Fix C (rc.5.1): on the INITIAL scan, a drive that refuses every read with
|
||||
/// CSS-locked sense (`05/6F/03`) is encrypted-but-locked →
|
||||
/// `ScrambledUncracked` (a hard failure), NOT `Unencrypted`. This is the
|
||||
/// rc4.3 bug: every VOB read came back `6F/03`, so the scan saw no scrambled
|
||||
/// sector and wrongly declared the disc unencrypted → 19 KB garbage.
|
||||
#[test]
|
||||
fn crack_outcome_css_locked_initial_is_scrambled_uncracked() {
|
||||
let mut src = MockSource::new(0x30);
|
||||
src.lock_all = true; // every read → 05/6F/03
|
||||
let extents = [Extent {
|
||||
start_lba: 0,
|
||||
sector_count: 100,
|
||||
}];
|
||||
let outcome = crack_key_outcome(&mut src, &extents, 1, None);
|
||||
assert!(
|
||||
outcome.is_scrambled_uncracked(),
|
||||
"every read 6F/03 on the initial scan → ScrambledUncracked, got {outcome:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// MISSING #1 guard: the re-crack path (the `Option`-returning `crack_key`,
|
||||
/// `fail_on_locked == false`) must NOT hard-fail on a CSS-locked read — it
|
||||
/// returns `None`. A lapsed-AGID re-crack of another VTS stays soft so a
|
||||
/// genuinely crackable title isn't killed by a transient locked read.
|
||||
#[test]
|
||||
fn crack_key_recrack_locked_is_none_not_hard_fail() {
|
||||
let mut src = MockSource::new(0x30);
|
||||
src.lock_all = true;
|
||||
let extents = [Extent {
|
||||
start_lba: 0,
|
||||
sector_count: 100,
|
||||
}];
|
||||
assert!(crack_key(&mut src, &extents, 1).is_none());
|
||||
}
|
||||
|
||||
/// Fix F: a fully CSS-locked scan early-bails near `CSS_LOCKED_BAIL`
|
||||
/// consecutive locked reads instead of grinding the whole 50_000-sector
|
||||
/// budget (the rc5 "stuck Scanning…" hang on a wedged bridge).
|
||||
#[test]
|
||||
fn crack_css_locked_scan_early_bails() {
|
||||
let mut src = MockSource::new(0x30);
|
||||
src.lock_all = true;
|
||||
let extents = [Extent {
|
||||
start_lba: 0,
|
||||
sector_count: 10_000,
|
||||
}];
|
||||
let _ = crack_key_outcome(&mut src, &extents, 1, None);
|
||||
let n = src.reads.borrow().len();
|
||||
assert!(
|
||||
n <= (CSS_LOCKED_BAIL as usize) + 1,
|
||||
"locked scan early-bails near {CSS_LOCKED_BAIL}, not 10000; read {n}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The budget spans ALL extents, not per-extent: two extents summing past
|
||||
/// the cap must still stop at 50_000 total reads.
|
||||
///
|
||||
/// Grounding: `tried` is declared outside the `for ext in extents` loop;
|
||||
/// `if tried >= max_tries { break }` after each extent.
|
||||
/// Mutation: move `let mut tried = 0` inside the extent loop -> each extent
|
||||
/// gets its own 50_000 budget; total reads would be 80_000, this fails.
|
||||
#[test]
|
||||
fn crack_key_budget_is_shared_across_extents() {
|
||||
let mut src = MockSource::new(0x00);
|
||||
let extents = [
|
||||
Extent {
|
||||
start_lba: 0,
|
||||
sector_count: 40_000,
|
||||
},
|
||||
Extent {
|
||||
start_lba: 100_000,
|
||||
sector_count: 40_000,
|
||||
},
|
||||
];
|
||||
let res = crack_key(&mut src, &extents, 1);
|
||||
assert!(res.is_none());
|
||||
assert_eq!(
|
||||
src.reads.borrow().len(),
|
||||
50_000,
|
||||
"the 50_000 budget is shared across all extents"
|
||||
);
|
||||
}
|
||||
|
||||
/// crack_key scans sequentially from each extent's start_lba. The first
|
||||
/// reads must be at the extent's start_lba, start_lba+1, ... pinning the
|
||||
/// LBA arithmetic `ext.start_lba + i`.
|
||||
///
|
||||
/// Grounding: `reader.read_sectors(ext.start_lba + i, 1, ...)`.
|
||||
/// Mutation: change `ext.start_lba + i` to just `i` -> the recorded LBAs
|
||||
/// would start at 0, not 5000; this fails.
|
||||
#[test]
|
||||
fn crack_key_scans_from_extent_start_lba() {
|
||||
let mut src = MockSource::new(0x00);
|
||||
let extents = [Extent {
|
||||
start_lba: 5_000,
|
||||
sector_count: 4,
|
||||
}];
|
||||
let _ = crack_key(&mut src, &extents, 1);
|
||||
let reads = src.reads.borrow();
|
||||
assert_eq!(
|
||||
&reads[..],
|
||||
&[5_000, 5_001, 5_002, 5_003],
|
||||
"sequential scan from start_lba"
|
||||
);
|
||||
}
|
||||
|
||||
/// A read error on a sector does NOT abort the scan: crack_key keeps
|
||||
/// scanning subsequent sectors (the error sector still counts toward the
|
||||
/// budget). With a small failing extent, every sector is attempted and the
|
||||
/// function returns None.
|
||||
///
|
||||
/// Grounding: `if reader.read_sectors(...).is_ok() && is_scrambled(...)` —
|
||||
/// an Err simply falls through to `i += 1`.
|
||||
/// Mutation: change the read-error handling to `reader.read_sectors(...)?`
|
||||
/// (propagate) -> crack_key would stop after the first error and read only
|
||||
/// 1 sector; this asserts all 10 were attempted.
|
||||
#[test]
|
||||
fn crack_key_continues_past_read_errors() {
|
||||
let mut src = MockSource::new(0x30);
|
||||
src.fail_all = true;
|
||||
let extents = [Extent {
|
||||
start_lba: 0,
|
||||
sector_count: 10,
|
||||
}];
|
||||
let res = crack_key(&mut src, &extents, 1);
|
||||
assert!(res.is_none());
|
||||
assert_eq!(
|
||||
src.reads.borrow().len(),
|
||||
10,
|
||||
"read errors must not abort the scan"
|
||||
);
|
||||
}
|
||||
|
||||
/// Empty extents (no sectors) -> crack_key reads nothing and returns None.
|
||||
/// A zero-sector extent must not read its start_lba.
|
||||
///
|
||||
/// Grounding: `while i < ext.sector_count` with sector_count == 0 never
|
||||
/// enters.
|
||||
/// Mutation: change `i < ext.sector_count` to `i <= ext.sector_count` ->
|
||||
/// one spurious read at start_lba; this asserts zero reads.
|
||||
#[test]
|
||||
fn crack_key_empty_extent_reads_nothing() {
|
||||
let mut src = MockSource::new(0x30);
|
||||
let extents = [Extent {
|
||||
start_lba: 42,
|
||||
sector_count: 0,
|
||||
}];
|
||||
let res = crack_key(&mut src, &extents, 1);
|
||||
assert!(res.is_none());
|
||||
assert_eq!(
|
||||
src.reads.borrow().len(),
|
||||
0,
|
||||
"zero-sector extent reads nothing"
|
||||
);
|
||||
}
|
||||
|
||||
/// No extents at all -> immediate None, zero reads.
|
||||
///
|
||||
/// Grounding: `for ext in extents` over an empty slice is a no-op.
|
||||
/// Mutation: any change that reads before the loop would break this.
|
||||
#[test]
|
||||
fn crack_key_no_extents_is_none() {
|
||||
let mut src = MockSource::new(0x30);
|
||||
let res = crack_key(&mut src, &[], 1);
|
||||
assert!(res.is_none());
|
||||
assert_eq!(src.reads.borrow().len(), 0);
|
||||
}
|
||||
|
||||
// ── Scan-level Cracked branch + per-VTS re-crack success (audit §2 / §5 #8) ─
|
||||
|
||||
/// SCAN-LEVEL CRACKED (audit gap "MockSource never yields a crackable
|
||||
/// sector"): drive the full `crack_key_scan` over a synthetic ISO whose
|
||||
/// scan hits a Stevenson-crackable scrambled sector. The outcome must be
|
||||
/// `CrackOutcome::Cracked` with a key that round-trips the sector, AND the
|
||||
/// `crack_span` must be recorded as the half-open extent span (the per-VTS
|
||||
/// routing key the mux path needs). Previously only the leaf crack and the
|
||||
/// Uncracked/Unencrypted branches were tested — the Cracked branch and
|
||||
/// `crack_span` recording were never exercised end-to-end.
|
||||
#[test]
|
||||
fn crack_outcome_reaches_cracked_with_span() {
|
||||
let title_key = [0x42, 0x13, 0x37, 0xBE, 0xEF];
|
||||
let seed = [0x11, 0x22, 0x33, 0x44, 0x55];
|
||||
let crackable = crackable_sector(&title_key, &seed, 8);
|
||||
// The crackable sector sits a few sectors into the extent.
|
||||
let mut src = MockSource::new(0x00); // surrounding sectors: clear
|
||||
src.crackable = Some((1003, crackable.clone()));
|
||||
let extents = [Extent {
|
||||
start_lba: 1000,
|
||||
sector_count: 50,
|
||||
}];
|
||||
let outcome = crack_key_outcome(&mut src, &extents, 4, None);
|
||||
let state = match outcome {
|
||||
CrackOutcome::Cracked(s) => s,
|
||||
other => panic!("expected Cracked, got {other:?}"),
|
||||
};
|
||||
// The recovered key descrambles the crackable sector body.
|
||||
let mut test = crackable.clone();
|
||||
descramble_sector(&state, &mut test);
|
||||
let mut plain = crackable;
|
||||
lfsr::descramble_sector(&title_key, &mut plain);
|
||||
assert_eq!(
|
||||
&test[0x80..],
|
||||
&plain[0x80..],
|
||||
"recovered key must round-trip the scrambled sector body"
|
||||
);
|
||||
// crack_span = half-open [start, start+count) of the scanned extent.
|
||||
assert_eq!(
|
||||
state.crack_span,
|
||||
Some((1000, 1050)),
|
||||
"crack_span must record the extent LBA span for per-VTS routing"
|
||||
);
|
||||
}
|
||||
|
||||
/// CSS_ERROR WIRING (audit §2 / §5 #7): an all-locked synthetic ISO (every
|
||||
/// VOB read returns CSS-locked sense `05/6F/03` across MULTIPLE extents, as a
|
||||
/// real encrypted-but-unauthenticated disc image does) must produce the exact
|
||||
/// outcome the scan converts into `disc.css_error = Some(Error::CssKeyMissing)`
|
||||
/// — i.e. `CrackOutcome::ScrambledUncracked` / `is_scrambled_uncracked()`,
|
||||
/// NOT `Unencrypted`. disc/mod.rs's `crack_key_outcome → ScrambledUncracked`
|
||||
/// arm (where it stamps css_error) is driven by exactly this signal, so this
|
||||
/// pins the css-layer contract that arm depends on without touching the
|
||||
/// scan plumbing.
|
||||
#[test]
|
||||
fn all_locked_synthetic_iso_yields_css_key_missing_signal() {
|
||||
let mut src = MockSource::new(0x30);
|
||||
src.lock_all = true; // every read → 05/6F/03 across the whole "ISO"
|
||||
let extents = [
|
||||
Extent {
|
||||
start_lba: 0,
|
||||
sector_count: 30,
|
||||
},
|
||||
Extent {
|
||||
start_lba: 5_000,
|
||||
sector_count: 30,
|
||||
},
|
||||
];
|
||||
let outcome = crack_key_outcome(&mut src, &extents, 16, None);
|
||||
assert!(
|
||||
outcome.is_scrambled_uncracked(),
|
||||
"all-locked ISO → ScrambledUncracked (the css_error=CssKeyMissing \
|
||||
signal), got {outcome:?}"
|
||||
);
|
||||
// The legacy Option wrapper still collapses it to None — callers that
|
||||
// surface the hard error must use crack_key_outcome, which this proves.
|
||||
let mut src2 = MockSource::new(0x30);
|
||||
src2.lock_all = true;
|
||||
assert!(crack_key(&mut src2, &extents, 16).is_none());
|
||||
}
|
||||
|
||||
/// PER-VTS RE-CRACK SUCCESS (audit gap "success path missing"): the prior
|
||||
/// re-crack test only covered the locked→None path. Here a re-crack
|
||||
/// (`crack_key`, `fail_on_locked == false`) over a DIFFERENT VTS's extents
|
||||
/// finds that VTS's own crackable sector and returns a `CssState` whose
|
||||
/// `crack_span` matches the new extents — proving a key cracked for one VTS
|
||||
/// is genuinely re-derived (not reused) for another.
|
||||
#[test]
|
||||
fn recrack_succeeds_on_other_vts_extents() {
|
||||
let title_key = [0xFE, 0xDC, 0xBA, 0x98, 0x76];
|
||||
let seed = [0x00, 0xFF, 0x80, 0x7F, 0x01];
|
||||
let crackable = crackable_sector(&title_key, &seed, 5);
|
||||
let mut src = MockSource::new(0x00);
|
||||
// The second VTS lives at a disjoint LBA range; its crackable sector is
|
||||
// the first one in the extent.
|
||||
src.crackable = Some((9000, crackable));
|
||||
let other_vts = [Extent {
|
||||
start_lba: 9000,
|
||||
sector_count: 20,
|
||||
}];
|
||||
let state = crack_key(&mut src, &other_vts, 4).expect("re-crack must recover a key");
|
||||
assert_eq!(
|
||||
state.crack_span,
|
||||
Some((9000, 9020)),
|
||||
"re-crack span must reflect the OTHER VTS extents, not a reused span"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,593 @@
|
||||
//! CSS title-key recovery — Frank A. Stevenson's divide-and-conquer attack
|
||||
//! (1999), ported exactly from libdvdcss `RecoverTitleKey` + `AttackPattern`
|
||||
//! (css.c).
|
||||
//!
|
||||
//! Recovers the 5-byte CSS title key from a single scrambled DVD sector with
|
||||
//! no player keys and no disc-key crack, using only known plaintext.
|
||||
//!
|
||||
//! # The cipher this attacks
|
||||
//!
|
||||
//! The content descrambler ([`super::lfsr::descramble_sector`], = libdvdcss
|
||||
//! `dvdcss_unscramble`) seeds its two LFSRs **directly** from
|
||||
//! `key = title_key XOR sector_seed` (seed = `sector[0x54..0x59]`):
|
||||
//!
|
||||
//! ```text
|
||||
//! i_t1 = (key[0] ^ sec[0x54]) | 0x100; // LFSR1 low (9-bit)
|
||||
//! i_t2 = key[1] ^ sec[0x55]; // LFSR1 high
|
||||
//! i_t3 = (key[2]|key[3]<<8|key[4]<<16) ^ seed3; // LFSR0 (24-bit feedback)
|
||||
//! i_t3 = i_t3*2 + 8 - (i_t3 & 7);
|
||||
//! // per byte: *p = TAB1[*p] ^ (i_t5 & 0xff)
|
||||
//! ```
|
||||
//!
|
||||
//! There is NO `decrypt_key` mangling on the content path. So the recovery
|
||||
//! is a single inversion of `dvdcss_unscramble`, not the multi-stage
|
||||
//! working-key inversion the previous (non-CSS) implementation used.
|
||||
//!
|
||||
//! # The attack
|
||||
//!
|
||||
//! 1. **Known plaintext → keystream.** Because the descramble applies TAB1
|
||||
//! to the ciphertext, the per-byte keystream is
|
||||
//! `buf[i] = TAB1[cipher[i]] ^ plain[i]` (matching libdvdcss
|
||||
//! `RecoverTitleKey`'s `p_buffer`).
|
||||
//! 2. **Brute the 16-bit LFSR1 seed.** For each of 2^16 seeds, run LFSR1
|
||||
//! forward; for the first four steps deduce the LFSR0 output bytes from
|
||||
//! the keystream (carry-tracked), reconstructing `i_t3`. For the next six
|
||||
//! steps clock LFSR0 normally and check it reproduces the keystream — a
|
||||
//! wrong LFSR1 seed fails fast.
|
||||
//! 3. **Back-clock LFSR0.** Run four backward `i_t3` steps (each a 256-way
|
||||
//! search for the byte shifted in) to reach the initial state, then undo
|
||||
//! `i_t3 = i_t3*2 + 8 - (i_t3 & 7)` to recover key[2..5].
|
||||
//! 4. **XOR back the seed.** `key[0..5] ^= sector_seed[0..5]` (plain XOR —
|
||||
//! the descramble seeds directly, so there is no inversion).
|
||||
//!
|
||||
//! `AttackPattern` finds known plaintext for step 1: the longest periodic
|
||||
//! run in the cleartext `sec[0x00..0x80]`, assumed to continue into the
|
||||
//! encrypted region at 0x80.
|
||||
|
||||
use super::lfsr::descramble_sector;
|
||||
use super::tables::{TAB1, TAB2, TAB3, TAB4, TAB5};
|
||||
|
||||
use crate::consts::SECTOR_BYTES;
|
||||
const ENCRYPTED_START: usize = 0x80; // byte 128
|
||||
const SEED_OFFSET: usize = 0x54; // sector seed at bytes 0x54-0x58
|
||||
const FLAG_BYTE: usize = 0x14;
|
||||
|
||||
/// RecoverTitleKey: recover the title key from cipher + known plaintext.
|
||||
///
|
||||
/// Exact port of libdvdcss `RecoverTitleKey` (css.c). `crypted` is the
|
||||
/// ciphertext starting at sector byte 0x80; `decrypted` is the matching
|
||||
/// known plaintext; `seed` is `sector[0x54..0x59]`. On success returns the
|
||||
/// recovered 5-byte title key; `None` if no LFSR seed reproduces the
|
||||
/// keystream.
|
||||
///
|
||||
/// At least 10 bytes of `crypted`/`decrypted` are required (the cipher is
|
||||
/// iterated 10 times: 4 to reconstruct LFSR0, 6 to validate).
|
||||
fn recover_title_key_from_plain(
|
||||
crypted: &[u8],
|
||||
decrypted: &[u8],
|
||||
seed: &[u8; 5],
|
||||
) -> Option<[u8; 5]> {
|
||||
if crypted.len() < 10 || decrypted.len() < 10 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// buf[i] = TAB1[cipher[i]] ^ plain[i] — the per-byte content keystream.
|
||||
let mut buffer = [0u8; 10];
|
||||
for (i, b) in buffer.iter_mut().enumerate() {
|
||||
*b = TAB1[crypted[i] as usize] ^ decrypted[i];
|
||||
}
|
||||
|
||||
let mut key = [0u8; 5];
|
||||
let mut found = false;
|
||||
|
||||
for i_try in 0u32..0x1_0000 {
|
||||
let mut i_t1 = (i_try >> 8) | 0x100;
|
||||
let mut i_t2 = i_try & 0xff;
|
||||
let mut i_t3: u32 = 0; // not needed yet
|
||||
let mut i_t5: u32 = 0;
|
||||
|
||||
// Iterate the cipher 4 times to reconstruct LFSR0 (i_t3).
|
||||
for &b in buffer.iter().take(4) {
|
||||
let i_t4 = (TAB2[i_t2 as usize] ^ TAB3[i_t1 as usize]) as u32;
|
||||
i_t2 = i_t1 >> 1;
|
||||
i_t1 = ((i_t1 & 1) << 8) ^ i_t4;
|
||||
let i_t4 = TAB5[i_t4 as usize] as u32;
|
||||
|
||||
// Deduce i_t6 (LFSR0 output, pre-TAB4) and the carry.
|
||||
let mut i_t6 = b as u32;
|
||||
if i_t5 != 0 {
|
||||
i_t6 = (i_t6 + 0xff) & 0xff;
|
||||
}
|
||||
if i_t6 < i_t4 {
|
||||
i_t6 += 0x100;
|
||||
}
|
||||
i_t6 -= i_t4;
|
||||
i_t5 += i_t6 + i_t4;
|
||||
let i_t6 = TAB4[i_t6 as usize] as u32;
|
||||
|
||||
i_t3 = (i_t3 << 8) | i_t6;
|
||||
i_t5 >>= 8;
|
||||
}
|
||||
|
||||
let i_candidate = i_t3;
|
||||
|
||||
// Iterate 6 more times to validate the candidate.
|
||||
let mut i = 4usize;
|
||||
while i < 10 {
|
||||
let i_t4 = (TAB2[i_t2 as usize] ^ TAB3[i_t1 as usize]) as u32;
|
||||
i_t2 = i_t1 >> 1;
|
||||
i_t1 = ((i_t1 & 1) << 8) ^ i_t4;
|
||||
let i_t4 = TAB5[i_t4 as usize] as u32;
|
||||
let mut i_t6 = (((((((i_t3 >> 3) ^ i_t3) >> 1) ^ i_t3) >> 8) ^ i_t3) >> 5) & 0xff;
|
||||
i_t3 = (i_t3 << 8) | i_t6;
|
||||
i_t6 = TAB4[i_t6 as usize] as u32;
|
||||
i_t5 += i_t6 + i_t4;
|
||||
if (i_t5 & 0xff) as u8 != buffer[i] {
|
||||
break;
|
||||
}
|
||||
i_t5 >>= 8;
|
||||
i += 1;
|
||||
}
|
||||
|
||||
if i != 10 {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Four backward steps of iterating i_t3 to deduce the initial state.
|
||||
i_t3 = i_candidate;
|
||||
for _ in 0..4 {
|
||||
let i_t1_byte = i_t3 & 0xff;
|
||||
i_t3 >>= 8;
|
||||
// Brute-force the byte shifted in (top byte of the 24-bit reg).
|
||||
for j in 0u32..256 {
|
||||
i_t3 = (i_t3 & 0x1_ffff) | (j << 17);
|
||||
let i_t6 = (((((((i_t3 >> 3) ^ i_t3) >> 1) ^ i_t3) >> 8) ^ i_t3) >> 5) & 0xff;
|
||||
if i_t6 == i_t1_byte {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Undo `i_t3 = i_t3*2 + 8 - (i_t3 & 7)` to recover key[2..5].
|
||||
let i_t4 = (i_t3 >> 1).wrapping_sub(4);
|
||||
for i_t5 in 0u32..8 {
|
||||
let val = i_t4.wrapping_add(i_t5);
|
||||
if val.wrapping_mul(2).wrapping_add(8).wrapping_sub(val & 7) == i_t3 {
|
||||
key[0] = (i_try >> 8) as u8;
|
||||
key[1] = (i_try & 0xff) as u8;
|
||||
key[2] = (val & 0xff) as u8;
|
||||
key[3] = ((val >> 8) & 0xff) as u8;
|
||||
key[4] = ((val >> 16) & 0xff) as u8;
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// First fully-validated candidate wins. The 48-bit keystream constraint
|
||||
// makes a second match cryptographically negligible on real sectors, but
|
||||
// continuing would let a later spurious match overwrite a correct key.
|
||||
if found {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if found {
|
||||
for (k, &s) in key.iter_mut().zip(seed.iter()) {
|
||||
*k ^= s;
|
||||
}
|
||||
Some(key)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Recover the CSS title key from a scrambled sector using a known plaintext
|
||||
/// for the encrypted region.
|
||||
///
|
||||
/// `plain` is the expected plaintext at byte 0x80 (at least 10 bytes).
|
||||
/// Returns the recovered key only if it actually descrambles the sector back
|
||||
/// to `plain` — guarding against the rare spurious LFSR-seed match.
|
||||
pub fn recover_title_key(sector: &[u8], plain: &[u8]) -> Option<[u8; 5]> {
|
||||
if sector.len() < SECTOR_BYTES || plain.len() < 10 {
|
||||
return None;
|
||||
}
|
||||
if sector[FLAG_BYTE] & 0x30 == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let seed: [u8; 5] = [
|
||||
sector[SEED_OFFSET],
|
||||
sector[SEED_OFFSET + 1],
|
||||
sector[SEED_OFFSET + 2],
|
||||
sector[SEED_OFFSET + 3],
|
||||
sector[SEED_OFFSET + 4],
|
||||
];
|
||||
|
||||
let crypted = §or[ENCRYPTED_START..ENCRYPTED_START + 10];
|
||||
let key = recover_title_key_from_plain(crypted, plain, &seed)?;
|
||||
|
||||
if descramble_matches(sector, &key, plain) {
|
||||
Some(key)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Verify a title key by descrambling a copy of `sector` and checking the
|
||||
/// known plaintext reappears at byte 0x80.
|
||||
fn descramble_matches(sector: &[u8], title: &[u8; 5], plain: &[u8]) -> bool {
|
||||
let mut test = sector.to_vec();
|
||||
test[FLAG_BYTE] |= 0x10; // ensure scramble flag set for the descrambler
|
||||
descramble_sector(title, &mut test);
|
||||
let n = plain.len().min(SECTOR_BYTES - ENCRYPTED_START);
|
||||
test[ENCRYPTED_START..ENCRYPTED_START + n] == plain[..n]
|
||||
}
|
||||
|
||||
/// AttackPattern: find a repeating pattern just before the encrypted region
|
||||
/// and assume the plaintext at 0x80 continues it.
|
||||
///
|
||||
/// Functionally-equivalent port of libdvdcss `AttackPattern` (css.c) — finds the
|
||||
/// same periodic cribs on real DVD data, though its byte-comparison anchor
|
||||
/// differs from the C on phase-misaligned runs. Scans cleartext
|
||||
/// `sec[0x00..0x80]` for the longest run that repeats with a cycle length in
|
||||
/// 2..0x2F. If the run is long enough (`plen > 3` and at least two full
|
||||
/// cycles), the known plaintext at 0x80 is taken to be the periodic run
|
||||
/// continuing forward, and [`recover_title_key_from_plain`] is applied.
|
||||
pub fn crack_title_key(sector: &[u8]) -> Option<[u8; 5]> {
|
||||
if sector.len() < SECTOR_BYTES {
|
||||
return None;
|
||||
}
|
||||
if sector[FLAG_BYTE] & 0x30 == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Runaway guard: a single sector's crack is a bounded 2^16 LFSR search and
|
||||
// should finish in well under a second on any modern CPU. If it ever
|
||||
// exceeds ~2s wall-clock, something pathological is happening — log it so a
|
||||
// hang is never silent.
|
||||
let crack_t0 = std::time::Instant::now();
|
||||
|
||||
let result = crack_title_key_inner(sector);
|
||||
|
||||
let elapsed = crack_t0.elapsed();
|
||||
if elapsed.as_secs_f64() > 2.0 {
|
||||
tracing::warn!(
|
||||
target: "freemkv::css",
|
||||
elapsed_ms = elapsed.as_millis() as u64,
|
||||
found = result.is_some(),
|
||||
"css crack: single-sector recovery exceeded 2s (runaway guard)"
|
||||
);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Inner body of [`crack_title_key`] — the actual AttackPattern search. Split
|
||||
/// out so the public entry point can wall-clock the whole attempt for the
|
||||
/// runaway guard without threading a timer through every return path.
|
||||
/// AttackPattern crib: the predicted 10-byte plaintext at byte 0x80.
|
||||
///
|
||||
/// Scans the clear header `sec[0x00..0x80]` (never scrambled) for the longest
|
||||
/// run that repeats with a cycle length in 2..0x2F. If the run is long enough
|
||||
/// (`plen > 3` and at least two full cycles), the plaintext at 0x80 is taken to
|
||||
/// be that periodic run continuing forward. Returns `None` for an unscrambled
|
||||
/// sector or one with no usable run — such a sector can be neither cracked nor
|
||||
/// key-validated, only descrambled with an externally-cached key.
|
||||
///
|
||||
/// The header is untouched by `descramble_sector`, so the crib is identical
|
||||
/// before and after descramble: the decrypt path uses it as a per-sector
|
||||
/// "did the cached key descramble correctly?" oracle (the predicted plaintext
|
||||
/// must reappear at 0x80), and the cracker uses it as its known plaintext.
|
||||
pub(crate) fn attack_crib(sector: &[u8]) -> Option<[u8; 10]> {
|
||||
if sector.len() < SECTOR_BYTES || sector[FLAG_BYTE] & 0x30 == 0 {
|
||||
return None;
|
||||
}
|
||||
let mut best_plen: usize = 0;
|
||||
let mut best_p: usize = 0;
|
||||
|
||||
// For all cycle lengths from 2 to 0x2F.
|
||||
for i in 2usize..0x30 {
|
||||
// Count bytes that repeat with cycle length i, scanning backward from
|
||||
// 0x7F. `sec[0x7F - (j % i)] == sec[0x7F - j]`.
|
||||
let mut j = i + 1;
|
||||
while j < 0x80 && sector[0x7f - (j % i)] == sector[0x7f - j] {
|
||||
if j > best_plen {
|
||||
best_plen = j;
|
||||
best_p = i;
|
||||
}
|
||||
j += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Need at least a few repeated bytes and at least one full cycle.
|
||||
if best_plen > 3 && best_p > 0 && best_plen / best_p >= 2 {
|
||||
// The known plaintext is the periodic run continuing past 0x80. The
|
||||
// crib starts at `0x80 - (best_plen/best_p)*best_p` and continues
|
||||
// through the encrypted region; the bytes at and after 0x80 are the
|
||||
// predicted plaintext (the pattern repeats with period best_p).
|
||||
let cycles = best_plen / best_p;
|
||||
let plain_start = 0x80 - cycles * best_p;
|
||||
|
||||
// Each predicted byte is the run sample one or more periods back:
|
||||
// `sec[plain_start + (i % best_p)]`. For in-run offsets
|
||||
// (`plain_start + i < 0x80`) the run is exactly periodic, so this
|
||||
// equals `sec[plain_start + i]`; for offsets at/after 0x80 the raw
|
||||
// byte is ciphertext, so we MUST wrap within the period rather than
|
||||
// read it. (Reading `&sec[plain_start..+10]` directly — as before —
|
||||
// pulled ciphertext into the crib whenever the run covered fewer than
|
||||
// 10 bytes before 0x80, producing false-negative key recovery.)
|
||||
let mut plain = [0u8; 10];
|
||||
for (i, p) in plain.iter_mut().enumerate() {
|
||||
*p = sector[plain_start + (i % best_p)];
|
||||
}
|
||||
Some(plain)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn crack_title_key_inner(sector: &[u8]) -> Option<[u8; 5]> {
|
||||
let plain = attack_crib(sector)?;
|
||||
let seed: [u8; 5] = [
|
||||
sector[SEED_OFFSET],
|
||||
sector[SEED_OFFSET + 1],
|
||||
sector[SEED_OFFSET + 2],
|
||||
sector[SEED_OFFSET + 3],
|
||||
sector[SEED_OFFSET + 4],
|
||||
];
|
||||
let crypted = §or[0x80..0x80 + 10];
|
||||
if let Some(key) = recover_title_key_from_plain(crypted, &plain, &seed) {
|
||||
// Verify against the same predicted plaintext.
|
||||
if descramble_matches(sector, &key, &plain) {
|
||||
return Some(key);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::lfsr::scramble_sector;
|
||||
use super::*;
|
||||
|
||||
/// Build a synthetic scrambled sector for a given title key and seed,
|
||||
/// with `plain` placed as the plaintext at byte 0x80, scrambled with
|
||||
/// EXACTLY the cipher `descramble_sector` inverts. Returns
|
||||
/// (scrambled_sector, full_plaintext_body).
|
||||
fn synth_sector(title_key: &[u8; 5], seed: &[u8; 5], plain: &[u8]) -> (Vec<u8>, Vec<u8>) {
|
||||
let mut plaintext = vec![0u8; SECTOR_BYTES];
|
||||
plaintext[0..4].copy_from_slice(&[0x00, 0x00, 0x01, 0xBA]);
|
||||
plaintext[FLAG_BYTE] = 0x10;
|
||||
plaintext[SEED_OFFSET..SEED_OFFSET + 5].copy_from_slice(seed);
|
||||
plaintext[ENCRYPTED_START..ENCRYPTED_START + plain.len()].copy_from_slice(plain);
|
||||
|
||||
let body = plaintext.clone();
|
||||
|
||||
// scramble_sector turns the plaintext body into ciphertext and sets
|
||||
// the scramble flag.
|
||||
scramble_sector(title_key, &mut plaintext);
|
||||
(plaintext, body)
|
||||
}
|
||||
|
||||
/// Build a synthetic scrambled sector whose CLEARTEXT (0x00..0x80) ends
|
||||
/// in a periodic run that continues into the encrypted region — the case
|
||||
/// `AttackPattern` (crack_title_key) is designed to crack.
|
||||
fn synth_periodic_sector(
|
||||
title_key: &[u8; 5],
|
||||
seed: &[u8; 5],
|
||||
period: usize,
|
||||
) -> (Vec<u8>, Vec<u8>) {
|
||||
let mut plaintext = vec![0u8; SECTOR_BYTES];
|
||||
plaintext[FLAG_BYTE] = 0x10;
|
||||
|
||||
// A clean periodic run occupying the tail of the cleartext header
|
||||
// (RUN_START..0x80) and continuing into the encrypted region. This
|
||||
// mirrors a real VOB: a periodic data run just before the scrambled
|
||||
// part. The run must NOT overlap the seed bytes (0x54..0x59), or the
|
||||
// AttackPattern detector would break mid-run. The phase is anchored to
|
||||
// offset 0 so the run is consistent across the 0x80 boundary.
|
||||
// Just above the seed (0x54..0x59); gives a 39-byte run (0x59..0x80)
|
||||
// — enough for >=2 cycles of every tested period (<=19).
|
||||
const RUN_START: usize = 0x59;
|
||||
let pat: Vec<u8> = (0..period)
|
||||
.map(|k| (0xA0u8.wrapping_add(k as u8)) ^ 0x5A)
|
||||
.collect();
|
||||
for (i, b) in plaintext.iter_mut().enumerate().skip(RUN_START) {
|
||||
*b = pat[i % period];
|
||||
}
|
||||
|
||||
// Seed sits below the run, undisturbed.
|
||||
plaintext[SEED_OFFSET..SEED_OFFSET + 5].copy_from_slice(seed);
|
||||
|
||||
let body = plaintext.clone();
|
||||
scramble_sector(title_key, &mut plaintext);
|
||||
(plaintext, body)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crack_unscrambled_returns_none() {
|
||||
let sector = vec![0u8; SECTOR_BYTES];
|
||||
assert!(crack_title_key(§or).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crack_too_short_returns_none() {
|
||||
let sector = vec![0u8; 100];
|
||||
assert!(crack_title_key(§or).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recover_needs_min_plain() {
|
||||
let sector = vec![0u8; SECTOR_BYTES];
|
||||
let short_plain = [0u8; 4];
|
||||
assert!(recover_title_key(§or, &short_plain).is_none());
|
||||
}
|
||||
|
||||
/// The known plaintext used at byte 0x80 for the direct-recovery tests.
|
||||
/// A realistic MPEG-2 PES header start.
|
||||
const PES: [u8; 10] = [0x00, 0x00, 0x01, 0xE0, 0x00, 0x00, 0x80, 0x80, 0x05, 0x21];
|
||||
|
||||
/// MANDATORY round-trip (Task C.1): synthesize a scrambled sector for a
|
||||
/// known (title_key, seed), then assert recover_title_key returns a key
|
||||
/// that descrambles the body back to plaintext. CSS title-key recovery is
|
||||
/// well-defined up to keys that scramble identically; we assert the full
|
||||
/// body round-trips (the true correctness property), and additionally
|
||||
/// that the EXACT key is returned for the common case.
|
||||
#[test]
|
||||
fn recover_round_trips_known_keys() {
|
||||
let cases: &[([u8; 5], [u8; 5])] = &[
|
||||
(
|
||||
[0x42, 0x13, 0x37, 0xBE, 0xEF],
|
||||
[0x11, 0x22, 0x33, 0x44, 0x55],
|
||||
),
|
||||
(
|
||||
[0x01, 0x02, 0x03, 0x04, 0x05],
|
||||
[0xDE, 0xAD, 0xBE, 0xEF, 0x42],
|
||||
),
|
||||
(
|
||||
[0xFE, 0xDC, 0xBA, 0x98, 0x76],
|
||||
[0x00, 0xFF, 0x80, 0x7F, 0x01],
|
||||
),
|
||||
(
|
||||
[0x9A, 0x78, 0x56, 0x34, 0x12],
|
||||
[0xA5, 0x5A, 0x0F, 0xF0, 0xCC],
|
||||
),
|
||||
(
|
||||
[0xFF, 0xFF, 0xFF, 0xFF, 0xFF],
|
||||
[0x01, 0x01, 0x01, 0x01, 0x01],
|
||||
),
|
||||
];
|
||||
for (title_key, seed) in cases {
|
||||
let (mut sector, body) = synth_sector(title_key, seed, &PES);
|
||||
let recovered =
|
||||
recover_title_key(§or, &PES).expect("recover_title_key returned None");
|
||||
descramble_sector(&recovered, &mut sector);
|
||||
assert_eq!(
|
||||
§or[ENCRYPTED_START..SECTOR_BYTES],
|
||||
&body[ENCRYPTED_START..SECTOR_BYTES],
|
||||
"recovered key did not descramble the full body for \
|
||||
title={title_key:02x?} seed={seed:02x?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// MANDATORY (Task C.1): the AttackPattern entry point crack_title_key —
|
||||
/// no plaintext supplied — recovers a round-tripping key when the
|
||||
/// cleartext ends in a periodic run that continues into 0x80.
|
||||
#[test]
|
||||
fn crack_title_key_recovers_via_attack_pattern() {
|
||||
for &period in &[2usize, 3, 5, 8, 16] {
|
||||
let title_key = [0x42, 0x13, 0x37, 0xBE, 0xEF];
|
||||
let seed = [0x11, 0x22, 0x33, 0x44, 0x55];
|
||||
let (sector, body) = synth_periodic_sector(&title_key, &seed, period);
|
||||
|
||||
let cracked = crack_title_key(§or)
|
||||
.unwrap_or_else(|| panic!("crack_title_key returned None for period {period}"));
|
||||
let mut test = sector.clone();
|
||||
descramble_sector(&cracked, &mut test);
|
||||
assert_eq!(
|
||||
&test[ENCRYPTED_START..SECTOR_BYTES],
|
||||
&body[ENCRYPTED_START..SECTOR_BYTES],
|
||||
"crack_title_key key did not round-trip the body (period {period})"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// recover_title_key_from_plain inverts dvdcss_unscramble exactly: scramble
|
||||
/// a known body, hand back the keystream-derived key, and the recovered
|
||||
/// key (XOR-back included) reproduces the plaintext.
|
||||
#[test]
|
||||
fn recovered_key_descrambles_back_to_plaintext() {
|
||||
let cases: &[([u8; 5], [u8; 5])] = &[
|
||||
(
|
||||
[0x42, 0x13, 0x37, 0xBE, 0xEF],
|
||||
[0x11, 0x22, 0x33, 0x44, 0x55],
|
||||
),
|
||||
(
|
||||
[0x9A, 0x78, 0x56, 0x34, 0x12],
|
||||
[0xA5, 0x5A, 0x0F, 0xF0, 0xCC],
|
||||
),
|
||||
(
|
||||
[0xFF, 0xFF, 0xFF, 0xFF, 0xFF],
|
||||
[0x01, 0x01, 0x01, 0x01, 0x01],
|
||||
),
|
||||
];
|
||||
for (title_key, seed) in cases {
|
||||
let (mut sector, body) = synth_sector(title_key, seed, &PES);
|
||||
let recovered =
|
||||
recover_title_key(§or, &PES).expect("recover_title_key returned None");
|
||||
descramble_sector(&recovered, &mut sector);
|
||||
assert_eq!(
|
||||
§or[ENCRYPTED_START..SECTOR_BYTES],
|
||||
&body[ENCRYPTED_START..SECTOR_BYTES],
|
||||
"descramble with recovered key did not reproduce the body \
|
||||
for title={title_key:02x?} seed={seed:02x?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── early-return guards ────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn recover_rejects_sector_one_byte_short() {
|
||||
let mut sector = vec![0u8; SECTOR_BYTES - 1];
|
||||
sector[FLAG_BYTE] = 0x30;
|
||||
assert!(recover_title_key(§or, &PES).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recover_rejects_unscrambled_sector() {
|
||||
let sector = vec![0x00u8; SECTOR_BYTES];
|
||||
assert!(recover_title_key(§or, &PES).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recover_high_flag_bits_are_not_scramble() {
|
||||
for &flag in &[0x40u8, 0x80, 0xC0] {
|
||||
let mut sector = vec![0x11u8; SECTOR_BYTES];
|
||||
sector[FLAG_BYTE] = flag;
|
||||
assert!(
|
||||
recover_title_key(§or, &PES).is_none(),
|
||||
"flag {flag:#04x} has scramble bits clear; recover must return None"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crack_high_flag_bits_are_not_scramble() {
|
||||
for &flag in &[0x40u8, 0x80, 0xC0] {
|
||||
let mut sector = vec![0x11u8; SECTOR_BYTES];
|
||||
sector[FLAG_BYTE] = flag;
|
||||
assert!(
|
||||
crack_title_key(§or).is_none(),
|
||||
"flag {flag:#04x} clear scramble bits -> crack must return None"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crack_rejects_sector_one_byte_short() {
|
||||
let mut sector = vec![0u8; SECTOR_BYTES - 1];
|
||||
if sector.len() > FLAG_BYTE {
|
||||
sector[FLAG_BYTE] = 0x30;
|
||||
}
|
||||
assert!(crack_title_key(§or).is_none());
|
||||
}
|
||||
|
||||
/// crack_title_key must never panic on a fully scrambled sector with
|
||||
/// arbitrary (non-periodic) content — it just returns None.
|
||||
#[test]
|
||||
fn crack_full_path_never_panics() {
|
||||
for seed in 0u32..3 {
|
||||
let mut sector = vec![0u8; SECTOR_BYTES];
|
||||
sector[FLAG_BYTE] = 0x30;
|
||||
let mut x = seed.wrapping_mul(2_654_435_761).wrapping_add(7);
|
||||
for b in sector.iter_mut().skip(0x80) {
|
||||
x = x.wrapping_mul(1_103_515_245).wrapping_add(12_345);
|
||||
*b = (x >> 16) as u8;
|
||||
}
|
||||
for (i, b) in sector[SEED_OFFSET..SEED_OFFSET + 5].iter_mut().enumerate() {
|
||||
*b = (seed.wrapping_add(i as u32) ^ 0xA5) as u8;
|
||||
}
|
||||
let _ = crack_title_key(§or);
|
||||
}
|
||||
}
|
||||
}
|
||||
+191
-28
@@ -25,6 +25,8 @@ pub const TAB1: [u8; 256] = [
|
||||
];
|
||||
|
||||
/// Table 2: LFSR1 high-byte feedback permutation.
|
||||
///
|
||||
/// Byte-identical to libdvdcss `p_css_tab2` (csstables.h).
|
||||
pub const TAB2: [u8; 256] = [
|
||||
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x09, 0x08, 0x0b, 0x0a, 0x0d, 0x0c, 0x0f, 0x0e,
|
||||
0x12, 0x13, 0x10, 0x11, 0x16, 0x17, 0x14, 0x15, 0x1b, 0x1a, 0x19, 0x18, 0x1f, 0x1e, 0x1d, 0x1c,
|
||||
@@ -40,11 +42,18 @@ pub const TAB2: [u8; 256] = [
|
||||
0xa4, 0xa5, 0xa6, 0xa7, 0xa0, 0xa1, 0xa2, 0xa3, 0xad, 0xac, 0xaf, 0xae, 0xa9, 0xa8, 0xab, 0xaa,
|
||||
0xdb, 0xda, 0xd9, 0xd8, 0xdf, 0xde, 0xdd, 0xdc, 0xd2, 0xd3, 0xd0, 0xd1, 0xd6, 0xd7, 0xd4, 0xd5,
|
||||
0xc9, 0xc8, 0xcb, 0xca, 0xcd, 0xcc, 0xcf, 0xce, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
|
||||
0xed, 0xec, 0xef, 0xee, 0xe9, 0xe8, 0xeb, 0xea, 0xe4, 0xe5, 0xe6, 0xe7, 0xe0, 0xe1, 0xe2, 0xe3,
|
||||
0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, 0xf6, 0xf7, 0xf4, 0xf5, 0xf2, 0xf3, 0xf0, 0xf1,
|
||||
0xed, 0xec, 0xef, 0xee, 0xe9, 0xe8, 0xeb, 0xea, 0xe4, 0xe5, 0xe6, 0xe7, 0xe0, 0xe1, 0xe2, 0xe3,
|
||||
];
|
||||
|
||||
/// Table 3: LFSR1 low-byte feedback permutation.
|
||||
/// Table 3: LFSR1 9-bit low-word feedback table (512 entries).
|
||||
///
|
||||
/// Byte-identical to libdvdcss `p_css_tab3` (csstables.h): the 8-value
|
||||
/// block `BASE[i & 7]` repeated 64 times. The CSS LFSR1 step indexes this
|
||||
/// table with the 9-bit low register (0x100..=0x1FF), but only the low 3
|
||||
/// bits select the output — the high bits are ignored, hence the constant
|
||||
/// blocks. The 512-entry width simply lets the 9-bit index be used without
|
||||
/// masking.
|
||||
pub const TAB3: [u8; 512] = [
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
@@ -54,30 +63,30 @@ pub const TAB3: [u8; 512] = [
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe, 0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe,
|
||||
0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe, 0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe,
|
||||
0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe, 0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe,
|
||||
0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe, 0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe,
|
||||
0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe, 0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe,
|
||||
0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe, 0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe,
|
||||
0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe, 0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe,
|
||||
0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe, 0x01, 0x25, 0x48, 0x6c, 0x93, 0xb7, 0xda, 0xfe,
|
||||
0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd, 0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd,
|
||||
0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd, 0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd,
|
||||
0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd, 0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd,
|
||||
0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd, 0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd,
|
||||
0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd, 0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd,
|
||||
0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd, 0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd,
|
||||
0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd, 0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd,
|
||||
0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd, 0x02, 0x26, 0x4b, 0x6f, 0x90, 0xb4, 0xd9, 0xfd,
|
||||
0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc, 0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc,
|
||||
0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc, 0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc,
|
||||
0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc, 0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc,
|
||||
0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc, 0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc,
|
||||
0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc, 0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc,
|
||||
0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc, 0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc,
|
||||
0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc, 0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc,
|
||||
0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc, 0x03, 0x27, 0x4a, 0x6e, 0x91, 0xb5, 0xd8, 0xfc,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff, 0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff,
|
||||
];
|
||||
|
||||
/// Table 4: LFSR0 byte permutation (used in initialization and output).
|
||||
@@ -100,8 +109,10 @@ pub const TAB4: [u8; 256] = [
|
||||
0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff,
|
||||
];
|
||||
|
||||
/// Table 5: LFSR1 output permutation for the Stevenson attack.
|
||||
/// This is the inverse byte-reversal of TAB4.
|
||||
/// Table 5: LFSR1 output permutation used in the keystream combiner.
|
||||
/// `TAB5[i] == TAB4[i] ^ 0xFF` (bitwise complement of the TAB4 bit-reversal
|
||||
/// table). Applied on the normal descramble/recrypt path (lfsr.rs) as well as
|
||||
/// in the key-recovery fallback (crack.rs).
|
||||
pub const TAB5: [u8; 256] = [
|
||||
0xff, 0x7f, 0xbf, 0x3f, 0xdf, 0x5f, 0x9f, 0x1f, 0xef, 0x6f, 0xaf, 0x2f, 0xcf, 0x4f, 0x8f, 0x0f,
|
||||
0xf7, 0x77, 0xb7, 0x37, 0xd7, 0x57, 0x97, 0x17, 0xe7, 0x67, 0xa7, 0x27, 0xc7, 0x47, 0x87, 0x07,
|
||||
@@ -120,3 +131,155 @@ pub const TAB5: [u8; 256] = [
|
||||
0xf8, 0x78, 0xb8, 0x38, 0xd8, 0x58, 0x98, 0x18, 0xe8, 0x68, 0xa8, 0x28, 0xc8, 0x48, 0x88, 0x08,
|
||||
0xf0, 0x70, 0xb0, 0x30, 0xd0, 0x50, 0x90, 0x10, 0xe0, 0x60, 0xa0, 0x20, 0xc0, 0x40, 0x80, 0x00,
|
||||
];
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Pins the documented relationship `TAB5[i] == TAB4[i] ^ 0xFF` so the
|
||||
/// table doc cannot drift from the data.
|
||||
#[test]
|
||||
fn tab5_is_complement_of_tab4() {
|
||||
for i in 0..256 {
|
||||
assert_eq!(
|
||||
TAB5[i],
|
||||
TAB4[i] ^ 0xFF,
|
||||
"TAB5[{i:#04x}] != TAB4[{i:#04x}] ^ 0xFF"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// TAB1 is a bijection on 0..256. CSS uses it as an invertible output
|
||||
/// permutation in css_DecryptKey's chained-XOR rounds; if two inputs
|
||||
/// collided, the key mangling would not be invertible.
|
||||
///
|
||||
/// Mutation: duplicate any value (e.g. set TAB1[1] = TAB1[0]) -> the
|
||||
/// "maps two inputs" assert fires.
|
||||
#[test]
|
||||
fn tab1_is_a_permutation() {
|
||||
let mut seen = [false; 256];
|
||||
for (i, &v) in TAB1.iter().enumerate() {
|
||||
assert!(
|
||||
!seen[v as usize],
|
||||
"TAB1 maps two inputs to {v:#04x} (collision at index {i:#04x})"
|
||||
);
|
||||
seen[v as usize] = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// TAB1's fixed structural anchors from the CSS spec table:
|
||||
/// TAB1[0x00] == 0x33 and the inverse TAB1[0x33] == 0x00. These two
|
||||
/// entries are the canonical first-row / inverse-lookup landmarks of the
|
||||
/// published CSS TAB1 and pin the table's orientation.
|
||||
///
|
||||
/// Grounding: CSS specification TAB1, row 0 col 0 = 0x33; index 0x33
|
||||
/// (row 3 col 3) = 0x00.
|
||||
/// Mutation: change the first literal `0x33` in TAB1 -> first assert fails.
|
||||
#[test]
|
||||
fn tab1_known_spec_anchors() {
|
||||
assert_eq!(TAB1[0x00], 0x33, "TAB1[0] is the published 0x33");
|
||||
assert_eq!(TAB1[0x33], 0x00, "TAB1[0x33] is the published 0x00");
|
||||
}
|
||||
|
||||
/// TAB2 is a permutation of 0..256 (it is the LFSR1 high-byte feedback
|
||||
/// substitution). A non-bijective TAB2 would bias the LFSR1 keystream.
|
||||
///
|
||||
/// Mutation: set TAB2[8] = 0x00 (collides with TAB2[0]) -> assert fires.
|
||||
#[test]
|
||||
fn tab2_is_a_permutation() {
|
||||
let mut seen = [false; 256];
|
||||
for (i, &v) in TAB2.iter().enumerate() {
|
||||
assert!(
|
||||
!seen[v as usize],
|
||||
"TAB2 maps two inputs to {v:#04x} (collision at index {i:#04x})"
|
||||
);
|
||||
seen[v as usize] = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// TAB3 is the libdvdcss `p_css_tab3`: the 8-value feedback block
|
||||
/// BASE = [0x00,0x24,0x49,0x6d,0x92,0xb6,0xdb,0xff]
|
||||
/// repeated 64 times — `TAB3[i] == BASE[i & 7]`. The high bits of the
|
||||
/// 9-bit index do not affect the output (libdvdcss's LFSR1 step indexes
|
||||
/// with the full 9-bit low register but only `& 7` matters). This pins
|
||||
/// all 512 entries to the published table.
|
||||
///
|
||||
/// Mutation: flip any single byte in the TAB3 literal -> the formula
|
||||
/// check fails at that index.
|
||||
#[test]
|
||||
fn tab3_matches_lfsr1_generating_formula() {
|
||||
const BASE: [u8; 8] = [0x00, 0x24, 0x49, 0x6d, 0x92, 0xb6, 0xdb, 0xff];
|
||||
for i in 0..512usize {
|
||||
let expected = BASE[i & 7];
|
||||
assert_eq!(
|
||||
TAB3[i], expected,
|
||||
"TAB3[{i:#05x}] = {:#04x}, formula BASE[i&7] = {expected:#04x}",
|
||||
TAB3[i]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// TAB4 is the exact bit-reversal of each byte (CSS uses it to permute
|
||||
/// LFSR0 bytes on seed and output). TAB4[b] reverses b's 8 bits MSB<->LSB.
|
||||
/// Therefore it is also an involution: TAB4[TAB4[b]] == b.
|
||||
///
|
||||
/// Grounding: TAB4[0x01]=0x80, TAB4[0x80]=0x01, TAB4[0x00]=0x00,
|
||||
/// TAB4[0xFF]=0xFF.
|
||||
/// Mutation: set TAB4[1] = 0x40 (not the reversal 0x80) -> bit-reversal
|
||||
/// check fails at index 1.
|
||||
#[test]
|
||||
fn tab4_is_exact_bit_reversal_and_involution() {
|
||||
for b in 0u16..256 {
|
||||
let rev = (0..8).fold(0u8, |acc, k| acc | (((b as u8 >> k) & 1) << (7 - k)));
|
||||
assert_eq!(
|
||||
TAB4[b as usize], rev,
|
||||
"TAB4[{b:#04x}] is not the bit-reversal {rev:#04x}"
|
||||
);
|
||||
}
|
||||
for b in 0..256usize {
|
||||
assert_eq!(
|
||||
TAB4[TAB4[b] as usize], b as u8,
|
||||
"TAB4 not an involution at {b:#04x}"
|
||||
);
|
||||
}
|
||||
// Spec landmark entries.
|
||||
assert_eq!(TAB4[0x01], 0x80);
|
||||
assert_eq!(TAB4[0x80], 0x01);
|
||||
assert_eq!(TAB4[0x00], 0x00);
|
||||
assert_eq!(TAB4[0xFF], 0xFF);
|
||||
}
|
||||
|
||||
/// TAB4 is a permutation (bit-reversal is bijective). Distinct from the
|
||||
/// reversal test: a table that is "reversal except two swapped entries"
|
||||
/// would still be a permutation, and a table that is "reversal except one
|
||||
/// duplicated entry" would fail this but might pass a sampled reversal
|
||||
/// check — the two tests pin different failure modes.
|
||||
///
|
||||
/// Mutation: set TAB4[2] = TAB4[1] -> permutation assert fires.
|
||||
#[test]
|
||||
fn tab4_is_a_permutation() {
|
||||
let mut seen = [false; 256];
|
||||
for &v in TAB4.iter() {
|
||||
assert!(!seen[v as usize], "TAB4 maps two inputs to {v:#04x}");
|
||||
seen[v as usize] = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// TAB5 is also a permutation (complement of a bijection is a bijection)
|
||||
/// and its own self-consistency landmark: TAB5[0x00] == 0xFF (TAB4[0]^0xFF)
|
||||
/// and TAB5[0xFF] == 0x00 (TAB4[0xFF]^0xFF). Pins orientation independent
|
||||
/// of the complement-loop test.
|
||||
///
|
||||
/// Mutation: change the first TAB5 literal 0xff -> 0xfe -> the landmark
|
||||
/// and permutation checks both catch it.
|
||||
#[test]
|
||||
fn tab5_is_permutation_with_anchors() {
|
||||
let mut seen = [false; 256];
|
||||
for &v in TAB5.iter() {
|
||||
assert!(!seen[v as usize], "TAB5 maps two inputs to {v:#04x}");
|
||||
seen[v as usize] = true;
|
||||
}
|
||||
assert_eq!(TAB5[0x00], 0xFF, "TAB5[0] = TAB4[0]^0xFF = 0xFF");
|
||||
assert_eq!(TAB5[0xFF], 0x00, "TAB5[0xFF] = TAB4[0xFF]^0xFF = 0x00");
|
||||
}
|
||||
}
|
||||
|
||||
+1315
-17
File diff suppressed because it is too large
Load Diff
+753
@@ -0,0 +1,753 @@
|
||||
//! Structured scan diagnostics — the `--log-level 3` self-diagnosing dump.
|
||||
//!
|
||||
//! A bug report log must be self-diagnosing: everything needed to explain
|
||||
//! *why* freemkv made the choices it did at scan must be in the log, in a
|
||||
//! compact, machine-parseable form. This module emits one terse line per row
|
||||
//! (title, cell, stream, decision) under the `tracing` target
|
||||
//! `freemkv::diag`, which the CLI routes to `log.txt` when `--log-level 3`
|
||||
//! (debug) is set.
|
||||
//!
|
||||
//! Format conventions (stable, greppable):
|
||||
//! - Every line is prefixed by a `tag=` so a log scraper can filter
|
||||
//! (`disc`, `title`, `dvd.cell`, `dvd.vattr`, `dvd.aattr`, `bd.clip`,
|
||||
//! `bd.mark`, `aacs`, `stream`, `decision`).
|
||||
//! - Raw bytes are shown as `0xNN` next to their decode so a wrong decode
|
||||
//! is obvious against the raw value.
|
||||
//! - This module only READS already-parsed scan state — it never re-reads
|
||||
//! the disc and never mutates anything.
|
||||
//!
|
||||
//! The DVD per-cell table (with the raw cell-category byte) is emitted from
|
||||
//! the IFO scan itself ([`dump_dvd_cells`]), because the per-cell
|
||||
//! `ifo::DvdCell` detail is lowered away before the `Disc` is built. The
|
||||
//! `Disc`-level dump ([`dump_disc`]) covers everything that survives
|
||||
//! lowering: titles, streams, the picked main feature, and AACS state.
|
||||
|
||||
use crate::disc::{
|
||||
AudioChannels, ColorSpace, Disc, DiscTitle, FrameRate, HdrFormat, Resolution, SampleRate,
|
||||
Stream,
|
||||
};
|
||||
use crate::ifo::{CellCategory, DvdTitle};
|
||||
|
||||
const DIAG: &str = "freemkv::diag";
|
||||
|
||||
// ── small format helpers (pure, unit-testable) ──────────────────────────────
|
||||
|
||||
/// Compact name for a [`Resolution`] with the interlace marker preserved.
|
||||
pub fn res_str(r: Resolution) -> &'static str {
|
||||
match r {
|
||||
Resolution::R480i => "480i",
|
||||
Resolution::R480p => "480p",
|
||||
Resolution::R576i => "576i",
|
||||
Resolution::R576p => "576p",
|
||||
Resolution::R720p => "720p",
|
||||
Resolution::R1080i => "1080i",
|
||||
Resolution::R1080p => "1080p",
|
||||
Resolution::R2160p => "2160p",
|
||||
Resolution::R4320p => "4320p",
|
||||
Resolution::Unknown => "res?",
|
||||
}
|
||||
}
|
||||
|
||||
/// Frames-per-second string for a [`FrameRate`].
|
||||
pub fn fps_str(f: FrameRate) -> &'static str {
|
||||
match f {
|
||||
FrameRate::F23_976 => "23.976",
|
||||
FrameRate::F24 => "24",
|
||||
FrameRate::F25 => "25",
|
||||
FrameRate::F29_97 => "29.97",
|
||||
FrameRate::F30 => "30",
|
||||
FrameRate::F50 => "50",
|
||||
FrameRate::F59_94 => "59.94",
|
||||
FrameRate::F60 => "60",
|
||||
FrameRate::Unknown => "fps?",
|
||||
}
|
||||
}
|
||||
|
||||
/// PAL/NTSC field-rate family inferred from the frame rate (DVD has no
|
||||
/// explicit field, so this is the colour/standard the muxer stamps).
|
||||
pub fn tv_system_str(f: FrameRate) -> &'static str {
|
||||
match f {
|
||||
FrameRate::F25 | FrameRate::F50 => "PAL",
|
||||
FrameRate::F23_976 | FrameRate::F29_97 | FrameRate::F59_94 => "NTSC",
|
||||
_ => "—",
|
||||
}
|
||||
}
|
||||
|
||||
/// CICP-ish short name for a [`ColorSpace`].
|
||||
pub fn color_str(c: ColorSpace) -> &'static str {
|
||||
match c {
|
||||
ColorSpace::Bt709 => "BT.709",
|
||||
ColorSpace::Bt2020 => "BT.2020",
|
||||
ColorSpace::Bt470bg => "BT.470BG",
|
||||
ColorSpace::Smpte170m => "SMPTE-170M",
|
||||
ColorSpace::Unknown => "color?",
|
||||
}
|
||||
}
|
||||
|
||||
/// HDR format short name.
|
||||
pub fn hdr_str(h: HdrFormat) -> &'static str {
|
||||
match h {
|
||||
HdrFormat::Sdr => "SDR",
|
||||
HdrFormat::Hdr10 => "HDR10",
|
||||
HdrFormat::Hdr10Plus => "HDR10+",
|
||||
HdrFormat::DolbyVision => "DoVi",
|
||||
HdrFormat::Hlg => "HLG",
|
||||
}
|
||||
}
|
||||
|
||||
/// Channel count from an [`AudioChannels`] layout (what lands in the MKV
|
||||
/// `Channels` element).
|
||||
pub fn channel_count(ch: AudioChannels) -> u8 {
|
||||
match ch {
|
||||
AudioChannels::Mono => 1,
|
||||
AudioChannels::Stereo => 2,
|
||||
AudioChannels::Stereo21 => 3,
|
||||
AudioChannels::Quad => 4,
|
||||
AudioChannels::Surround50 => 5,
|
||||
AudioChannels::Surround51 => 6,
|
||||
AudioChannels::Surround61 => 7,
|
||||
AudioChannels::Surround71 => 8,
|
||||
AudioChannels::Unknown => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sample-rate in Hz for a [`SampleRate`].
|
||||
pub fn sample_rate_hz(s: SampleRate) -> u32 {
|
||||
match s {
|
||||
SampleRate::S44_1 => 44100,
|
||||
SampleRate::S48 => 48000,
|
||||
SampleRate::S88_2 => 88200,
|
||||
SampleRate::S96 => 96000,
|
||||
SampleRate::S176_4 => 176400,
|
||||
SampleRate::S192 => 192000,
|
||||
SampleRate::S48_96 => 96000,
|
||||
SampleRate::S48_192 => 192000,
|
||||
SampleRate::Unknown => 0,
|
||||
}
|
||||
}
|
||||
|
||||
// ── DVD cell-category dump (from the IFO scan, pre-lowering) ─────────────────
|
||||
|
||||
/// One formatted cell row for the DVD per-PGC cell table. Returned as a
|
||||
/// string so it can be unit-tested without a logger.
|
||||
///
|
||||
/// Columns: `idx`, raw category (`cat=0xNN`) + decoded fields, first/last
|
||||
/// sector, duration, and the keep/drop verdict from the bug-4 leading-cell
|
||||
/// filter.
|
||||
pub fn dvd_cell_row(idx: usize, cell: &crate::ifo::DvdCell, dropped: bool) -> String {
|
||||
let c = CellCategory::decode(cell.category);
|
||||
// Per-cell keep/skip REASON (self-sufficient bug log): a dropped cell is a
|
||||
// leading secondary angle/interleave block piece; a kept cell is either the
|
||||
// first feature cell or genuine feature content. This makes the
|
||||
// leading-cell-filter decision auditable from the log without the disc.
|
||||
let verdict = if dropped {
|
||||
"DROP(leading-secondary-block-piece)"
|
||||
} else if c.is_secondary_block_piece() {
|
||||
// Kept despite being a secondary piece — only happens past the leading
|
||||
// run (the filter stops at the first plain feature cell).
|
||||
"keep(feature-body)"
|
||||
} else {
|
||||
"keep(plain-feature)"
|
||||
};
|
||||
format!(
|
||||
"tag=dvd.cell idx={idx} cat=0x{:02X} block_mode={} block_type={} \
|
||||
seamless={} ilv={} stc={} angle={} plain={} first={} last={} dur={:.1}s {}",
|
||||
cell.category,
|
||||
c.block_mode,
|
||||
c.block_type,
|
||||
c.seamless_play as u8,
|
||||
c.interleaved as u8,
|
||||
c.stc_discontinuity as u8,
|
||||
c.seamless_angle as u8,
|
||||
c.is_plain_feature() as u8,
|
||||
cell.first_sector,
|
||||
cell.last_sector,
|
||||
cell.duration_secs,
|
||||
verdict,
|
||||
)
|
||||
}
|
||||
|
||||
/// Emit the per-PGC cell table for one DVD title during the IFO scan.
|
||||
///
|
||||
/// `vts`/`title` identify the row group; `title` is the `DvdTitle` whose
|
||||
/// cells (and bug-4 leading-cell verdict) are dumped. Called from
|
||||
/// `scan_dvd_titles` while the `DvdTitle` is still in scope (the per-cell
|
||||
/// category byte is lowered away before the `Disc` exists).
|
||||
pub fn dump_dvd_cells(vts: u8, title_num: u16, title: &DvdTitle) {
|
||||
if !tracing::enabled!(target: DIAG, tracing::Level::DEBUG) {
|
||||
return;
|
||||
}
|
||||
let feature_start = title.feature_start_cell();
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=dvd.pgc vts={vts} title={title_num} cells={} chapters={} \
|
||||
dur={:.1}s feature_start_cell={feature_start}",
|
||||
title.cells.len(),
|
||||
title.chapters,
|
||||
title.duration_secs,
|
||||
);
|
||||
for (i, cell) in title.cells.iter().enumerate() {
|
||||
tracing::debug!(target: DIAG, "{}", dvd_cell_row(i, cell, i < feature_start));
|
||||
}
|
||||
// Chapter/PTT map (program → cumulative start time).
|
||||
for (i, &t) in title.chapter_times.iter().enumerate() {
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=dvd.chap vts={vts} title={title_num} ch={} time={:.1}s",
|
||||
i + 1,
|
||||
t,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Emit the IFO `video_attr` / `audio_attr` decode for one DVD title set,
|
||||
/// showing the raw bytes next to their decoded meaning. Called from the IFO
|
||||
/// scan with the still-parsed `ifo::DvdTitleSet` view.
|
||||
pub fn dump_dvd_attrs(ts: &crate::ifo::DvdTitleSet) {
|
||||
if !tracing::enabled!(target: DIAG, tracing::Level::DEBUG) {
|
||||
return;
|
||||
}
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=dvd.vobs vts={} vob_start_sector={}",
|
||||
ts.vts_number,
|
||||
ts.vob_start_sector,
|
||||
);
|
||||
let v = &ts.video;
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=dvd.vattr vts={} codec={:?} res={} aspect={:?} std={:?}",
|
||||
ts.vts_number,
|
||||
v.codec,
|
||||
res_str(v.resolution),
|
||||
v.aspect,
|
||||
v.standard,
|
||||
);
|
||||
for (i, a) in ts.audio_streams.iter().enumerate() {
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=dvd.aattr vts={} idx={i} codec={:?} ch={} sr={}Hz lang={:?} sub_id={:?}",
|
||||
ts.vts_number,
|
||||
a.codec,
|
||||
a.channels,
|
||||
a.sample_rate,
|
||||
a.language,
|
||||
a.sub_stream_id.map(|x| format!("0x{x:02X}")),
|
||||
);
|
||||
}
|
||||
for (i, s) in ts.subtitle_streams.iter().enumerate() {
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=dvd.sattr vts={} idx={i} lang={:?}",
|
||||
ts.vts_number,
|
||||
s.language,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Emit the ACTUAL per-physical-sub-stream AC-3 channel counts read off the VOB
|
||||
/// during the mux-time sub-stream probe (the Silence-of-the-Lambs wrong-stream
|
||||
/// fix). This is the ground truth the IFO nibble is compared against: each row
|
||||
/// is `sub_id=0x8x channels=N` for a physical `private_stream_1` AC-3 sub-stream
|
||||
/// whose first frame was decoded. An empty probe (scrambled / unreadable / short
|
||||
/// VOB) logs a single `probed=0` line so the absence is explicit in a bug log.
|
||||
///
|
||||
/// Self-sufficiency: with `tag=dvd.aattr` (the IFO's declared sub_id + claimed
|
||||
/// channels) and these `tag=dvd.substream` rows (the physical reality), a bug
|
||||
/// log alone shows whether the ordinal `0x80` actually carries the declared
|
||||
/// channel layout — no disc needed to diagnose a wrong-substream rip.
|
||||
pub fn dump_dvd_substream_probe(title_id: u16, probed: &std::collections::BTreeMap<u8, u8>) {
|
||||
if !tracing::enabled!(target: DIAG, tracing::Level::DEBUG) {
|
||||
return;
|
||||
}
|
||||
if probed.is_empty() {
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=dvd.substream title={title_id} probed=0 (no AC-3 sync in feature head — scrambled/unreadable/none)",
|
||||
);
|
||||
return;
|
||||
}
|
||||
for (sub, ch) in probed {
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=dvd.substream title={title_id} sub_id=0x{sub:02X} channels={ch} (physical acmod read from VOB)",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── MKV TrackEntry dump (the ACTUAL container elements written) ──────────────
|
||||
|
||||
/// `true` when the `--log-level 3` diagnostic target is enabled. Hot-path
|
||||
/// callers (the opening-frame capture) check this once and skip all work when
|
||||
/// off, so a normal run pays nothing.
|
||||
pub fn diag_enabled() -> bool {
|
||||
tracing::enabled!(target: DIAG, tracing::Level::DEBUG)
|
||||
}
|
||||
|
||||
/// Cap on the number of codecPrivate bytes rendered to hex in a `tag=mkv.track`
|
||||
/// line. The sequence header / avcC / hvcC prefix that matters for diagnosis
|
||||
/// (resolution, frame rate, profile) is at the front; a multi-KB blob past this
|
||||
/// is summarised as `..(+NB)` rather than flooding the log.
|
||||
const CODEC_PRIVATE_HEX_CAP: usize = 64;
|
||||
|
||||
/// Render a track's codecPrivate as an uppercase-hex string for the diagnostic
|
||||
/// line, capped at [`CODEC_PRIVATE_HEX_CAP`] bytes (`..(+NB)` suffix beyond).
|
||||
/// `None` / empty → `"none"`. Pure (no logging) so it is directly unit-testable.
|
||||
fn codec_private_hex(cp: Option<&[u8]>) -> String {
|
||||
match cp {
|
||||
Some(b) if !b.is_empty() => {
|
||||
use std::fmt::Write;
|
||||
let shown = b.len().min(CODEC_PRIVATE_HEX_CAP);
|
||||
let mut s = String::with_capacity(shown * 2 + 8);
|
||||
for byte in &b[..shown] {
|
||||
let _ = write!(s, "{byte:02X}");
|
||||
}
|
||||
if b.len() > CODEC_PRIVATE_HEX_CAP {
|
||||
let _ = write!(s, "..(+{}B)", b.len() - CODEC_PRIVATE_HEX_CAP);
|
||||
}
|
||||
s
|
||||
}
|
||||
_ => "none".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Frame the raw bytes of one captured opening frame for the `.opening.bin` side
|
||||
/// file: `[track:u8][keyframe:u8][pts_ns:i64 LE][len:u32 LE][raw bytes]`. Pure
|
||||
/// (no I/O) so the record layout is directly unit-testable; `record` appends the
|
||||
/// returned bytes to the side file.
|
||||
fn frame_record(track_idx: usize, pts_ns: i64, keyframe: bool, data: &[u8]) -> Vec<u8> {
|
||||
let mut rec = Vec::with_capacity(14 + data.len());
|
||||
rec.push(track_idx as u8);
|
||||
rec.push(keyframe as u8);
|
||||
rec.extend_from_slice(&pts_ns.to_le_bytes());
|
||||
rec.extend_from_slice(&(data.len() as u32).to_le_bytes());
|
||||
rec.extend_from_slice(data);
|
||||
rec
|
||||
}
|
||||
|
||||
/// Emit the MKV `TrackEntry` elements the muxer is about to WRITE for one
|
||||
/// track — the Windows-fps-class metadata (FlagInterlaced, FieldOrder,
|
||||
/// DefaultDuration, DefaultDecodedFieldDuration, Display dims) plus the
|
||||
/// codecPrivate as hex. With this row a bug log alone is enough to verify why
|
||||
/// Windows Explorer reports a given frame rate for an interlaced SD track: the
|
||||
/// container values that drive its fps derivation are all present, no disc and
|
||||
/// no MediaInfo needed.
|
||||
///
|
||||
/// `track_number` is the 1-based MKV track number; `track` is the built
|
||||
/// [`crate::mux::mkv::MkvTrack`] whose fields map one-to-one onto the emitted
|
||||
/// elements (see `MkvMuxer::new`). No-op unless the diag target is on.
|
||||
pub fn dump_mkv_track(track_number: u64, track: &crate::mux::mkv::MkvTrack) {
|
||||
if !diag_enabled() {
|
||||
return;
|
||||
}
|
||||
// codecPrivate as hex (capped so a multi-KB hvcC doesn't flood the log; the
|
||||
// sequence header / avcC prefix that matters for diagnosis is at the front).
|
||||
let cp = codec_private_hex(track.codec_private.as_deref());
|
||||
let field_order = match track.field_order {
|
||||
crate::mux::ebml::FIELD_ORDER_TFF => "TFF",
|
||||
crate::mux::ebml::FIELD_ORDER_BFF => "BFF",
|
||||
_ => "—",
|
||||
};
|
||||
// FlagInterlaced is only written for video tracks (1=interlaced/2=progressive);
|
||||
// report what the muxer will emit, or "—" for non-video tracks where the
|
||||
// element is omitted entirely.
|
||||
let interlaced = if track.track_type == crate::mux::ebml::TRACK_TYPE_VIDEO {
|
||||
if track.interlaced {
|
||||
"1(interlaced)"
|
||||
} else {
|
||||
"2(progressive)"
|
||||
}
|
||||
} else {
|
||||
"—"
|
||||
};
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=mkv.track num={track_number} type={} codec={} flag_interlaced={interlaced} \
|
||||
field_order={field_order} default_duration_ns={} field_duration_ns={} \
|
||||
pixel={}x{} display={}x{} cp_len={} cp_hex={cp}",
|
||||
track.track_type,
|
||||
track.codec_id,
|
||||
track.default_duration_ns,
|
||||
track.field_duration_ns,
|
||||
track.pixel_width,
|
||||
track.pixel_height,
|
||||
track.display_width,
|
||||
track.display_height,
|
||||
track.codec_private.as_ref().map_or(0, |b| b.len()),
|
||||
);
|
||||
}
|
||||
|
||||
// ── Opening-frame capture (first ~N coded frames per track → side file) ──────
|
||||
|
||||
/// Number of coded frames captured PER TRACK before the capture goes dormant.
|
||||
/// ~100 frames covers a DVD's first few seconds of every track (the
|
||||
/// opening-GOP / still-frame / menu window where mid-GOP open or PTS-floor bugs
|
||||
/// show up) while bounding the side file to a few MB even for HD I-frames.
|
||||
const OPENING_FRAMES_PER_TRACK: usize = 100;
|
||||
|
||||
/// Captures the first [`OPENING_FRAMES_PER_TRACK`] coded frames of EACH track to
|
||||
/// a side file (`<output>.opening.bin`) and logs a per-frame summary line, so an
|
||||
/// opening-GOP / menu / mid-GOP-open issue is diagnosable from a future log +
|
||||
/// side file WITHOUT the disc. Gated to `--log-level 3`: constructed only when
|
||||
/// the diag target is on, so a normal run never opens the file or records a byte.
|
||||
///
|
||||
/// Side-file record framing (so a reader can split it back into frames):
|
||||
/// `[track:u8][keyframe:u8][pts_ns:i64 LE][len:u32 LE][raw frame bytes]`.
|
||||
pub struct OpeningCapture {
|
||||
file: std::fs::File,
|
||||
/// Frames captured so far, per track index. Capture for a track stops once
|
||||
/// its counter reaches [`OPENING_FRAMES_PER_TRACK`].
|
||||
counts: Vec<usize>,
|
||||
}
|
||||
|
||||
impl OpeningCapture {
|
||||
/// Open `<output>.opening.bin` next to the MKV output. Returns `None` (no
|
||||
/// capture) when the diag target is off OR the side file can't be created —
|
||||
/// a diagnostic must never fail the rip. `track_count` sizes the per-track
|
||||
/// counters.
|
||||
pub fn new(output_path: &std::path::Path, track_count: usize) -> Option<Self> {
|
||||
if !diag_enabled() {
|
||||
return None;
|
||||
}
|
||||
let mut name = output_path.as_os_str().to_os_string();
|
||||
name.push(".opening.bin");
|
||||
match std::fs::File::create(&name) {
|
||||
Ok(file) => {
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=mkv.opening.open path={:?} per_track_cap={OPENING_FRAMES_PER_TRACK}",
|
||||
std::path::Path::new(&name),
|
||||
);
|
||||
Some(Self {
|
||||
file,
|
||||
counts: vec![0; track_count],
|
||||
})
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=mkv.opening.open path={:?} failed={e} (capture disabled, rip unaffected)",
|
||||
std::path::Path::new(&name),
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Record one coded frame for `track_idx` if that track is still under its
|
||||
/// per-track cap. Writes the framed raw bytes to the side file and logs a
|
||||
/// one-line summary. A write error disables further capture for the track
|
||||
/// (counter pinned to the cap) but never propagates — the rip is unaffected.
|
||||
pub fn record(&mut self, track_idx: usize, pts_ns: i64, keyframe: bool, data: &[u8]) {
|
||||
let Some(count) = self.counts.get_mut(track_idx) else {
|
||||
return;
|
||||
};
|
||||
if *count >= OPENING_FRAMES_PER_TRACK {
|
||||
return;
|
||||
}
|
||||
use std::io::Write;
|
||||
let rec = frame_record(track_idx, pts_ns, keyframe, data);
|
||||
if let Err(e) = self.file.write_all(&rec) {
|
||||
// Stop trying on this track; a broken side file must not stall mux.
|
||||
*count = OPENING_FRAMES_PER_TRACK;
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=mkv.opening.frame track={track_idx} write_failed={e} (capture stopped for track)",
|
||||
);
|
||||
return;
|
||||
}
|
||||
*count += 1;
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=mkv.opening.frame track={track_idx} n={count} type={} size={} pts_ns={pts_ns}",
|
||||
if keyframe { "key" } else { "delta" },
|
||||
data.len(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Disc-level dump (post-lowering: titles, streams, decisions, AACS) ────────
|
||||
|
||||
/// Emit the full scan diagnostic block for a built [`Disc`]. Terse, one line
|
||||
/// per row, under target `freemkv::diag` at DEBUG. No-op unless that target
|
||||
/// is enabled, so it costs nothing when `--log-level 3` is off.
|
||||
pub fn dump_disc(disc: &Disc) {
|
||||
if !tracing::enabled!(target: DIAG, tracing::Level::DEBUG) {
|
||||
return;
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=disc vol={:?} format={:?} content={:?} cap_sectors={} layers={} titles={} encrypted={}",
|
||||
disc.volume_id,
|
||||
disc.format,
|
||||
disc.content_format,
|
||||
disc.capacity_sectors,
|
||||
disc.layers,
|
||||
disc.titles.len(),
|
||||
disc.encrypted,
|
||||
);
|
||||
|
||||
dump_aacs(disc);
|
||||
|
||||
for (ti, title) in disc.titles.iter().enumerate() {
|
||||
dump_title(ti, title);
|
||||
}
|
||||
|
||||
// freemkv's top-level DECISION: which title is the main feature.
|
||||
if let Some(main) = disc.titles.first() {
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=decision pick=main_feature title_idx=0 playlist={:?} dur={:.1}s \
|
||||
size={}B clips={} reason=canonical_title_order(fits-disc, fewest-clips, longest, richest-audio)",
|
||||
main.playlist,
|
||||
main.duration_secs,
|
||||
main.size_bytes,
|
||||
main.clips.len(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn dump_aacs(disc: &Disc) {
|
||||
let Some(a) = disc.aacs.as_ref() else {
|
||||
if disc.css.is_some() {
|
||||
tracing::debug!(target: DIAG, "tag=aacs none crypto=CSS(DVD)");
|
||||
} else if disc.encrypted {
|
||||
tracing::debug!(target: DIAG, "tag=aacs none crypto=encrypted-no-keys");
|
||||
} else {
|
||||
tracing::debug!(target: DIAG, "tag=aacs none crypto=clear");
|
||||
}
|
||||
return;
|
||||
};
|
||||
// CPS-unit / unit-key counts: at scan `unit_keys` is empty (keys are
|
||||
// resolved later); the unit-key count is the BE16 in the raw
|
||||
// Unit_Key_RO.inf if captured. Report both: resolved count and raw len.
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=aacs version={} bus_enc={} mkb_version={:?} disc_hash={} key_source={:?} \
|
||||
vuk={} unit_keys_resolved={} uk_ro_bytes={} mkb_bytes={}",
|
||||
a.version,
|
||||
a.bus_encryption,
|
||||
a.mkb_version,
|
||||
a.disc_hash,
|
||||
a.key_source.name(),
|
||||
a.vuk.is_some(),
|
||||
a.unit_keys.len(),
|
||||
a.uk_ro.len(),
|
||||
a.mkb.len(),
|
||||
);
|
||||
}
|
||||
|
||||
fn dump_title(ti: usize, title: &DiscTitle) {
|
||||
let (mut nv, mut na, mut ns) = (0u32, 0u32, 0u32);
|
||||
for s in &title.streams {
|
||||
match s {
|
||||
Stream::Video(_) => nv += 1,
|
||||
Stream::Audio(_) => na += 1,
|
||||
Stream::Subtitle(_) => ns += 1,
|
||||
}
|
||||
}
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=title idx={ti} playlist={:?} id={} dur={:.1}s size={}B clips={} \
|
||||
extents={} chapters={} v={nv} a={na} s={ns} fmt={:?}",
|
||||
title.playlist,
|
||||
title.playlist_id,
|
||||
title.duration_secs,
|
||||
title.size_bytes,
|
||||
title.clips.len(),
|
||||
title.extents.len(),
|
||||
title.chapters.len(),
|
||||
title.content_format,
|
||||
);
|
||||
|
||||
// Per-clip rows (BD: PlayItem/CLPI; DVD has none).
|
||||
for (ci, c) in title.clips.iter().enumerate() {
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=clip title={ti} idx={ci} id={:?} in={} out={} dur={:.1}s src_packets={}",
|
||||
c.clip_id,
|
||||
c.in_time,
|
||||
c.out_time,
|
||||
c.duration_secs,
|
||||
c.source_packets,
|
||||
);
|
||||
}
|
||||
|
||||
// Per-extent rows (the sectors freemkv will actually rip — the bug-4
|
||||
// decision is visible here: leading non-feature cells are already gone).
|
||||
for (ei, e) in title.extents.iter().enumerate() {
|
||||
tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=extent title={ti} idx={ei} start_lba={} sectors={}",
|
||||
e.start_lba,
|
||||
e.sector_count,
|
||||
);
|
||||
}
|
||||
|
||||
// freemkv's per-stream DECISIONS (what the muxer will write).
|
||||
for (si, s) in title.streams.iter().enumerate() {
|
||||
match s {
|
||||
Stream::Video(v) => tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=stream title={ti} idx={si} kind=video pid=0x{:04X} codec={:?} \
|
||||
res={} interlaced={} fps={} std={} color={} hdr={} aspect={:?} secondary={}",
|
||||
v.pid,
|
||||
v.codec,
|
||||
res_str(v.resolution),
|
||||
v.resolution.is_interlaced(),
|
||||
fps_str(v.frame_rate),
|
||||
tv_system_str(v.frame_rate),
|
||||
color_str(v.color_space),
|
||||
hdr_str(v.hdr),
|
||||
v.display_aspect,
|
||||
v.secondary,
|
||||
),
|
||||
Stream::Audio(a) => tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=stream title={ti} idx={si} kind=audio pid=0x{:04X} codec={:?} \
|
||||
channels={}({}) sr={}Hz lang={:?} secondary={}",
|
||||
a.pid,
|
||||
a.codec,
|
||||
a.channels,
|
||||
channel_count(a.channels),
|
||||
sample_rate_hz(a.sample_rate),
|
||||
a.language,
|
||||
a.secondary,
|
||||
),
|
||||
Stream::Subtitle(sub) => tracing::debug!(
|
||||
target: DIAG,
|
||||
"tag=stream title={ti} idx={si} kind=subtitle pid=0x{:04X} codec={:?} \
|
||||
lang={:?} forced={}",
|
||||
sub.pid,
|
||||
sub.codec,
|
||||
sub.language,
|
||||
sub.forced,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn res_str_keeps_interlace_marker() {
|
||||
assert_eq!(res_str(Resolution::R576i), "576i");
|
||||
assert_eq!(res_str(Resolution::R480i), "480i");
|
||||
assert_eq!(res_str(Resolution::R2160p), "2160p");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fps_and_tv_system() {
|
||||
assert_eq!(fps_str(FrameRate::F25), "25");
|
||||
assert_eq!(tv_system_str(FrameRate::F25), "PAL");
|
||||
assert_eq!(fps_str(FrameRate::F29_97), "29.97");
|
||||
assert_eq!(tv_system_str(FrameRate::F29_97), "NTSC");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn color_and_hdr() {
|
||||
assert_eq!(color_str(ColorSpace::Bt470bg), "BT.470BG");
|
||||
assert_eq!(color_str(ColorSpace::Bt2020), "BT.2020");
|
||||
assert_eq!(hdr_str(HdrFormat::Hdr10), "HDR10");
|
||||
assert_eq!(hdr_str(HdrFormat::DolbyVision), "DoVi");
|
||||
assert_eq!(hdr_str(HdrFormat::Sdr), "SDR");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn channel_count_matches_layout() {
|
||||
assert_eq!(channel_count(AudioChannels::Mono), 1);
|
||||
assert_eq!(channel_count(AudioChannels::Stereo), 2);
|
||||
assert_eq!(channel_count(AudioChannels::Surround51), 6);
|
||||
assert_eq!(channel_count(AudioChannels::Surround71), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sample_rate_hz_values() {
|
||||
assert_eq!(sample_rate_hz(SampleRate::S48), 48000);
|
||||
assert_eq!(sample_rate_hz(SampleRate::S96), 96000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_private_hex_renders_caps_and_handles_empty() {
|
||||
// None / empty → "none" (no hex). The Windows-fps diagnosis only needs
|
||||
// the seq-header prefix, so render it but cap long blobs.
|
||||
assert_eq!(codec_private_hex(None), "none");
|
||||
assert_eq!(codec_private_hex(Some(&[])), "none");
|
||||
// Short blob: full uppercase hex, no suffix. An MPEG-2 seq header starts
|
||||
// 00 00 01 B3 — exactly what a reader greps for in a bug log.
|
||||
assert_eq!(
|
||||
codec_private_hex(Some(&[0x00, 0x00, 0x01, 0xB3])),
|
||||
"000001B3"
|
||||
);
|
||||
// Over the cap: first CODEC_PRIVATE_HEX_CAP bytes + a "..(+NB)" summary.
|
||||
let big = vec![0xABu8; CODEC_PRIVATE_HEX_CAP + 5];
|
||||
let s = codec_private_hex(Some(&big));
|
||||
assert!(s.starts_with(&"AB".repeat(CODEC_PRIVATE_HEX_CAP)), "{s}");
|
||||
assert!(s.ends_with("..(+5B)"), "{s}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_record_layout_is_parseable() {
|
||||
// The .opening.bin record framing must round-trip so a future tool can
|
||||
// split the side file back into frames without the disc:
|
||||
// [track:u8][keyframe:u8][pts_ns:i64 LE][len:u32 LE][raw bytes].
|
||||
let data = [0xDEu8, 0xAD, 0xBE, 0xEF];
|
||||
let rec = frame_record(2, -40_000_000, true, &data);
|
||||
assert_eq!(rec.len(), 14 + data.len());
|
||||
assert_eq!(rec[0], 2, "track index");
|
||||
assert_eq!(rec[1], 1, "keyframe flag");
|
||||
assert_eq!(
|
||||
i64::from_le_bytes(rec[2..10].try_into().unwrap()),
|
||||
-40_000_000,
|
||||
"pts_ns survives (signed — opening back-anchor can be negative)"
|
||||
);
|
||||
assert_eq!(
|
||||
u32::from_le_bytes(rec[10..14].try_into().unwrap()),
|
||||
4,
|
||||
"len"
|
||||
);
|
||||
assert_eq!(&rec[14..], &data, "raw frame bytes follow");
|
||||
// A non-keyframe records the flag as 0.
|
||||
let delta = frame_record(0, 0, false, &[]);
|
||||
assert_eq!(delta[1], 0);
|
||||
assert_eq!(u32::from_le_bytes(delta[10..14].try_into().unwrap()), 0);
|
||||
}
|
||||
|
||||
/// The cell row shows the raw category byte (0xNN) beside the decode, and
|
||||
/// the keep/drop verdict. A plain feature cell (0x00) is "keep"; a leading
|
||||
/// secondary-block cell flagged dropped reads "DROP".
|
||||
#[test]
|
||||
fn cell_row_shows_raw_byte_and_verdict() {
|
||||
let plain = crate::ifo::DvdCell {
|
||||
first_sector: 100,
|
||||
last_sector: 199,
|
||||
category: 0x00,
|
||||
duration_secs: 12.5,
|
||||
};
|
||||
let row = dvd_cell_row(0, &plain, false);
|
||||
assert!(row.contains("cat=0x00"), "{row}");
|
||||
assert!(row.contains("block_mode=0"), "{row}");
|
||||
assert!(row.contains("first=100"), "{row}");
|
||||
assert!(row.contains("last=199"), "{row}");
|
||||
assert!(row.contains("dur=12.5s"), "{row}");
|
||||
assert!(row.contains("keep(plain-feature)"), "{row}");
|
||||
assert!(!row.contains("DROP"), "{row}");
|
||||
|
||||
// 0x90 = in-block cell of an angle block (block_mode=2, block_type=1),
|
||||
// shown dropped as a leading secondary piece.
|
||||
let sec = crate::ifo::DvdCell {
|
||||
first_sector: 0,
|
||||
last_sector: 9,
|
||||
category: 0x90,
|
||||
duration_secs: 1.0,
|
||||
};
|
||||
let row = dvd_cell_row(0, &sec, true);
|
||||
assert!(row.contains("cat=0x90"), "{row}");
|
||||
assert!(row.contains("block_mode=2"), "{row}");
|
||||
assert!(row.contains("block_type=1"), "{row}");
|
||||
assert!(row.contains("DROP(leading-secondary-block-piece)"), "{row}");
|
||||
}
|
||||
}
|
||||
+1386
-31
File diff suppressed because it is too large
Load Diff
+1206
-17
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,466 @@
|
||||
//! Physical AC-3 sub-stream probing for DVD audio routing.
|
||||
//!
|
||||
//! ## Why this exists (Silence-of-the-Lambs wrong-substream bug)
|
||||
//!
|
||||
//! A DVD VTS IFO declares its audio streams in a fixed table, and freemkv's
|
||||
//! scan assigns each declared stream a `private_stream_1` sub-stream id purely
|
||||
//! by per-codec ordinal — the first AC-3 stream becomes `0x80`, the second
|
||||
//! `0x81`, and so on (`ifo::assign_audio_sub_stream_ids`). That assumes the
|
||||
//! physical sub-stream order on the wire matches the IFO declaration order.
|
||||
//!
|
||||
//! On some discs it does NOT. The R2 PAL "The Silence of the Lambs" feature
|
||||
//! declares ONE AC-3 audio stream the IFO nibble marks as 5.1 (6 channels), but
|
||||
//! the physical VOB carries the 5.1 main mix and a 2.0 down-mix on DIFFERENT
|
||||
//! `0x8x` sub-stream ids, and the 2.0 is the one that happens to land at the
|
||||
//! ordinal `0x80` slot. Routing the declared 5.1 stream to `0x80` by ordinal
|
||||
//! therefore muxes the 2.0 down-mix while labelling it 5.1 — the wrong physical
|
||||
//! track.
|
||||
//!
|
||||
//! The robust fix is data-driven and codec/disc agnostic: read each physical
|
||||
//! AC-3 sub-stream's REAL channel count from the VOB (the `acmod`/`lfeon` of its
|
||||
//! first frame after the `0x0B77` sync) and route each IFO-declared AC-3 stream
|
||||
//! to the physical sub-stream whose actual channel count matches the IFO's
|
||||
//! declared count — instead of trusting the ordinal. This never re-reads the
|
||||
//! disc beyond a bounded head-of-feature probe and degrades to the original
|
||||
//! ordinal mapping when the probe yields nothing (unreadable/short VOB).
|
||||
|
||||
use crate::disc::Stream;
|
||||
use crate::mux::codec::ac3;
|
||||
use crate::mux::ps::PsDemuxer;
|
||||
use crate::sector::SectorSource;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// How many 2048-byte sectors of the first feature extent to probe. The head of
|
||||
/// a DVD feature opens with logos/warnings whose audio is frequently a thin 2.0
|
||||
/// bed on the FIRST sub-stream only — the other physical `0x8x` sub-streams and
|
||||
/// the main 5.1 mix do not appear until a sector or two further in. 512 sectors
|
||||
/// (1 MiB) was too short: on Greenland it saw ONLY `0x80`, and only its opening
|
||||
/// 2.0 frames. 1024 sectors (2 MiB) reliably contains at least one frame of
|
||||
/// every physical AC-3 sub-stream AND enough of `0x80` to reach its 5.1 frames.
|
||||
/// Still bounded so a live drive is never hammered (see the project "don't
|
||||
/// hammer the live drive" rule).
|
||||
const PROBE_SECTORS: u16 = 1024;
|
||||
|
||||
/// Decode the real per-sub-stream AC-3 channel count from a buffer of decrypted
|
||||
/// MPEG-PS (DVD VOB) bytes.
|
||||
///
|
||||
/// Demuxes `private_stream_1` (0xBD), and for each AC-3 sub-stream id
|
||||
/// (`0x80..=0x87`) records the MAXIMUM channel count seen across EVERY decodable
|
||||
/// frame in the probe window (`acmod` + `lfeon` at each `0x0B77` sync). Pure and
|
||||
/// unit-testable — takes the already-read bytes, never touches the disc.
|
||||
///
|
||||
/// ## Why the maximum, not the first frame
|
||||
///
|
||||
/// The first frame of a sub-stream at the head of a feature is NOT
|
||||
/// representative. A DVD opens with logos/warnings, and the main `0x80`
|
||||
/// sub-stream there frequently carries a thin 2.0 bed before transitioning to
|
||||
/// its real 5.1 main mix a fraction of a second later (observed on Greenland:
|
||||
/// `0x80`'s first frames are acmod=2 → 2 channels, then it becomes acmod=7+lfe →
|
||||
/// 6 channels within the same 2 MiB window). Recording only the FIRST frame read
|
||||
/// `0x80=2` and missed the 5.1 entirely, defeating the channel-match routing.
|
||||
/// The 5.1 capability of a sub-stream is the *maximum* channel count any of its
|
||||
/// frames carries, so we scan them all and keep the max.
|
||||
///
|
||||
/// Returns a map `sub_id -> max channels`. Sub-streams whose frames are all too
|
||||
/// short to carry the BSI bits, or that never appear in the buffer, are absent
|
||||
/// from the map.
|
||||
pub fn probe_ac3_substream_channels(ps_bytes: &[u8]) -> BTreeMap<u8, u8> {
|
||||
let mut found: BTreeMap<u8, u8> = BTreeMap::new();
|
||||
let mut demux = PsDemuxer::new();
|
||||
let mut packets = demux.feed(ps_bytes);
|
||||
packets.extend(demux.flush());
|
||||
for p in packets {
|
||||
// Only private_stream_1 AC-3 sub-streams (0x80..=0x87).
|
||||
let Some(sub) = p.sub_stream_id else { continue };
|
||||
if !(0x80..=0x87).contains(&sub) {
|
||||
continue;
|
||||
}
|
||||
// The PS demux strips the 4-byte AC-3 sub-header but does not align to a
|
||||
// frame. Walk EVERY 0x0B77 sync in this sub-stream's payload, decode
|
||||
// each frame's channel count, and keep the largest — the sub-stream's
|
||||
// real (main-mix) channel capability. See the doc comment above for why
|
||||
// the first frame alone is unreliable.
|
||||
if let Some(ch) = max_substream_channels(&p.data) {
|
||||
let slot = found.entry(sub).or_insert(0);
|
||||
*slot = (*slot).max(ch);
|
||||
}
|
||||
}
|
||||
found
|
||||
}
|
||||
|
||||
/// Largest AC-3 channel count over every decodable frame in a single
|
||||
/// sub-stream's payload. Returns `None` when no frame carries enough BSI bits.
|
||||
///
|
||||
/// Each frame is advanced by its real `ac3_frame_size` so a frame's compressed
|
||||
/// body (which can contain stray `0x0B77` byte pairs) cannot be mistaken for a
|
||||
/// new frame; only when a size is unmappable do we fall back to a +2 byte
|
||||
/// rescan to re-lock the next genuine sync.
|
||||
fn max_substream_channels(data: &[u8]) -> Option<u8> {
|
||||
let mut best: Option<u8> = None;
|
||||
let mut pos = 0;
|
||||
while pos < data.len() {
|
||||
let Some(rel) = ac3::find_ac3_sync(&data[pos..]) else {
|
||||
break;
|
||||
};
|
||||
let start = pos + rel;
|
||||
let frame = &data[start..];
|
||||
if let Some(ch) = ac3::acmod_channels(frame) {
|
||||
if ch > 0 {
|
||||
best = Some(best.map_or(ch, |b| b.max(ch)));
|
||||
}
|
||||
}
|
||||
// Advance past this frame by its declared size when that is mappable;
|
||||
// otherwise step 2 bytes past the sync and re-scan for the next one.
|
||||
let size = ac3::ac3_frame_size(frame);
|
||||
pos = if (6..=8192).contains(&size) {
|
||||
start + size
|
||||
} else {
|
||||
start + 2
|
||||
};
|
||||
}
|
||||
best
|
||||
}
|
||||
|
||||
/// Re-route the title's declared AC-3 audio streams onto the physical
|
||||
/// sub-stream ids whose REAL channel counts match, using a probed
|
||||
/// `sub_id -> channels` map.
|
||||
///
|
||||
/// For each declared AC-3 audio stream (in IFO order), it picks the physical
|
||||
/// `0x8x` sub-stream whose probed channel count equals the stream's declared
|
||||
/// channel count, never re-using a sub-stream already claimed by an earlier
|
||||
/// stream. The chosen sub-stream's PID (`0xBD00 | sub_id`) is written back onto
|
||||
/// the `Stream::Audio` so BOTH mux demux paths (`DiscStream` and the file-backed
|
||||
/// highway) route by it.
|
||||
///
|
||||
/// Conservative — it only ever REASSIGNS among the physical sub-streams the
|
||||
/// probe actually saw, and only when a better (exact-channel) match exists than
|
||||
/// the stream's current assignment. A stream whose current sub-stream already
|
||||
/// matches is left alone; a stream with no matching physical sub-stream keeps
|
||||
/// its ordinal assignment. So a normal disc (physical order == IFO order) is a
|
||||
/// no-op.
|
||||
///
|
||||
/// Returns the number of streams whose PID was changed (for diagnostics).
|
||||
pub fn remap_audio_pids(streams: &mut [Stream], probed: &BTreeMap<u8, u8>) -> usize {
|
||||
if probed.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
// Sub-streams already claimed by a remapped (or matching) earlier stream,
|
||||
// so two declared streams never collide on one physical sub-stream.
|
||||
let mut claimed: Vec<u8> = Vec::new();
|
||||
let mut changed = 0usize;
|
||||
|
||||
for s in streams.iter_mut() {
|
||||
let Stream::Audio(a) = s else { continue };
|
||||
if a.codec != crate::disc::Codec::Ac3 {
|
||||
continue;
|
||||
}
|
||||
let declared = a.channels.count();
|
||||
// The sub-id this stream currently routes by (low byte of its PID).
|
||||
let current_sub = (a.pid & 0x00FF) as u8;
|
||||
|
||||
// If the stream's current physical sub-stream already matches its
|
||||
// declared channel count, keep it and claim it.
|
||||
if probed.get(¤t_sub) == Some(&declared) {
|
||||
claimed.push(current_sub);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Otherwise find an unclaimed physical sub-stream whose REAL channel
|
||||
// count equals the declared count.
|
||||
let pick = probed
|
||||
.iter()
|
||||
.find(|(sub, ch)| **ch == declared && !claimed.contains(*sub))
|
||||
.map(|(sub, _)| *sub);
|
||||
|
||||
if let Some(sub) = pick {
|
||||
let new_pid = 0xBD00 | sub as u16;
|
||||
if new_pid != a.pid {
|
||||
tracing::debug!(
|
||||
target: "freemkv::scan",
|
||||
old_pid = a.pid,
|
||||
new_pid,
|
||||
declared_channels = declared,
|
||||
"dvd: re-routed AC-3 audio to physical sub-stream matching channel count"
|
||||
);
|
||||
a.pid = new_pid;
|
||||
changed += 1;
|
||||
}
|
||||
claimed.push(sub);
|
||||
} else {
|
||||
// No physical match — leave the ordinal assignment, but claim its
|
||||
// current sub so later streams don't steal a slot it may still use.
|
||||
claimed.push(current_sub);
|
||||
}
|
||||
}
|
||||
changed
|
||||
}
|
||||
|
||||
/// Probe the first feature extent of a DVD title through a (decrypted) sector
|
||||
/// source and re-route its AC-3 audio PIDs to the physically-correct
|
||||
/// sub-streams. A bounded, best-effort scan: any read error or empty probe
|
||||
/// leaves the ordinal assignment untouched.
|
||||
///
|
||||
/// `reader` MUST yield PLAINTEXT VOB bytes (i.e. a `DecryptingSectorSource` on a
|
||||
/// CSS disc) — probing scrambled sectors yields no AC-3 syncs and is a safe
|
||||
/// no-op. Returns the number of audio streams whose PID changed.
|
||||
pub fn probe_and_remap<S: SectorSource + ?Sized>(
|
||||
reader: &mut S,
|
||||
title: &mut crate::disc::DiscTitle,
|
||||
) {
|
||||
// Only DVD (MPEG-PS) titles carry private_stream_1 AC-3 sub-streams.
|
||||
if title.content_format != crate::disc::ContentFormat::MpegPs {
|
||||
return;
|
||||
}
|
||||
// Nothing to disambiguate unless there is at least one AC-3 audio stream.
|
||||
let has_ac3 = title
|
||||
.streams
|
||||
.iter()
|
||||
.any(|s| matches!(s, Stream::Audio(a) if a.codec == crate::disc::Codec::Ac3));
|
||||
if !has_ac3 {
|
||||
return;
|
||||
}
|
||||
let Some(ext) = title.extents.first() else {
|
||||
return;
|
||||
};
|
||||
let count: u16 = ext.sector_count.min(PROBE_SECTORS as u32) as u16;
|
||||
if count == 0 {
|
||||
return;
|
||||
}
|
||||
let mut buf = vec![0u8; count as usize * 2048];
|
||||
// `recovery=false`: a single best-effort attempt — the probe must never
|
||||
// stall the mux or hammer a marginal drive. On any error, bail to ordinal.
|
||||
let n = match reader.read_sectors(ext.start_lba, count, &mut buf, false) {
|
||||
Ok(n) => n,
|
||||
Err(_) => return,
|
||||
};
|
||||
buf.truncate(n);
|
||||
let probed = probe_ac3_substream_channels(&buf);
|
||||
crate::diag::dump_dvd_substream_probe(title.playlist_id, &probed);
|
||||
remap_audio_pids(&mut title.streams, &probed);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::disc::{AudioChannels, AudioStream, Codec, LabelPurpose, SampleRate};
|
||||
|
||||
/// Build a single, correctly-SIZED AC-3 frame whose `acmod`/`lfeon` encode a
|
||||
/// known channel count. `byte4` is `fscod=0 | frmsizecod=0`, so
|
||||
/// `ac3_frame_size` reports 128 bytes and the frame is zero-padded to exactly
|
||||
/// that — this lets `max_substream_channels` advance frame-by-frame over a
|
||||
/// multi-frame payload exactly as it does on real VOB data. The BSI bits are
|
||||
/// laid down with a writer so the test never hand-miscomputes the lfeon
|
||||
/// offset, matching `acmod_channels`' reader.
|
||||
fn ac3_frame(acmod: u8, lfeon: bool) -> Vec<u8> {
|
||||
let mut bits: Vec<u8> = Vec::new();
|
||||
let push = |val: u32, n: usize, bits: &mut Vec<u8>| {
|
||||
for i in (0..n).rev() {
|
||||
bits.push(((val >> i) & 1) as u8);
|
||||
}
|
||||
};
|
||||
push(acmod as u32, 3, &mut bits);
|
||||
if (acmod & 0x1) != 0 && acmod != 0x1 {
|
||||
push(0, 2, &mut bits); // cmixlev
|
||||
}
|
||||
if (acmod & 0x4) != 0 {
|
||||
push(0, 2, &mut bits); // surmixlev
|
||||
}
|
||||
if acmod == 0x2 {
|
||||
push(0, 2, &mut bits); // dsurmod
|
||||
}
|
||||
push(lfeon as u32, 1, &mut bits);
|
||||
// Pack the bit vector MSB-first into bytes (byte6 onward).
|
||||
let mut tail = Vec::new();
|
||||
let mut cur = 0u8;
|
||||
for (i, b) in bits.iter().enumerate() {
|
||||
cur = (cur << 1) | b;
|
||||
if i % 8 == 7 {
|
||||
tail.push(cur);
|
||||
cur = 0;
|
||||
}
|
||||
}
|
||||
let rem = bits.len() % 8;
|
||||
if rem != 0 {
|
||||
cur <<= 8 - rem;
|
||||
tail.push(cur);
|
||||
}
|
||||
// AC-3 frame: 0x0B 0x77 crc(2) byte4(fscod=0,frmsizecod=0) bsid<<3 then BSI.
|
||||
let mut frame = vec![0x0B, 0x77, 0x00, 0x00, 0x00, 8u8 << 3];
|
||||
frame.extend_from_slice(&tail);
|
||||
// frmsizecod=0 @ 48kHz → 64 words = 128 bytes. Pad to the real size so
|
||||
// the frame-stepping in max_substream_channels lands on the next sync.
|
||||
frame.resize(128, 0);
|
||||
frame
|
||||
}
|
||||
|
||||
/// Build a minimal `private_stream_1` PES carrying `frames` for `sub_id`,
|
||||
/// each preceded only by the 4-byte AC-3 sub-header at the PES head. Mirrors
|
||||
/// the on-disc layout the PS demux expects: PES start `0x000001BD`, length,
|
||||
/// PES header (no PTS), sub-header `[sub_id, frame_count, ptr_hi, ptr_lo]`,
|
||||
/// then the concatenated AC-3 frames.
|
||||
fn ps_ac3_frames(sub_id: u8, frames: &[Vec<u8>]) -> Vec<u8> {
|
||||
// PES sub-header for AC-3: sub_id + frame_count + 2-byte access ptr.
|
||||
let mut payload = vec![sub_id, frames.len() as u8, 0x00, 0x04];
|
||||
for f in frames {
|
||||
payload.extend_from_slice(f);
|
||||
}
|
||||
// PES packet: start code 00 00 01 BD, length(2), flags(2), hdr_len(0).
|
||||
let pes_payload_len = 3 + payload.len(); // flags(2)+hdrlen(1)+payload
|
||||
let mut pkt = vec![0x00, 0x00, 0x01, 0xBD];
|
||||
pkt.extend_from_slice(&(pes_payload_len as u16).to_be_bytes());
|
||||
pkt.extend_from_slice(&[0x80, 0x00, 0x00]); // no PTS, header_data_len=0
|
||||
pkt.extend_from_slice(&payload);
|
||||
pkt
|
||||
}
|
||||
|
||||
/// Single-frame `private_stream_1` PES — the common case in existing tests.
|
||||
fn ps_ac3(sub_id: u8, acmod: u8, lfeon: bool) -> Vec<u8> {
|
||||
ps_ac3_frames(sub_id, &[ac3_frame(acmod, lfeon)])
|
||||
}
|
||||
|
||||
fn ac3_stream(pid: u16, channels: AudioChannels) -> Stream {
|
||||
Stream::Audio(AudioStream {
|
||||
pid,
|
||||
codec: Codec::Ac3,
|
||||
channels,
|
||||
language: "en".into(),
|
||||
sample_rate: SampleRate::S48,
|
||||
secondary: false,
|
||||
purpose: LabelPurpose::Normal,
|
||||
label: String::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The probe decodes the real channel count of each physical sub-stream.
|
||||
/// 0x80 carries a 2.0 frame (acmod=2,no lfe → 2ch); 0x81 carries 5.1
|
||||
/// (acmod=7 + lfe → 6ch).
|
||||
#[test]
|
||||
fn probe_decodes_per_substream_channels() {
|
||||
let mut bytes = ps_ac3(0x80, 2, false);
|
||||
bytes.extend(ps_ac3(0x81, 7, true));
|
||||
let probed = probe_ac3_substream_channels(&bytes);
|
||||
assert_eq!(probed.get(&0x80), Some(&2), "0x80 is the 2.0 down-mix");
|
||||
assert_eq!(probed.get(&0x81), Some(&6), "0x81 is the 5.1 main mix");
|
||||
}
|
||||
|
||||
/// GREENLAND regression — the probe must read each sub-stream's TRUE
|
||||
/// (max-mix) channel count, not be poisoned by an unrepresentative head
|
||||
/// frame, and must NOT cross-contaminate between sub-streams.
|
||||
///
|
||||
/// Mirrors the real on-disc layout that caused the mis-read: the feature
|
||||
/// head carries `0x80` opening with a 2.0 frame and THEN a 5.1 frame (its
|
||||
/// real main mix), interleaved with `0x81` carrying only 2.0. The old
|
||||
/// first-frame probe read `0x80=2` (the logo bed) and missed the 5.1; the
|
||||
/// max-over-frames probe must report `0x80=6` and `0x81=2`.
|
||||
#[test]
|
||||
fn probe_reads_max_channels_no_cross_contamination() {
|
||||
let mut bytes = Vec::new();
|
||||
// 0x80 opens with a 2.0 frame (the logo bed)...
|
||||
bytes.extend(ps_ac3_frames(0x80, &[ac3_frame(2, false)]));
|
||||
// ...0x81 interleaves a pure-2.0 PES (must NOT bleed 6 into 0x80)...
|
||||
bytes.extend(ps_ac3_frames(
|
||||
0x81,
|
||||
&[ac3_frame(2, false), ac3_frame(2, false)],
|
||||
));
|
||||
// ...then 0x80 reaches its real 5.1 main mix (acmod=7 + lfe → 6 ch),
|
||||
// with a trailing 2.0 frame in the SAME PES to prove we take the max,
|
||||
// not the last frame.
|
||||
bytes.extend(ps_ac3_frames(
|
||||
0x80,
|
||||
&[ac3_frame(7, true), ac3_frame(2, false)],
|
||||
));
|
||||
|
||||
let probed = probe_ac3_substream_channels(&bytes);
|
||||
assert_eq!(
|
||||
probed.get(&0x80),
|
||||
Some(&6),
|
||||
"0x80's real 5.1 mix must win over its 2.0 head/tail frames"
|
||||
);
|
||||
assert_eq!(
|
||||
probed.get(&0x81),
|
||||
Some(&2),
|
||||
"0x81 is a pure 2.0 stream — must not absorb 0x80's 6-channel frame"
|
||||
);
|
||||
}
|
||||
|
||||
/// SILENCE-OF-THE-LAMBS regression: the IFO declares ONE 5.1 AC-3 stream and
|
||||
/// the ordinal mapping put it at 0x80, but physically 0x80 is the 2.0
|
||||
/// down-mix and the 5.1 lives at 0x81. After probe+remap the declared 5.1
|
||||
/// stream must route to 0x81 (PID 0xBD81), NOT the ordinal 0x80.
|
||||
#[test]
|
||||
fn remap_routes_declared_51_to_physical_51_substream() {
|
||||
// Physical layout: 0x80 = 2.0, 0x81 = 5.1 (reversed vs ordinal).
|
||||
let mut probed = BTreeMap::new();
|
||||
probed.insert(0x80u8, 2u8);
|
||||
probed.insert(0x81u8, 6u8);
|
||||
|
||||
// Declared: one 5.1 stream, ordinally assigned 0x80 (PID 0xBD80).
|
||||
let mut streams = vec![ac3_stream(0xBD80, AudioChannels::Surround51)];
|
||||
let changed = remap_audio_pids(&mut streams, &probed);
|
||||
assert_eq!(changed, 1, "the one 5.1 stream must be re-routed");
|
||||
let Stream::Audio(a) = &streams[0] else {
|
||||
panic!("audio")
|
||||
};
|
||||
assert_eq!(
|
||||
a.pid, 0xBD81,
|
||||
"declared 5.1 must route to physical 0x81 (the real 5.1), not ordinal 0x80"
|
||||
);
|
||||
}
|
||||
|
||||
/// Conservative no-op: when the physical order already matches the IFO
|
||||
/// order (0x80 = 5.1 as declared), remap changes nothing.
|
||||
#[test]
|
||||
fn remap_noop_when_physical_matches_ordinal() {
|
||||
let mut probed = BTreeMap::new();
|
||||
probed.insert(0x80u8, 6u8); // 0x80 really is the 5.1
|
||||
let mut streams = vec![ac3_stream(0xBD80, AudioChannels::Surround51)];
|
||||
let changed = remap_audio_pids(&mut streams, &probed);
|
||||
assert_eq!(changed, 0, "matching physical order is a no-op");
|
||||
let Stream::Audio(a) = &streams[0] else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(a.pid, 0xBD80);
|
||||
}
|
||||
|
||||
/// Two declared streams (5.1 + 2.0) where the physical order is reversed:
|
||||
/// 0x80=2.0, 0x81=5.1. The 5.1 declaration must claim 0x81 and the 2.0
|
||||
/// declaration must claim 0x80 — no collision, both correct.
|
||||
#[test]
|
||||
fn remap_two_streams_no_collision() {
|
||||
let mut probed = BTreeMap::new();
|
||||
probed.insert(0x80u8, 2u8);
|
||||
probed.insert(0x81u8, 6u8);
|
||||
// Declared order: 5.1 first (ordinal 0x80), 2.0 second (ordinal 0x81).
|
||||
let mut streams = vec![
|
||||
ac3_stream(0xBD80, AudioChannels::Surround51),
|
||||
ac3_stream(0xBD81, AudioChannels::Stereo),
|
||||
];
|
||||
remap_audio_pids(&mut streams, &probed);
|
||||
let pids: Vec<u16> = streams
|
||||
.iter()
|
||||
.filter_map(|s| match s {
|
||||
Stream::Audio(a) => Some(a.pid),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
pids,
|
||||
vec![0xBD81, 0xBD80],
|
||||
"5.1→0x81, 2.0→0x80, no collision"
|
||||
);
|
||||
}
|
||||
|
||||
/// Empty probe (unreadable / scrambled VOB) is a no-op — the ordinal
|
||||
/// assignment survives so behaviour never regresses below today's.
|
||||
#[test]
|
||||
fn remap_empty_probe_is_noop() {
|
||||
let probed = BTreeMap::new();
|
||||
let mut streams = vec![ac3_stream(0xBD80, AudioChannels::Surround51)];
|
||||
let changed = remap_audio_pids(&mut streams, &probed);
|
||||
assert_eq!(changed, 0);
|
||||
let Stream::Audio(a) = &streams[0] else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(a.pid, 0xBD80, "no probe data → keep ordinal");
|
||||
}
|
||||
}
|
||||
+1003
-65
File diff suppressed because it is too large
Load Diff
+1660
File diff suppressed because it is too large
Load Diff
+1173
-24
File diff suppressed because it is too large
Load Diff
+4417
-1494
File diff suppressed because it is too large
Load Diff
+3142
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,226 @@
|
||||
//! `Disc::sweep`'s consumer-side `Sink<WorkItem>`.
|
||||
//!
|
||||
//! Background: the original sweep loop runs strictly serialised —
|
||||
//! SCSI read → decrypt → seek + write → mapfile.record → next iter.
|
||||
//! On a healthy disc the SCSI read costs ~5-12 ms per 64 KB batch and
|
||||
//! the post-read work (decrypt 1-3 ms + file write + mapfile fsync
|
||||
//! 5-15 ms) adds another batch's worth of latency. The drive idles
|
||||
//! during the post-read work; throughput tops out at the *sum* of
|
||||
//! both costs.
|
||||
//!
|
||||
//! A producer/consumer split overlaps the two stages on the generic
|
||||
//! [`crate::io::Pipeline`] + [`crate::io::Sink`] primitive. This module
|
||||
//! is the sweep-specific `Sink` impl; the producer-side state machine
|
||||
//! (read_error context, decrypt, set_speed, halt) stays in
|
||||
//! `Disc::sweep` in `disc/mod.rs`.
|
||||
//!
|
||||
//! Correctness invariants preserved:
|
||||
//! - Mapfile is single-writer (consumer-only). No locking.
|
||||
//! - All `read_error::ReadCtx` state stays on the producer thread.
|
||||
//! - `set_speed` calls happen on the producer thread (same thread that
|
||||
//! owns the `SectorSource`). No new SCSI concurrency.
|
||||
//! - Per-iteration ordering of file-write → mapfile-record is kept
|
||||
//! intact in the consumer (write before record), so the on-disk
|
||||
//! invariant "mapfile only marks Finished what the file has
|
||||
//! received" survives a crash mid-pass.
|
||||
//! - Only one SCSI command is in flight at a time; error-path timing
|
||||
//! is identical and no new retry logic is introduced.
|
||||
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
use std::sync::mpsc::{Receiver, SyncSender, sync_channel};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::io::{Flow, Sink};
|
||||
|
||||
use super::mapfile::{MapStats, Mapfile, SectorStatus};
|
||||
|
||||
/// Reusable zero buffer for SkipFill / GapFill / BisectBad. 64 KB
|
||||
/// matches the existing zero_gap chunk size used by the pre-split
|
||||
/// sweep loop.
|
||||
const ZERO_CHUNK: usize = 64 * 1024;
|
||||
|
||||
/// Producer → Consumer messages. The consumer applies these in FIFO
|
||||
/// order; ordering of file writes and mapfile records across items is
|
||||
/// preserved.
|
||||
pub(super) enum WorkItem {
|
||||
/// Successful batch read. Producer has already decrypted `buf` if
|
||||
/// `opts.decrypt` was set. Consumer writes `buf` at `pos` and
|
||||
/// records the range as `Finished`.
|
||||
Good { pos: u64, buf: Vec<u8> },
|
||||
|
||||
/// Bisect inner-loop good single sector (already decrypted by the
|
||||
/// producer). 2048 bytes.
|
||||
BisectGood { pos: u64, buf: Box<[u8; 2048]> },
|
||||
|
||||
/// Bisect inner-loop bad single sector. Consumer writes 2048
|
||||
/// zeros at `pos` and records the sector as `NonTrimmed`.
|
||||
BisectBad { pos: u64 },
|
||||
|
||||
/// Whole-batch zero-fill (failed batch on `SkipBlock`, or the
|
||||
/// failed batch portion of `JumpAhead`). Consumer streams zeros
|
||||
/// across `[pos, pos+len)` and records the range as `NonTrimmed`.
|
||||
SkipFill { pos: u64, len: u64 },
|
||||
|
||||
/// Gap fill following a `JumpAhead`. Same effect as `SkipFill`;
|
||||
/// distinguished only so future logging / instrumentation can
|
||||
/// tell them apart without parsing a flag.
|
||||
GapFill { pos: u64, len: u64 },
|
||||
|
||||
/// Post-read verify downgrade. The producer's `UnitVerifier` found that the
|
||||
/// just-`Finished` clip unit at `[pos, pos+len)` is confidently undecryptable
|
||||
/// (a silent bad read). The consumer re-records the range as `NonTrimmed` so
|
||||
/// the patch pass re-reads it — the ISO bytes (ciphertext) already written by
|
||||
/// the preceding `Good` are left in place for the patch to overwrite. FIFO
|
||||
/// pipe ordering guarantees this arrives AFTER the `Good` that wrote them.
|
||||
MarkBad { pos: u64, len: u64 },
|
||||
|
||||
/// Producer wants the latest mapfile stats for the progress
|
||||
/// callback. Consumer responds on `prog_tx` with a fresh
|
||||
/// [`ProgressSnapshot`]. Best-effort: if the producer hasn't
|
||||
/// drained the previous snapshot, the new one is silently
|
||||
/// dropped — the producer's local cache stays current enough.
|
||||
StatsRequest,
|
||||
}
|
||||
|
||||
/// Snapshot the consumer sends back to the producer for the progress
|
||||
/// callback.
|
||||
pub(super) struct ProgressSnapshot {
|
||||
pub stats: MapStats,
|
||||
pub bad_ranges: Vec<(u64, u64)>,
|
||||
}
|
||||
|
||||
/// Final summary returned by the consumer thread on shutdown — what
|
||||
/// `SweepSink::close` produces, surfaced to the producer via
|
||||
/// `Pipeline::finish`.
|
||||
pub(super) struct ConsumerSummary {
|
||||
pub stats: MapStats,
|
||||
}
|
||||
|
||||
/// Drain any pending progress snapshots from the consumer. Returns
|
||||
/// the most recent one, if any. The producer caches it and uses it
|
||||
/// for subsequent progress callbacks until a fresh one arrives.
|
||||
pub(super) fn try_recv_progress(rx: &Receiver<ProgressSnapshot>) -> Option<ProgressSnapshot> {
|
||||
let mut latest = None;
|
||||
while let Ok(snap) = rx.try_recv() {
|
||||
latest = Some(snap);
|
||||
}
|
||||
latest
|
||||
}
|
||||
|
||||
/// `Sink<WorkItem>` for sweep. Owns the writeback file + mapfile +
|
||||
/// progress back-channel. `apply` carries the file-write +
|
||||
/// mapfile.record per item; `close` drains the writeback pipeline,
|
||||
/// fsyncs the ISO, and flushes the mapfile.
|
||||
pub(super) struct SweepSink {
|
||||
file: crate::io::WritebackFile,
|
||||
map: Mapfile,
|
||||
/// `sync_all`-on-failure-is-an-error iff the output is a regular
|
||||
/// file. `/dev/null` and pipes always fail `sync_all`; that's not
|
||||
/// a real error.
|
||||
is_regular: bool,
|
||||
/// Back-channel for `StatsRequest` responses. The producer caches
|
||||
/// the latest snapshot and uses it for the progress callback;
|
||||
/// dropped sends on a full channel are by design.
|
||||
prog_tx: SyncSender<ProgressSnapshot>,
|
||||
/// Reusable zero buffer for SkipFill / GapFill / BisectBad. Held
|
||||
/// in the sink so each apply call doesn't reallocate.
|
||||
zero: Box<[u8; ZERO_CHUNK]>,
|
||||
}
|
||||
|
||||
impl SweepSink {
|
||||
/// Construct a new `SweepSink` plus the matching progress
|
||||
/// receiver. Channel depth on the back-channel is `1` — the
|
||||
/// producer's cache is the source of truth between snapshots.
|
||||
pub(super) fn new(
|
||||
file: crate::io::WritebackFile,
|
||||
map: Mapfile,
|
||||
is_regular: bool,
|
||||
) -> (Self, Receiver<ProgressSnapshot>) {
|
||||
let (prog_tx, prog_rx) = sync_channel::<ProgressSnapshot>(1);
|
||||
let sink = SweepSink {
|
||||
file,
|
||||
map,
|
||||
is_regular,
|
||||
prog_tx,
|
||||
zero: Box::new([0u8; ZERO_CHUNK]),
|
||||
};
|
||||
(sink, prog_rx)
|
||||
}
|
||||
}
|
||||
|
||||
impl Sink<WorkItem> for SweepSink {
|
||||
type Output = ConsumerSummary;
|
||||
|
||||
fn apply(&mut self, item: WorkItem) -> Result<Flow, Error> {
|
||||
match item {
|
||||
WorkItem::Good { pos, buf } => {
|
||||
// Decrypt is on the producer; consumer assumes plaintext.
|
||||
let len = buf.len() as u64;
|
||||
self.file.seek(SeekFrom::Start(pos))?;
|
||||
self.file.write_all(&buf)?;
|
||||
self.map.record(pos, len, SectorStatus::Finished)?;
|
||||
}
|
||||
WorkItem::BisectGood { pos, buf } => {
|
||||
self.file.seek(SeekFrom::Start(pos))?;
|
||||
self.file.write_all(&buf[..])?;
|
||||
self.map.record(pos, 2048, SectorStatus::Finished)?;
|
||||
}
|
||||
WorkItem::BisectBad { pos } => {
|
||||
self.file.seek(SeekFrom::Start(pos))?;
|
||||
self.file.write_all(&self.zero[..2048])?;
|
||||
self.map.record(pos, 2048, SectorStatus::NonTrimmed)?;
|
||||
}
|
||||
WorkItem::SkipFill { pos, len } | WorkItem::GapFill { pos, len } => {
|
||||
self.file.seek(SeekFrom::Start(pos))?;
|
||||
// Subsequent writes are sequential; `WritebackFile`'s
|
||||
// seek-elision keeps them on the writeback pipeline path.
|
||||
let mut filled = 0u64;
|
||||
while filled < len {
|
||||
let chunk = (len - filled).min(self.zero.len() as u64) as usize;
|
||||
self.file.write_all(&self.zero[..chunk])?;
|
||||
filled += chunk as u64;
|
||||
}
|
||||
self.map.record(pos, len, SectorStatus::NonTrimmed)?;
|
||||
}
|
||||
WorkItem::MarkBad { pos, len } => {
|
||||
// Verify downgrade: the ISO bytes are already written by the
|
||||
// preceding Good; only the mapfile status changes so patch
|
||||
// re-reads this range. No file write.
|
||||
self.map.record(pos, len, SectorStatus::NonTrimmed)?;
|
||||
}
|
||||
WorkItem::StatsRequest => {
|
||||
let stats = self.map.stats();
|
||||
let bad_ranges = self.map.ranges_with(&[
|
||||
SectorStatus::NonTrimmed,
|
||||
SectorStatus::Unreadable,
|
||||
SectorStatus::NonScraped,
|
||||
SectorStatus::NonTried,
|
||||
]);
|
||||
// Best-effort: drop on backpressure; producer's cache
|
||||
// stays current enough.
|
||||
let _ = self
|
||||
.prog_tx
|
||||
.try_send(ProgressSnapshot { stats, bad_ranges });
|
||||
}
|
||||
}
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(mut self) -> Result<Self::Output, Error> {
|
||||
// Drain the writeback pipeline + fsync the ISO, then persist
|
||||
// any pending mapfile state. Same finalisation order as the
|
||||
// pre-Pipeline consumer loop.
|
||||
if let Err(e) = self.file.sync_all() {
|
||||
if self.is_regular {
|
||||
return Err(Error::IoError { source: e });
|
||||
}
|
||||
// Non-regular outputs (/dev/null, pipes) always fail
|
||||
// sync_all; that's not a real error.
|
||||
}
|
||||
self.map.flush()?;
|
||||
|
||||
Ok(ConsumerSummary {
|
||||
stats: self.map.stats(),
|
||||
})
|
||||
}
|
||||
}
|
||||
+1019
File diff suppressed because it is too large
Load Diff
@@ -25,8 +25,14 @@ pub struct DriveCapture {
|
||||
/// A single GET CONFIGURATION feature response from the drive.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CapturedFeature {
|
||||
/// MMC-6 GET CONFIGURATION feature code (e.g. `0x010D` = AACS).
|
||||
pub code: u16,
|
||||
/// Static human-readable label from the internal `FEATURES` table —
|
||||
/// not a device-reported string.
|
||||
pub name: &'static str,
|
||||
/// Raw feature-descriptor payload bytes, with the 8-byte GET
|
||||
/// CONFIGURATION header stripped (i.e. `buf[8..]`). Unlike
|
||||
/// [`DriveCapture::gc_010c`], which retains the full header.
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
@@ -114,3 +120,77 @@ pub fn mask_bytes(data: &[u8]) -> Vec<u8> {
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
//! Privacy-masking + capture-orchestration tests.
|
||||
//!
|
||||
//! `mask_string` / `mask_bytes` redact identifying characters before
|
||||
//! a drive capture leaves the machine: every ASCII letter → 'A',
|
||||
//! every ASCII digit → '0', everything else (punctuation, spaces,
|
||||
//! control bytes, non-ASCII) is preserved verbatim so structural
|
||||
//! framing (offsets, separators) survives for diffing.
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn mask_string_letters_become_a_digits_become_zero() {
|
||||
// Mixed case letters all collapse to 'A'; digits to '0'.
|
||||
assert_eq!(mask_string("HL-DT-ST"), "AA-AA-AA");
|
||||
assert_eq!(mask_string("BU40N"), "AA00A");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_string_preserves_non_alnum_punctuation_and_space() {
|
||||
// Separators and spaces must be preserved so the masked output
|
||||
// keeps the same shape as the original (the whole point of a
|
||||
// structure-preserving redaction).
|
||||
assert_eq!(mask_string("1.04"), "0.00");
|
||||
assert_eq!(mask_string("a b-c.d_e"), "A A-A.A_A");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_string_preserves_non_ascii_chars() {
|
||||
// is_ascii_alphabetic/is_ascii_digit are false for non-ASCII, so
|
||||
// multibyte chars pass through unchanged (no mojibake, no panic).
|
||||
// 'c','a','f' are ASCII letters → 'A'; 'é' is non-ASCII →
|
||||
// preserved; '9' → '0'.
|
||||
assert_eq!(mask_string("café9"), "AAAé0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_bytes_matches_string_masking_for_ascii() {
|
||||
// mask_bytes is the byte-wise analogue: letters→b'A', digits→b'0'.
|
||||
assert_eq!(mask_bytes(b"HL-DT-ST"), b"AA-AA-AA".to_vec());
|
||||
assert_eq!(mask_bytes(b"1.04"), b"0.00".to_vec());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_bytes_preserves_non_alnum_and_high_bytes() {
|
||||
// Control bytes (0x00), high bytes (0xFF), and punctuation are
|
||||
// not ASCII alnum and must survive verbatim — INQUIRY payloads
|
||||
// are space-padded binary and the framing must be diffable.
|
||||
let input = [0x00u8, b'A', 0x20, b'7', 0xFF, b'-'];
|
||||
assert_eq!(mask_bytes(&input), vec![0x00, b'A', 0x20, b'0', 0xFF, b'-']);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feature_table_has_no_duplicate_codes() {
|
||||
// capture_drive_data iterates FEATURES once per code; a duplicate
|
||||
// code would silently capture the same feature twice (and bloat
|
||||
// the report). Each MMC-6 feature code must be unique.
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for &(code, _name) in FEATURES {
|
||||
assert!(seen.insert(code), "duplicate feature code {code:#06x}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feature_table_includes_aacs_010d() {
|
||||
// AACS (0x010D) is the feature that gates UHD decryption capture;
|
||||
// it must be in the table or AACS drives capture incompletely.
|
||||
assert!(
|
||||
FEATURES.iter().any(|&(c, _)| c == 0x010D),
|
||||
"AACS feature 0x010D must be captured"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+68
-16
@@ -1,18 +1,33 @@
|
||||
//! Linux drive discovery and device resolution.
|
||||
|
||||
use crate::drive::DeviceResolution;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::identity::DriveId;
|
||||
|
||||
/// SCSI peripheral device type 5 = MMC / optical (CD/DVD/BD), held in the
|
||||
/// low 5 bits of INQUIRY byte 0 (the high 3 bits are the peripheral
|
||||
/// qualifier, masked off here).
|
||||
const SCSI_PERIPHERAL_TYPE_OPTICAL: u8 = 0x05;
|
||||
|
||||
/// Discover optical drives by enumerating `/dev/sg*` SCSI-generic nodes,
|
||||
/// opening each, running INQUIRY, and keeping only devices whose
|
||||
/// peripheral device type is optical (MMC, type 0x05).
|
||||
///
|
||||
/// Devices where `scsi::open` or `DriveId::from_drive` fail are silently
|
||||
/// skipped — that is intentional for enumeration (a busy or wedged node
|
||||
/// shouldn't abort discovery of the others).
|
||||
pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
let mut drives = Vec::new();
|
||||
for i in 0..16 {
|
||||
let path = format!("/dev/sg{i}");
|
||||
for name in enumerate_sg_names() {
|
||||
let path = format!("/dev/{name}");
|
||||
if !std::path::Path::new(&path).exists() {
|
||||
continue;
|
||||
}
|
||||
if let Ok(mut transport) = crate::scsi::open(std::path::Path::new(&path)) {
|
||||
if let Ok(id) = DriveId::from_drive(transport.as_mut()) {
|
||||
if !id.raw_inquiry.is_empty() && (id.raw_inquiry[0] & 0x1F) == 0x05 {
|
||||
if !id.raw_inquiry.is_empty()
|
||||
&& (id.raw_inquiry[0] & 0x1F) == SCSI_PERIPHERAL_TYPE_OPTICAL
|
||||
{
|
||||
drives.push((path, id));
|
||||
}
|
||||
}
|
||||
@@ -21,41 +36,78 @@ pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
drives
|
||||
}
|
||||
|
||||
/// Enumerate `sg*` device names. Linux assigns `/dev/sgN` sequentially
|
||||
/// across *all* SCSI-generic devices (disks, tape, HBAs, optical), so a
|
||||
/// fixed `sg0..15` range can miss an optical drive on a host with many
|
||||
/// targets. Prefer the exact present-device list from
|
||||
/// `/sys/class/scsi_generic/`; fall back to a bounded `sg0..15` probe
|
||||
/// only when sysfs is unreadable (minimal containers).
|
||||
fn enumerate_sg_names() -> Vec<String> {
|
||||
let mut names = Vec::new();
|
||||
if let Ok(entries) = std::fs::read_dir("/sys/class/scsi_generic") {
|
||||
for entry in entries.flatten() {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.starts_with("sg") {
|
||||
names.push(name);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for i in 0..16 {
|
||||
let name = format!("sg{i}");
|
||||
if std::path::Path::new(&format!("/dev/{name}")).exists() {
|
||||
names.push(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
/// Resolve a device path to its raw `/dev/sg*` SCSI-generic node.
|
||||
///
|
||||
/// - `/dev/sg*` paths pass through unchanged ([`DeviceResolution::Direct`]).
|
||||
/// - `/dev/sr*` block paths are matched (by vendor/product/serial) to the
|
||||
/// corresponding `/dev/sg*` node ([`DeviceResolution::SrToSg`]); if no
|
||||
/// match is found the original path is returned with
|
||||
/// [`DeviceResolution::SrNoSgMatch`].
|
||||
/// - Any other existing path passes through as [`DeviceResolution::Direct`].
|
||||
#[allow(dead_code)]
|
||||
pub fn resolve_device(path: &str) -> Result<(String, Option<String>)> {
|
||||
pub fn resolve_device(path: &str) -> Result<(String, DeviceResolution)> {
|
||||
if path.contains("/sg") {
|
||||
if !std::path::Path::new(path).exists() {
|
||||
return Err(Error::DeviceNotFound {
|
||||
path: path.to_string(),
|
||||
});
|
||||
}
|
||||
return Ok((path.to_string(), None));
|
||||
return Ok((path.to_string(), DeviceResolution::Direct));
|
||||
}
|
||||
if path.contains("/sr") {
|
||||
let mut sr_transport = crate::scsi::open(std::path::Path::new(path))?;
|
||||
let sr_id = DriveId::from_drive(sr_transport.as_mut())?;
|
||||
drop(sr_transport);
|
||||
for (sg_path, sg_id) in find_drives() {
|
||||
if sg_id.vendor_id == sr_id.vendor_id
|
||||
// Require a non-empty serial before treating vendor/product/
|
||||
// serial as a unique match. serial_number falls back to an
|
||||
// empty string when GET CONFIGURATION 0108h is unavailable
|
||||
// (common on OEM drives); two same-model drives would then
|
||||
// both compare equal and the first in enumeration order would
|
||||
// win silently, resolving sr1 to sr0's sg node. An empty
|
||||
// serial can't disambiguate, so fall through to the no-match
|
||||
// path instead.
|
||||
if !sr_id.serial_number.is_empty()
|
||||
&& sg_id.vendor_id == sr_id.vendor_id
|
||||
&& sg_id.product_id == sr_id.product_id
|
||||
&& sg_id.serial_number == sr_id.serial_number
|
||||
{
|
||||
let warning =
|
||||
format!("{path} is a block device (sr) — using {sg_path} (sg) for raw access");
|
||||
return Ok((sg_path, Some(warning)));
|
||||
return Ok((sg_path, DeviceResolution::SrToSg));
|
||||
}
|
||||
}
|
||||
return Ok((
|
||||
path.to_string(),
|
||||
Some(format!(
|
||||
"{path} is a block device (sr) — no matching sg device found"
|
||||
)),
|
||||
));
|
||||
return Ok((path.to_string(), DeviceResolution::SrNoSgMatch));
|
||||
}
|
||||
if !std::path::Path::new(path).exists() {
|
||||
return Err(Error::DeviceNotFound {
|
||||
path: path.to_string(),
|
||||
});
|
||||
}
|
||||
Ok((path.to_string(), None))
|
||||
Ok((path.to_string(), DeviceResolution::Direct))
|
||||
}
|
||||
|
||||
+22
-12
@@ -4,9 +4,20 @@
|
||||
//! to discover optical drives without exclusive access or unmounts. Only
|
||||
//! the returned paths are then opened for INQUIRY to build full `DriveId`.
|
||||
|
||||
use crate::drive::DeviceResolution;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::identity::DriveId;
|
||||
|
||||
/// SCSI peripheral device type 5 = MMC / optical, in the low 5 bits of
|
||||
/// INQUIRY byte 0.
|
||||
const SCSI_PERIPHERAL_TYPE_OPTICAL: u8 = 0x05;
|
||||
|
||||
/// Discover optical drives via the IOKit registry (`scsi::list_drives`),
|
||||
/// then open each candidate for INQUIRY to build a full `DriveId`.
|
||||
///
|
||||
/// Any drive where `scsi::open` or `DriveId::from_drive` fails, or whose
|
||||
/// peripheral device type is not optical (MMC, type 0x05), is silently
|
||||
/// skipped — the same MMC filter the Linux and Windows backends apply.
|
||||
pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
let mut drives = Vec::new();
|
||||
let discovered = crate::scsi::list_drives();
|
||||
@@ -15,7 +26,11 @@ pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
match crate::scsi::open(path) {
|
||||
Ok(mut transport) => {
|
||||
if let Ok(id) = DriveId::from_drive(transport.as_mut()) {
|
||||
drives.push((info.path.clone(), id));
|
||||
if !id.raw_inquiry.is_empty()
|
||||
&& (id.raw_inquiry[0] & 0x1F) == SCSI_PERIPHERAL_TYPE_OPTICAL
|
||||
{
|
||||
drives.push((info.path.clone(), id));
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
@@ -26,20 +41,15 @@ pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
drives
|
||||
}
|
||||
|
||||
pub fn resolve_device(path: &str) -> Result<(String, Option<String>)> {
|
||||
// Accept /dev/diskN or /dev/rdiskN paths as-is
|
||||
if path.contains("/disk") || path.contains("/rdisk") {
|
||||
if !std::path::Path::new(path).exists() {
|
||||
return Err(Error::DeviceNotFound {
|
||||
path: path.to_string(),
|
||||
});
|
||||
}
|
||||
return Ok((path.to_string(), None));
|
||||
}
|
||||
/// Resolve a device path on macOS. There is no `sr`→`sg` style
|
||||
/// substitution here (that is a Linux concern), so any existing path is
|
||||
/// returned unchanged as [`DeviceResolution::Direct`]; the
|
||||
/// [`DeviceResolution`] return exists for cross-platform signature parity.
|
||||
pub fn resolve_device(path: &str) -> Result<(String, DeviceResolution)> {
|
||||
if !std::path::Path::new(path).exists() {
|
||||
return Err(Error::DeviceNotFound {
|
||||
path: path.to_string(),
|
||||
});
|
||||
}
|
||||
Ok((path.to_string(), None))
|
||||
Ok((path.to_string(), DeviceResolution::Direct))
|
||||
}
|
||||
|
||||
+1269
-99
File diff suppressed because it is too large
Load Diff
+23
-8
@@ -1,9 +1,18 @@
|
||||
//! Windows drive discovery and device resolution.
|
||||
|
||||
use crate::drive::DeviceResolution;
|
||||
use crate::error::Result;
|
||||
use crate::identity::DriveId;
|
||||
use std::path::Path;
|
||||
|
||||
/// SCSI peripheral device type 5 = MMC / optical, in the low 5 bits of
|
||||
/// INQUIRY byte 0.
|
||||
const SCSI_PERIPHERAL_TYPE_OPTICAL: u8 = 0x05;
|
||||
|
||||
/// Discover optical drives. Probes `\\.\CdRom0..15` first; only if none
|
||||
/// are found does it fall back to scanning drive letters `D..Z`. Each
|
||||
/// candidate is opened, INQUIRY'd, and kept only if its peripheral device
|
||||
/// type is optical (MMC, type 0x05). Returns normalized `\\.\` paths.
|
||||
pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
let mut drives = Vec::new();
|
||||
|
||||
@@ -12,7 +21,9 @@ pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
let path = format!("\\\\.\\CdRom{}", i);
|
||||
if let Ok(mut transport) = crate::scsi::open(Path::new(&path)) {
|
||||
if let Ok(id) = DriveId::from_drive(transport.as_mut()) {
|
||||
if !id.raw_inquiry.is_empty() && (id.raw_inquiry[0] & 0x1F) == 0x05 {
|
||||
if !id.raw_inquiry.is_empty()
|
||||
&& (id.raw_inquiry[0] & 0x1F) == SCSI_PERIPHERAL_TYPE_OPTICAL
|
||||
{
|
||||
drives.push((path, id));
|
||||
}
|
||||
}
|
||||
@@ -25,8 +36,12 @@ pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
let path = format!("{}:", letter as char);
|
||||
if let Ok(mut transport) = crate::scsi::open(Path::new(&path)) {
|
||||
if let Ok(id) = DriveId::from_drive(transport.as_mut()) {
|
||||
if !id.raw_inquiry.is_empty() && (id.raw_inquiry[0] & 0x1F) == 0x05 {
|
||||
drives.push((path, id));
|
||||
if !id.raw_inquiry.is_empty()
|
||||
&& (id.raw_inquiry[0] & 0x1F) == SCSI_PERIPHERAL_TYPE_OPTICAL
|
||||
{
|
||||
// Normalize so returned paths are consistently in
|
||||
// \\.\ form regardless of which loop matched.
|
||||
drives.push((normalize_path(&path), id));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -36,8 +51,11 @@ pub fn find_drives() -> Vec<(String, DriveId)> {
|
||||
drives
|
||||
}
|
||||
|
||||
pub fn resolve_device(path: &str) -> Result<(String, Option<String>)> {
|
||||
Ok((normalize_path(path), None))
|
||||
/// Resolve a device path to its normalized Windows `\\.\` form. Windows
|
||||
/// has no `sr`→`sg` symlink-target indirection, so resolution is purely a
|
||||
/// path normalization and always reports [`DeviceResolution::Direct`].
|
||||
pub fn resolve_device(path: &str) -> Result<(String, DeviceResolution)> {
|
||||
Ok((normalize_path(path), DeviceResolution::Direct))
|
||||
}
|
||||
|
||||
/// Normalize a device path to Windows \\.\X: format.
|
||||
@@ -55,9 +73,6 @@ fn normalize_path(path: &str) -> String {
|
||||
if trimmed.len() == 2 && trimmed.as_bytes()[1] == b':' {
|
||||
return format!("\\\\.\\{}", trimmed);
|
||||
}
|
||||
if path.to_lowercase().starts_with("cdrom") {
|
||||
return format!("\\\\.\\{}", path);
|
||||
}
|
||||
format!("\\\\.\\{}", path)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
//! DVD-Video navigation — read-only resolver for the **main-feature start
|
||||
//! point** (issue #40). Mirrors what a DVD player's nav VM resolves: First-Play
|
||||
//! → menu "Play" → title dispatch → the first cell of the feature, so the rip
|
||||
//! starts at the movie rather than at raw cell 0 (e.g. skipping a leading
|
||||
//! logo/warning segment when the disc's own navigation does).
|
||||
//!
|
||||
//! Byte layout follows the DVD-Video specification (VMGI/VTSI headers,
|
||||
//! PGC/cell tables, PCI/HLI button packets); the VM command decoder is
|
||||
//! verified against libdvdnav's decoder.
|
||||
//!
|
||||
//! Current contents: [`vmcmd`] — the VM command decoder (proven against the
|
||||
//! SOTL/Greenland test discs). The IFO/PCI parsing and the navigation executor
|
||||
//! that resolves the start cell build on top of this.
|
||||
|
||||
pub mod vmcmd;
|
||||
|
||||
use crate::sector::SectorSource;
|
||||
|
||||
/// Resolve the feature title's **true start cell** (0-based index into the
|
||||
/// title PGC's cell list) by following the disc's own navigation — First-Play →
|
||||
/// menu "Play" → title dispatch — the way a player reaches the movie. This is
|
||||
/// what lets the rip begin at the feature instead of at raw cell 0 when the
|
||||
/// disc's nav enters the title past a leading logo/warning segment (e.g. a
|
||||
/// disc whose "Play" resolves to a later cell than cell 0).
|
||||
///
|
||||
/// Returns `None` when navigation cannot be resolved, so the caller falls back
|
||||
/// to the structural leading-cell filter (today's behaviour, ≈ cell 0 / 0:00).
|
||||
///
|
||||
/// TODO(#40): the IFO/PCI parsing + nav executor (built on [`vmcmd`]) land
|
||||
/// incrementally. Until the executor is complete this returns `None`, so wiring
|
||||
/// it in is behaviour-neutral; improvements to the resolver take effect here
|
||||
/// without touching the call site.
|
||||
pub fn resolve_feature_start(
|
||||
reader: &mut dyn SectorSource,
|
||||
udf: &crate::udf::UdfFs,
|
||||
vtsn: u16,
|
||||
vts_ttn: u16,
|
||||
) -> Option<usize> {
|
||||
// `reader`/`udf` are the seam inputs the nav executor will consume to read
|
||||
// VIDEO_TS.IFO + the VTS IFOs/menu VOBs. Reserved until that lands.
|
||||
let _ = (reader, udf);
|
||||
tracing::trace!(
|
||||
target: "freemkv::dvdnav",
|
||||
vtsn,
|
||||
vts_ttn,
|
||||
"nav start-cell resolver: unresolved — caller falls back to leading-cell filter"
|
||||
);
|
||||
None
|
||||
}
|
||||
@@ -0,0 +1,408 @@
|
||||
//! DVD-Video VM command decoder.
|
||||
//!
|
||||
//! An 8-byte navigation command as found in PGC command tables (pre/post/cell)
|
||||
//! and PCI button info. Decoded per the DVD-Video VM instruction set and
|
||||
//! verified against libdvdnav's command decoder.
|
||||
//!
|
||||
//! Bit model: the 8 bytes are a big-endian 64-bit word. `byte0` bits 7-5 are the
|
||||
//! command **type**; for type 1, `byte0` bit 4 selects Link (0) vs Jump (1), and
|
||||
//! `byte1` bits 3-0 are the sub-command. Compare predicates live in `byte1`
|
||||
//! bits 6-4 with the operands in bytes 2-5.
|
||||
//!
|
||||
//! This module is pure decode + a register model — no I/O, no English (numeric
|
||||
//! semantics only), matching libfreemkv conventions. The navigation *executor*
|
||||
//! and IFO/PCI parsing build on top of this.
|
||||
|
||||
/// A decoded navigation instruction. Only the variants freemkv's start-point
|
||||
/// resolver needs are modelled explicitly; everything else is [`Instr::Other`].
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Instr {
|
||||
Nop,
|
||||
/// Stop executing the current command list (resume cell playback).
|
||||
Break,
|
||||
/// Goto command line within the same list (1-based).
|
||||
Goto {
|
||||
line: u8,
|
||||
},
|
||||
/// Leave the current domain.
|
||||
Exit,
|
||||
/// Jump to a VMG title (1-based TT_SRPT index).
|
||||
JumpTt {
|
||||
ttn: u8,
|
||||
},
|
||||
/// Jump to a title within the current VTS (1-based VTS title index).
|
||||
JumpVtsTt {
|
||||
ttn: u8,
|
||||
},
|
||||
/// Jump to a part-of-title (chapter) within a VTS title.
|
||||
JumpVtsPtt {
|
||||
ttn: u8,
|
||||
pttn: u16,
|
||||
},
|
||||
/// Jump to the First-Play PGC.
|
||||
JumpSsFp,
|
||||
/// Jump to a Video-Manager menu (`menu` = menu id).
|
||||
JumpSsVmgm {
|
||||
menu: u8,
|
||||
},
|
||||
/// Jump to a Video-Title-Set menu.
|
||||
JumpSsVtsm {
|
||||
vts: u8,
|
||||
ttn: u8,
|
||||
menu: u8,
|
||||
},
|
||||
/// Jump to a specific VMGM menu PGC.
|
||||
JumpSsVmgmPgc {
|
||||
pgcn: u16,
|
||||
},
|
||||
/// Call a sub-domain (raw retained; resume handled by the executor).
|
||||
CallSs {
|
||||
sub: u8,
|
||||
},
|
||||
/// Link to a PGC number within the current domain.
|
||||
LinkPgcn {
|
||||
pgcn: u16,
|
||||
},
|
||||
/// Link to a part-of-title within the current PGC's title.
|
||||
LinkPttn {
|
||||
pttn: u16,
|
||||
},
|
||||
/// Link to a program number within the current PGC (1-based).
|
||||
LinkPgn {
|
||||
pgn: u8,
|
||||
},
|
||||
/// Link to a cell number within the current PGC (1-based).
|
||||
LinkCn {
|
||||
cn: u8,
|
||||
},
|
||||
/// A link "subset" op (LinkTopCell/NextPG/RSM/…); `sub` is the raw code.
|
||||
LinkSub {
|
||||
sub: u8,
|
||||
},
|
||||
/// Set a GPRM. `op` is the set-op code (1=mov, 3=add, …); value is immediate
|
||||
/// (`imm`) when `immediate`, else the contents of register `src`.
|
||||
SetGprm {
|
||||
reg: u8,
|
||||
op: u8,
|
||||
immediate: bool,
|
||||
imm: u16,
|
||||
src: u8,
|
||||
},
|
||||
/// Set a system parameter / unmodelled set — executor may ignore.
|
||||
SetSystem,
|
||||
/// Anything not individually modelled (kept as raw bytes).
|
||||
Other([u8; 8]),
|
||||
}
|
||||
|
||||
/// A compare predicate carried by a command (`byte1` bits 6-4). `None` = always.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Compare {
|
||||
/// Compare op: 1=&,2===,3=!=,4=>=,5=>,6=<=,7=<.
|
||||
pub op: u8,
|
||||
/// Left register index (GPRM 0-15, SPRM 128+).
|
||||
pub lhs_reg: u8,
|
||||
/// Right side: immediate when `immediate`, else register `rhs_reg`.
|
||||
pub immediate: bool,
|
||||
pub imm: u16,
|
||||
pub rhs_reg: u8,
|
||||
}
|
||||
|
||||
/// A fully decoded command: its predicate (if any) and the instruction.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Command {
|
||||
pub compare: Option<Compare>,
|
||||
pub instr: Instr,
|
||||
}
|
||||
|
||||
// Command types — `byte0` bits 7-5.
|
||||
const TYPE_SPECIAL: u8 = 0;
|
||||
const TYPE_LINK_JUMP: u8 = 1;
|
||||
const TYPE_SET_SYSTEM: u8 = 2;
|
||||
const TYPE_SET_GPRM: u8 = 3;
|
||||
|
||||
// Special (type 0) sub-commands — `byte1` bits 3-0.
|
||||
const SP_GOTO: u8 = 1;
|
||||
const SP_BREAK: u8 = 2;
|
||||
|
||||
// Jump/Call (type 1, direct=1) sub-commands.
|
||||
const JP_EXIT: u8 = 1;
|
||||
const JP_JUMP_TT: u8 = 2;
|
||||
const JP_JUMP_VTS_TT: u8 = 3;
|
||||
const JP_JUMP_VTS_PTT: u8 = 5;
|
||||
const JP_JUMP_SS: u8 = 6;
|
||||
const JP_CALL_SS: u8 = 8;
|
||||
|
||||
// Link (type 1, direct=0) sub-commands. NOTE: sub-op 0 is NOP/no-link and 1 is
|
||||
// the LinkSub form (libdvdnav `decoder.c` `eval_link_instruction`).
|
||||
const LK_SUB: u8 = 1;
|
||||
const LK_PGCN: u8 = 4;
|
||||
const LK_PTTN: u8 = 5;
|
||||
const LK_PGN: u8 = 6;
|
||||
const LK_CN: u8 = 7;
|
||||
|
||||
// JumpSS sub-domain selector — `byte5` bits 7-6.
|
||||
const SS_FP: u8 = 0;
|
||||
const SS_VMGM_MENU: u8 = 1;
|
||||
const SS_VTSM: u8 = 2;
|
||||
|
||||
// Operand field widths (spec-defined bit counts).
|
||||
const MASK_TTN: u8 = 0x7F; // 7-bit title number
|
||||
const MASK_PGN: u8 = 0x7F; // 7-bit program number
|
||||
const MASK_LINKOP: u8 = 0x1F; // 5-bit link sub-op
|
||||
const MASK_REG: u8 = 0x0F; // 4-bit GPRM index
|
||||
const MASK_MENU: u8 = 0x0F; // 4-bit menu id
|
||||
const MASK_PTTN: u16 = 0x03FF; // 10-bit part-of-title
|
||||
const MASK_PGCN: u16 = 0x7FFF; // 15-bit PGC number
|
||||
|
||||
#[inline]
|
||||
fn be16(b: &[u8; 8], o: usize) -> u16 {
|
||||
((b[o] as u16) << 8) | b[o + 1] as u16
|
||||
}
|
||||
|
||||
// Compare-operand layouts ("if_version"s) per libdvdnav `decoder.c`. The op
|
||||
// nibble is always `byte1` bits 6-4; the immediate flag is `byte1` bit 7. The
|
||||
// operand *offsets* differ by command family.
|
||||
//
|
||||
// v1 (special + link): lhs reg = b[3]; rhs imm = bytes4-5 / rhs reg = b[4].
|
||||
// v2 (jump + system-set): lhs reg = b[6]; rhs reg = b[7] (registers only).
|
||||
// v3 (set-GPRM): lhs reg = b[2]; rhs imm = bytes6-7 / rhs reg = b[6].
|
||||
fn if_v1(b: &[u8; 8]) -> Option<Compare> {
|
||||
let op = (b[1] >> 4) & 7;
|
||||
(op != 0).then(|| Compare {
|
||||
op,
|
||||
lhs_reg: b[3],
|
||||
immediate: b[1] >> 7 != 0,
|
||||
imm: be16(b, 4),
|
||||
rhs_reg: b[4],
|
||||
})
|
||||
}
|
||||
fn if_v2(b: &[u8; 8]) -> Option<Compare> {
|
||||
let op = (b[1] >> 4) & 7;
|
||||
(op != 0).then(|| Compare {
|
||||
op,
|
||||
lhs_reg: b[6],
|
||||
immediate: false,
|
||||
imm: 0,
|
||||
rhs_reg: b[7],
|
||||
})
|
||||
}
|
||||
fn if_v3(b: &[u8; 8]) -> Option<Compare> {
|
||||
let op = (b[1] >> 4) & 7;
|
||||
(op != 0).then(|| Compare {
|
||||
op,
|
||||
lhs_reg: b[2],
|
||||
immediate: b[1] >> 7 != 0,
|
||||
imm: be16(b, 6),
|
||||
rhs_reg: b[6],
|
||||
})
|
||||
}
|
||||
|
||||
/// Decode an 8-byte VM command.
|
||||
pub fn decode(b: &[u8; 8]) -> Command {
|
||||
let typ = b[0] >> 5;
|
||||
let direct = (b[0] >> 4) & 1;
|
||||
let setop = b[0] & 0x0F;
|
||||
let cmd = b[1] & 0x0F;
|
||||
|
||||
// Compare predicate, with the operand layout for this command family
|
||||
// (libdvdnav `decoder.c` `vm_eval_command` type dispatch).
|
||||
let compare = match (typ, direct) {
|
||||
(TYPE_SPECIAL, _) => if_v1(b),
|
||||
(TYPE_LINK_JUMP, 1) => if_v2(b), // jump
|
||||
(TYPE_LINK_JUMP, 0) => if_v1(b), // link
|
||||
(TYPE_SET_SYSTEM, _) => if_v2(b),
|
||||
(TYPE_SET_GPRM, _) => if_v3(b),
|
||||
_ => None, // 4/5/6 compound — not needed by the resolver
|
||||
};
|
||||
|
||||
// JumpSS sub-domain selector lives in byte5 bits 7-6.
|
||||
let ss_sel = b[5] >> 6;
|
||||
|
||||
let instr = match typ {
|
||||
TYPE_LINK_JUMP if direct == 1 => match cmd {
|
||||
JP_EXIT => Instr::Exit,
|
||||
JP_JUMP_TT => Instr::JumpTt {
|
||||
ttn: b[5] & MASK_TTN,
|
||||
},
|
||||
JP_JUMP_VTS_TT => Instr::JumpVtsTt {
|
||||
ttn: b[5] & MASK_TTN,
|
||||
},
|
||||
JP_JUMP_VTS_PTT => Instr::JumpVtsPtt {
|
||||
ttn: b[5] & MASK_TTN,
|
||||
pttn: be16(b, 2) & MASK_PTTN,
|
||||
},
|
||||
JP_JUMP_SS => match ss_sel {
|
||||
SS_FP => Instr::JumpSsFp,
|
||||
SS_VMGM_MENU => Instr::JumpSsVmgm {
|
||||
menu: b[5] & MASK_MENU,
|
||||
},
|
||||
SS_VTSM => Instr::JumpSsVtsm {
|
||||
vts: b[4],
|
||||
ttn: b[3],
|
||||
menu: b[5] & MASK_MENU,
|
||||
},
|
||||
_ => Instr::JumpSsVmgmPgc {
|
||||
pgcn: be16(b, 2) & MASK_PGCN,
|
||||
},
|
||||
},
|
||||
JP_CALL_SS => Instr::CallSs { sub: ss_sel },
|
||||
_ => Instr::Nop,
|
||||
},
|
||||
TYPE_LINK_JUMP => match cmd {
|
||||
// direct == 0 (link). sub-op 0 = NOP/no-link.
|
||||
LK_SUB => Instr::LinkSub {
|
||||
sub: b[7] & MASK_LINKOP,
|
||||
},
|
||||
LK_PGCN => Instr::LinkPgcn {
|
||||
pgcn: be16(b, 6) & MASK_PGCN,
|
||||
},
|
||||
LK_PTTN => Instr::LinkPttn {
|
||||
pttn: be16(b, 6) & MASK_PTTN,
|
||||
},
|
||||
LK_PGN => Instr::LinkPgn {
|
||||
pgn: b[7] & MASK_PGN,
|
||||
},
|
||||
LK_CN => Instr::LinkCn { cn: b[7] },
|
||||
_ => Instr::Nop,
|
||||
},
|
||||
TYPE_SPECIAL => match cmd {
|
||||
SP_GOTO => Instr::Goto { line: b[7] },
|
||||
SP_BREAK => Instr::Break,
|
||||
_ => Instr::Nop,
|
||||
},
|
||||
TYPE_SET_GPRM => Instr::SetGprm {
|
||||
reg: b[3] & MASK_REG,
|
||||
op: setop,
|
||||
immediate: direct != 0,
|
||||
imm: be16(b, 4),
|
||||
src: b[5],
|
||||
},
|
||||
TYPE_SET_SYSTEM => Instr::SetSystem,
|
||||
_ => Instr::Other(*b),
|
||||
};
|
||||
|
||||
Command { compare, instr }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn h(s: &str) -> [u8; 8] {
|
||||
let v: Vec<u8> = (0..8)
|
||||
.map(|i| u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).unwrap())
|
||||
.collect();
|
||||
v.try_into().unwrap()
|
||||
}
|
||||
|
||||
// KATs taken from the real SOTL / Greenland discs (decoded in the PoC).
|
||||
#[test]
|
||||
fn greenland_first_play_is_jumptt_1() {
|
||||
let c = decode(&h("3002000000010000"));
|
||||
assert_eq!(c.instr, Instr::JumpTt { ttn: 1 });
|
||||
assert!(c.compare.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sotl_first_play_is_jumpss_vtsm_root() {
|
||||
// 30 06 ... byte5=0x83 -> sub 2 (VTSM), vts=byte4=1, menu=byte5&0xF=3 (root)
|
||||
let c = decode(&h("3006000101830000"));
|
||||
assert_eq!(
|
||||
c.instr,
|
||||
Instr::JumpSsVtsm {
|
||||
vts: 1,
|
||||
ttn: 1,
|
||||
menu: 3
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sotl_title_dispatch_is_conditional_linkpgn_2() {
|
||||
// 20 a6 ... CmpLink: if GPRM0 == 2 -> LinkPGN 2 (cell 2 = the 5:02 start)
|
||||
let c = decode(&h("20a6000000020002"));
|
||||
assert_eq!(c.instr, Instr::LinkPgn { pgn: 2 });
|
||||
let cmp = c.compare.expect("conditional");
|
||||
assert_eq!(cmp.op, 2); // ==
|
||||
assert_eq!(cmp.lhs_reg, 0); // GPRM0
|
||||
assert!(cmp.immediate);
|
||||
assert_eq!(cmp.imm, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sotl_root_button_is_linkpgcn_37() {
|
||||
assert_eq!(
|
||||
decode(&h("2004000000000025")).instr,
|
||||
Instr::LinkPgcn { pgcn: 37 }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn greenland_scene_button_is_linkpgn() {
|
||||
assert_eq!(
|
||||
decode(&h("2006000000001401")).instr,
|
||||
Instr::LinkPgn { pgn: 1 }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jumpvts_ptt_decodes_ttn_and_pttn() {
|
||||
// synthetic: 30 05 | ptt(bytes2-3)=0x0002 | ttn(byte5)=1
|
||||
let c = decode(&h("3005000200010000"));
|
||||
assert_eq!(c.instr, Instr::JumpVtsPtt { ttn: 1, pttn: 2 });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn setgprm_immediate_mov() {
|
||||
// SOTL First-Play pre[0]: 71 00 | reg=byte3=6 | imm(bytes4-5)=0x03e8 -> g6 = 1000
|
||||
match decode(&h("7100000603e80000")).instr {
|
||||
Instr::SetGprm {
|
||||
reg,
|
||||
op,
|
||||
immediate,
|
||||
imm,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(reg, 6);
|
||||
assert_eq!(op, 1); // mov
|
||||
assert!(immediate);
|
||||
assert_eq!(imm, 1000);
|
||||
}
|
||||
other => panic!("expected SetGprm, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// Regression for the libdvdnav cross-check: link sub-op 0 = NOP, 1 = LinkSub.
|
||||
#[test]
|
||||
fn link_subop_zero_is_nop_one_is_linksub() {
|
||||
assert_eq!(decode(&h("2000000000000000")).instr, Instr::Nop);
|
||||
assert_eq!(
|
||||
decode(&h("2001000000000010")).instr,
|
||||
Instr::LinkSub { sub: 0x10 }
|
||||
);
|
||||
}
|
||||
|
||||
// if_version_1 register compare: rhs register is byte4 (not byte5).
|
||||
#[test]
|
||||
fn link_register_compare_rhs_is_byte4() {
|
||||
// 20 26: link, cmp=EQ(2), dircmp=0(register) ; cmd=6 LinkPGN
|
||||
let c = decode(&h("2026000304000002"));
|
||||
assert_eq!(c.instr, Instr::LinkPgn { pgn: 2 });
|
||||
let cmp = c.compare.expect("conditional");
|
||||
assert!(!cmp.immediate);
|
||||
assert_eq!(cmp.lhs_reg, 3);
|
||||
assert_eq!(cmp.rhs_reg, 4);
|
||||
}
|
||||
|
||||
// if_version_2 jump compare: both operands are registers in byte6 / byte7.
|
||||
#[test]
|
||||
fn jump_compare_uses_bytes6_and_7() {
|
||||
// 30 22: jump, cmp=EQ(2) ; cmd=2 JumpTT ttn=byte5=5
|
||||
let c = decode(&h("3022000000050607"));
|
||||
assert_eq!(c.instr, Instr::JumpTt { ttn: 5 });
|
||||
let cmp = c.compare.expect("conditional");
|
||||
assert!(!cmp.immediate);
|
||||
assert_eq!(cmp.lhs_reg, 6);
|
||||
assert_eq!(cmp.rhs_reg, 7);
|
||||
}
|
||||
}
|
||||
+948
-9
File diff suppressed because it is too large
Load Diff
+33
-1
@@ -8,11 +8,17 @@
|
||||
//! disc.rip(&mut session, 0, output, |event| {
|
||||
//! match event.kind {
|
||||
//! EventKind::BytesRead { bytes, total } => update_progress(bytes, total),
|
||||
//! EventKind::ReadError { sector, .. } => log_error(sector),
|
||||
//! EventKind::SectorSkipped { sector } => log_skip(sector),
|
||||
//! EventKind::BatchSizeChanged { new_size, .. } => note_recovery(new_size),
|
||||
//! _ => {}
|
||||
//! }
|
||||
//! });
|
||||
//! ```
|
||||
//!
|
||||
//! Note: the library currently emits only `BytesRead`, `SectorSkipped`,
|
||||
//! and `BatchSizeChanged`. The other [`EventKind`] variants are part of
|
||||
//! the stable event vocabulary for consumers (and future emit sites) but
|
||||
//! are not produced by the library today.
|
||||
|
||||
use crate::error::Error;
|
||||
|
||||
@@ -116,3 +122,29 @@ pub enum BatchSizeReason {
|
||||
|
||||
/// A no-op event handler. Ignores all events.
|
||||
pub fn ignore(_event: Event) {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// BatchSizeReason::Shrunk != BatchSizeReason::Probed.
|
||||
/// These two variants carry distinct meanings (error vs. recovery); they
|
||||
/// must not compare as equal.
|
||||
/// Mutation: deriving PartialEq without proper variant discrimination
|
||||
/// could make two distinct variants equal.
|
||||
#[test]
|
||||
fn batch_size_reason_variants_are_not_equal() {
|
||||
assert_ne!(BatchSizeReason::Shrunk, BatchSizeReason::Probed);
|
||||
}
|
||||
|
||||
/// BatchSizeReason is Clone + Copy: cloning does not move the original.
|
||||
/// This is required because EventKind::BatchSizeChanged embeds it by value.
|
||||
/// Mutation: removing Copy would require the caller to clone explicitly;
|
||||
/// code that passes reason by value would fail to compile.
|
||||
#[test]
|
||||
fn batch_size_reason_is_copy() {
|
||||
let r = BatchSizeReason::Shrunk;
|
||||
let _r2 = r; // copy, not move
|
||||
let _r3 = r; // r still usable after copy
|
||||
}
|
||||
}
|
||||
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
//! One-bit cooperative cancellation flag.
|
||||
//!
|
||||
//! `Halt` is a clonable token wrapping `Arc<AtomicBool>`. Pass clones into
|
||||
//! every long-running loop; the loop polls `is_cancelled()` and bails out
|
||||
//! cleanly. Calling `cancel()` from any clone flips the shared flag, and
|
||||
//! every other clone observes it on its next poll.
|
||||
//!
|
||||
//! Why: `Ordering::Relaxed` is sufficient on both load and store because
|
||||
//! this flag is purely advisory — no other memory operations piggyback on
|
||||
//! it for happens-before ordering. Callers that need to publish data
|
||||
//! across threads do so via channels or other synchronization, not via
|
||||
//! this bit.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
/// Clonable, infallible cooperative-cancellation token.
|
||||
///
|
||||
/// Clones share the same underlying flag. `cancel()` is one-way; there is
|
||||
/// no `reset()` by design — construct a fresh `Halt` for a fresh
|
||||
/// operation.
|
||||
///
|
||||
/// Construct with [`Halt::new`] (or [`Halt::default`]). The `Default`
|
||||
/// impl forwards to `new()` — both produce a fresh, uncancelled token.
|
||||
/// The pair exists because clippy's `new_without_default` lint requires
|
||||
/// `Default` whenever a public `new()` is present, even when the two
|
||||
/// would do exactly the same thing.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Halt(Arc<AtomicBool>);
|
||||
|
||||
impl Halt {
|
||||
/// Construct a fresh, uncancelled token.
|
||||
pub fn new() -> Self {
|
||||
Self(Arc::new(AtomicBool::new(false)))
|
||||
}
|
||||
|
||||
/// Wrap an existing `Arc<AtomicBool>` as a `Halt`. A bridge for
|
||||
/// callers that already hold an `Arc<AtomicBool>` cancellation flag
|
||||
/// and want to adopt the token API without allocating a new flag.
|
||||
///
|
||||
/// Cancelling either side flips the same bit — the wrapping `Halt`
|
||||
/// and the original `Arc` are two views over one shared flag.
|
||||
pub fn from_arc(flag: Arc<AtomicBool>) -> Self {
|
||||
Self(flag)
|
||||
}
|
||||
|
||||
/// Borrow the underlying `Arc<AtomicBool>`. The inverse of
|
||||
/// [`from_arc`](Self::from_arc): hand the shared flag to an API that
|
||||
/// still takes a raw `Arc<AtomicBool>` rather than a `Halt`.
|
||||
pub fn as_arc(&self) -> &Arc<AtomicBool> {
|
||||
&self.0
|
||||
}
|
||||
|
||||
/// Flip the shared flag to cancelled. Idempotent.
|
||||
pub fn cancel(&self) {
|
||||
self.0.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Read the shared flag.
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.0.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Halt {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared poll interval for halt-aware loops.
|
||||
///
|
||||
/// `bounded_syscall` checks the cancellation flag and the deadline
|
||||
/// every [`POLL_INTERVAL`] while blocked on a worker; the same
|
||||
/// cadence governs `Pipeline::send_with_halt`'s `try_send` retry.
|
||||
/// 250 ms is the sweet spot between responsiveness (operator presses
|
||||
/// Stop, sees it take effect within ~quarter-second) and waste
|
||||
/// (atomic load + clock read is cheap but not free at thousands of
|
||||
/// hertz).
|
||||
///
|
||||
/// Centralised here so the half-dozen halt-polling loops across `io`
|
||||
/// can't drift apart silently.
|
||||
pub const POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(250);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn fresh_is_not_cancelled() {
|
||||
let h = Halt::new();
|
||||
assert!(!h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_flips_state() {
|
||||
let h = Halt::new();
|
||||
assert!(!h.is_cancelled());
|
||||
h.cancel();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_is_idempotent() {
|
||||
let h = Halt::new();
|
||||
h.cancel();
|
||||
h.cancel();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state() {
|
||||
let original = Halt::new();
|
||||
let cloned = original.clone();
|
||||
assert!(!original.is_cancelled());
|
||||
assert!(!cloned.is_cancelled());
|
||||
|
||||
// Cancel via the clone; the original observes it.
|
||||
cloned.cancel();
|
||||
assert!(original.is_cancelled());
|
||||
assert!(cloned.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state_reverse_direction() {
|
||||
let original = Halt::new();
|
||||
let cloned = original.clone();
|
||||
|
||||
// Cancel via the original; the clone observes it.
|
||||
original.cancel();
|
||||
assert!(cloned.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_shares_state_across_threads() {
|
||||
let h = Halt::new();
|
||||
let h2 = h.clone();
|
||||
let handle = std::thread::spawn(move || {
|
||||
h2.cancel();
|
||||
});
|
||||
handle.join().unwrap();
|
||||
assert!(h.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_arc_shares_state() {
|
||||
// The 0.18 deprecation-window bridge: a Halt built from an
|
||||
// existing Arc<AtomicBool> must be a *view* over the same bit,
|
||||
// not a fresh copy. Cancelling either side flips both.
|
||||
let arc = Arc::new(AtomicBool::new(false));
|
||||
let halt = Halt::from_arc(arc.clone());
|
||||
assert!(!halt.is_cancelled());
|
||||
assert!(!arc.load(Ordering::Relaxed));
|
||||
|
||||
// Cancel via the wrapping Halt; the original Arc observes it.
|
||||
halt.cancel();
|
||||
assert!(arc.load(Ordering::Relaxed));
|
||||
|
||||
// Conversely: flip the Arc directly; the Halt view observes it.
|
||||
let arc2 = Arc::new(AtomicBool::new(false));
|
||||
let halt2 = Halt::from_arc(arc2.clone());
|
||||
arc2.store(true, Ordering::Relaxed);
|
||||
assert!(halt2.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn as_arc_returns_backing_flag() {
|
||||
// `as_arc()` must hand back the *same* Arc, not a clone of a
|
||||
// different bit. Verified by writing through the borrowed Arc
|
||||
// and observing through the Halt.
|
||||
let halt = Halt::new();
|
||||
let arc = halt.as_arc().clone();
|
||||
assert!(!halt.is_cancelled());
|
||||
arc.store(true, Ordering::Relaxed);
|
||||
assert!(halt.is_cancelled());
|
||||
}
|
||||
|
||||
// ── New comprehensive tests ────────────────────────────────────────────────
|
||||
|
||||
/// POLL_INTERVAL is 250ms — a specific value that the multi-thread halt
|
||||
/// loops depend on for responsiveness guarantees.
|
||||
/// Mutation: setting POLL_INTERVAL to 5s makes stop requests take 5s to notice.
|
||||
#[test]
|
||||
fn poll_interval_is_250ms() {
|
||||
assert_eq!(
|
||||
POLL_INTERVAL,
|
||||
std::time::Duration::from_millis(250),
|
||||
"POLL_INTERVAL must be 250ms for the guaranteed ~quarter-second cancel latency"
|
||||
);
|
||||
}
|
||||
}
|
||||
+209
-14
@@ -57,27 +57,48 @@ impl DriveId {
|
||||
let cdb_inq = [0x12, 0x00, 0x00, 0x00, 0x60, 0x00];
|
||||
transport.execute(&cdb_inq, DataDirection::FromDevice, &mut inquiry, 5000)?;
|
||||
|
||||
// GET CONFIGURATION Feature 010Ch — MMC-6 §6.6
|
||||
// GET CONFIGURATION Feature 010Ch — MMC-6 §6.6.
|
||||
// Best-effort: 010Ch (Firmware Information) is an optional feature.
|
||||
// A drive that lacks it may CHECK CONDITION rather than return an
|
||||
// empty descriptor, so a failure here is treated as feature-absent
|
||||
// (empty firmware date + empty raw bytes) instead of aborting the
|
||||
// whole identity probe.
|
||||
let mut gc = vec![0u8; 256];
|
||||
let cdb_gc = [0x46, 0x02, 0x01, 0x0C, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00];
|
||||
let result = transport.execute(&cdb_gc, DataDirection::FromDevice, &mut gc, 5000)?;
|
||||
// `bytes_transferred` is device-reported and untrusted; clamp every
|
||||
// slice end to the actual buffer length before indexing.
|
||||
let (firmware_date, raw_gc_010c) =
|
||||
match transport.execute(&cdb_gc, DataDirection::FromDevice, &mut gc, 5000) {
|
||||
Ok(result) => {
|
||||
let end = result.bytes_transferred.min(gc.len());
|
||||
let date = if end > 12 {
|
||||
String::from_utf8_lossy(&gc[12..24.min(end)])
|
||||
.trim()
|
||||
.to_string()
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
(date, gc[..end].to_vec())
|
||||
}
|
||||
Err(_) => (String::new(), Vec::new()),
|
||||
};
|
||||
|
||||
let firmware_date = if result.bytes_transferred > 12 {
|
||||
String::from_utf8_lossy(&gc[12..24.min(result.bytes_transferred)])
|
||||
.trim()
|
||||
.to_string()
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
// GET CONFIGURATION Feature 0108h — Serial Number
|
||||
// GET CONFIGURATION Feature 0108h — Serial Number.
|
||||
// Best-effort, like 010Ch above: the serial-number feature is
|
||||
// optional, so a drive that lacks it (CHECK CONDITION) or reports
|
||||
// too few bytes deliberately yields an empty serial rather than
|
||||
// failing the identity probe.
|
||||
let mut gc_serial = vec![0u8; 256];
|
||||
let cdb_serial = [0x46, 0x02, 0x01, 0x08, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00];
|
||||
let serial_number = if let Ok(r) =
|
||||
transport.execute(&cdb_serial, DataDirection::FromDevice, &mut gc_serial, 5000)
|
||||
{
|
||||
if r.bytes_transferred > 12 {
|
||||
String::from_utf8_lossy(&gc_serial[12..r.bytes_transferred])
|
||||
// `bytes_transferred` is device-reported and untrusted; clamp
|
||||
// the slice end to the buffer length to avoid an out-of-range
|
||||
// panic on an oversized reported count.
|
||||
let end = r.bytes_transferred.min(gc_serial.len());
|
||||
String::from_utf8_lossy(&gc_serial[12..end])
|
||||
.trim()
|
||||
.to_string()
|
||||
} else {
|
||||
@@ -94,8 +115,8 @@ impl DriveId {
|
||||
vendor_specific: ascii_field(&inquiry, 36, 43),
|
||||
firmware_date,
|
||||
serial_number,
|
||||
raw_inquiry: inquiry.to_vec(),
|
||||
raw_gc_010c: gc[..result.bytes_transferred].to_vec(),
|
||||
raw_inquiry: inquiry,
|
||||
raw_gc_010c,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -155,6 +176,49 @@ fn ascii_field(data: &[u8], start: usize, end: usize) -> String {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::scsi::{ScsiResult, ScsiTransport};
|
||||
|
||||
/// Transport that returns the requested data length but reports a
|
||||
/// bytes_transferred larger than the caller's buffer — models a drive
|
||||
/// that lies about its transfer count. The old slicing code panicked
|
||||
/// on this; the clamps must keep it from indexing out of range.
|
||||
struct OversizedCountTransport;
|
||||
|
||||
impl ScsiTransport for OversizedCountTransport {
|
||||
fn execute(
|
||||
&mut self,
|
||||
cdb: &[u8],
|
||||
_dir: DataDirection,
|
||||
buf: &mut [u8],
|
||||
_timeout_ms: u32,
|
||||
) -> Result<ScsiResult> {
|
||||
// Fill plausible ASCII so the from_utf8_lossy paths run.
|
||||
for b in buf.iter_mut() {
|
||||
*b = b'A';
|
||||
}
|
||||
// INQUIRY (0x12): honest count. GET CONFIGURATION (0x46): lie.
|
||||
let bytes_transferred = if cdb.first() == Some(&0x12) {
|
||||
buf.len()
|
||||
} else {
|
||||
buf.len() + 4096
|
||||
};
|
||||
Ok(ScsiResult {
|
||||
status: 0,
|
||||
bytes_transferred,
|
||||
sense: [0u8; 32],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_drive_clamps_oversized_bytes_transferred() {
|
||||
// Must not panic despite the transport reporting a transfer count
|
||||
// far beyond the 256-byte GET CONFIGURATION buffers.
|
||||
let mut t = OversizedCountTransport;
|
||||
let id = DriveId::from_drive(&mut t).expect("from_drive must not error");
|
||||
// raw_gc_010c is clamped to the 256-byte buffer, never the lie.
|
||||
assert_eq!(id.raw_gc_010c.len(), 256);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bu40n_identity() {
|
||||
@@ -190,4 +254,135 @@ mod tests {
|
||||
assert_eq!(id.vendor_specific.trim(), "16/04/");
|
||||
assert_eq!(id.firmware_date, "201604250000");
|
||||
}
|
||||
|
||||
// ── New comprehensive tests ────────────────────────────────────────────────
|
||||
|
||||
/// ascii_field with a buffer shorter than `start` returns empty string
|
||||
/// rather than panicking.
|
||||
/// Spec: SPC-4 §6.4.2 — bytes[8:16] are vendor ID; a truncated buffer
|
||||
/// (e.g. a device that reports fewer than 8 bytes) must not panic.
|
||||
/// Mutation: removing the `data.len() > start` guard makes it panic on short inputs.
|
||||
#[test]
|
||||
fn ascii_field_short_buffer_returns_empty() {
|
||||
// Buffer of length 5: start=8 is beyond the end → empty string.
|
||||
let buf = vec![0u8; 5];
|
||||
let result = ascii_field(&buf, 8, 16); // SPC-4 vendor ID range
|
||||
assert!(result.is_empty(), "short buffer must yield empty string");
|
||||
}
|
||||
|
||||
/// ascii_field with a buffer that covers start but not end is clamped.
|
||||
/// Spec: `ascii_field` documents "clamps to data.len()".
|
||||
/// Mutation: using `end` directly without `min(data.len())` panics here.
|
||||
#[test]
|
||||
fn ascii_field_partial_buffer_is_clamped_not_panicked() {
|
||||
// Buffer of length 12: vendor_id range is [8..16], but only [8..12] present.
|
||||
let mut buf = vec![0u8; 12];
|
||||
buf[8..12].copy_from_slice(b"SONY");
|
||||
let result = ascii_field(&buf, 8, 16);
|
||||
// Must not panic; the returned string holds what we wrote.
|
||||
assert_eq!(result, "SONY");
|
||||
}
|
||||
|
||||
/// from_inquiry extracts the product_id field from INQUIRY bytes [16:32].
|
||||
/// Spec: SPC-4 §6.4.2 — PRODUCT IDENTIFICATION at offset 16, length 16.
|
||||
/// Mutation: shifting the product_id slice to [8:24] makes this fail.
|
||||
#[test]
|
||||
fn from_inquiry_extracts_product_id_at_offset_16() {
|
||||
let mut inquiry = vec![0u8; 96];
|
||||
// Leave vendor_id (8..16) as zeros, write product_id at 16..32.
|
||||
inquiry[16..32].copy_from_slice(b"BD-RW BDR-209M");
|
||||
let id = DriveId::from_inquiry(&inquiry, "");
|
||||
assert_eq!(
|
||||
id.product_id, "BD-RW BDR-209M",
|
||||
"product_id must come from INQUIRY bytes 16..32 (SPC-4 §6.4.2)"
|
||||
);
|
||||
}
|
||||
|
||||
/// from_inquiry extracts product_revision from INQUIRY bytes [32:36].
|
||||
/// Spec: SPC-4 §6.4.2 — PRODUCT REVISION LEVEL at offset 32, length 4.
|
||||
/// Mutation: reading revision from [36:40] produces the wrong value.
|
||||
#[test]
|
||||
fn from_inquiry_extracts_revision_at_offset_32() {
|
||||
let mut inquiry = vec![0u8; 96];
|
||||
inquiry[32..36].copy_from_slice(b"1.53");
|
||||
let id = DriveId::from_inquiry(&inquiry, "");
|
||||
assert_eq!(
|
||||
id.product_revision, "1.53",
|
||||
"product_revision must come from INQUIRY bytes 32..36 (SPC-4 §6.4.2)"
|
||||
);
|
||||
}
|
||||
|
||||
/// from_inquiry extracts vendor_specific from INQUIRY bytes [36:43].
|
||||
/// Spec: SPC-4 §6.4.2 — VENDOR SPECIFIC at offset 36, length 8.
|
||||
/// Mutation: reading vendor_specific from [32:39] returns the revision instead.
|
||||
#[test]
|
||||
fn from_inquiry_extracts_vendor_specific_at_offset_36() {
|
||||
let mut inquiry = vec![0u8; 96];
|
||||
inquiry[36..43].copy_from_slice(b"MM01234");
|
||||
let id = DriveId::from_inquiry(&inquiry, "");
|
||||
assert_eq!(
|
||||
id.vendor_specific, "MM01234",
|
||||
"vendor_specific must come from INQUIRY bytes 36..43 (SPC-4 §6.4.2)"
|
||||
);
|
||||
}
|
||||
|
||||
/// from_inquiry stores the raw inquiry bytes in raw_inquiry unchanged.
|
||||
/// Mutation: copying only a slice of inquiry into raw_inquiry truncates it.
|
||||
#[test]
|
||||
fn from_inquiry_stores_raw_inquiry() {
|
||||
let mut inquiry = vec![0u8; 96];
|
||||
inquiry[8..16].copy_from_slice(b"TESTDRVR");
|
||||
let id = DriveId::from_inquiry(&inquiry, "");
|
||||
assert_eq!(
|
||||
id.raw_inquiry, inquiry,
|
||||
"raw_inquiry must preserve the full 96-byte buffer"
|
||||
);
|
||||
}
|
||||
|
||||
/// GET CONFIGURATION failure (transport error) must not abort the
|
||||
/// identity probe — firmware_date is empty, raw_gc_010c is empty.
|
||||
/// Mutation: propagating the GET_CONFIGURATION error with `?` aborts from_drive.
|
||||
#[test]
|
||||
fn from_drive_gc_failure_yields_empty_firmware_date() {
|
||||
struct GcFailTransport;
|
||||
impl ScsiTransport for GcFailTransport {
|
||||
fn execute(
|
||||
&mut self,
|
||||
cdb: &[u8],
|
||||
_dir: DataDirection,
|
||||
buf: &mut [u8],
|
||||
_timeout_ms: u32,
|
||||
) -> Result<ScsiResult> {
|
||||
if cdb.first() == Some(&0x12) {
|
||||
// INQUIRY succeeds with a plausible response.
|
||||
buf[8..16].copy_from_slice(b"TESTDRV ");
|
||||
buf[16..32].copy_from_slice(b"FAKE DRIVE MODEL");
|
||||
buf[32..36].copy_from_slice(b"0001");
|
||||
buf[36..43].copy_from_slice(b"X000001");
|
||||
Ok(ScsiResult {
|
||||
status: 0,
|
||||
bytes_transferred: buf.len(),
|
||||
sense: [0u8; 32],
|
||||
})
|
||||
} else {
|
||||
// GET CONFIGURATION fails.
|
||||
Err(crate::error::Error::ScsiError {
|
||||
opcode: cdb[0],
|
||||
status: crate::scsi::SCSI_STATUS_CHECK_CONDITION,
|
||||
sense: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut t = GcFailTransport;
|
||||
let id = DriveId::from_drive(&mut t).expect("from_drive must succeed despite GC failure");
|
||||
assert!(
|
||||
id.firmware_date.is_empty(),
|
||||
"firmware_date must be empty when GC fails"
|
||||
);
|
||||
assert!(
|
||||
id.raw_gc_010c.is_empty(),
|
||||
"raw_gc_010c must be empty when GC fails"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+957
-55
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,295 @@
|
||||
//! Bounded-syscall primitive: run a (potentially-blocking) operation
|
||||
//! on a worker thread, with a hard wall-clock deadline and an optional
|
||||
//! cooperative [`Halt`] poll. The calling thread is never trapped
|
||||
//! inside a kernel call.
|
||||
//!
|
||||
//! ## Why this exists
|
||||
//!
|
||||
//! [`crate::halt::Halt`] is cooperative: callers poll
|
||||
//! `is_cancelled()`. It cannot reach inside a syscall the kernel
|
||||
//! currently owns the thread for — `libc::sync_file_range`,
|
||||
//! `libc::fsync`, `File::write` on NFS, and so on. `/api/stop` from
|
||||
//! autorip therefore can't unstick a thread sitting in such a syscall.
|
||||
//!
|
||||
//! [`bounded_syscall`] is the escape hatch: it runs `op` on a fresh
|
||||
//! worker thread, then `recv_timeout`s on a rendezvous channel for the
|
||||
//! result. The wait is broken into ~250 ms slices so the calling
|
||||
//! thread can poll the supplied [`Halt`] in between. If the deadline
|
||||
//! elapses or the halt fires, the worker is intentionally leaked — the
|
||||
//! syscall will unwind whenever the kernel decides, or at process
|
||||
//! exit, but the caller is free to fall back to a degraded code path
|
||||
//! (skip the sync, log loudly, etc.).
|
||||
//!
|
||||
//! ## Trade-offs
|
||||
//!
|
||||
//! - **Thread per call.** Cheap (`std::thread::spawn` is < 100 µs on
|
||||
//! Linux/macOS), but not free. Use on coarse-grained finalisation
|
||||
//! syscalls (`sync_all`, `sync_file_range(WAIT_AFTER)`), not on hot
|
||||
//! inner-loop writes.
|
||||
//! - **Leak on timeout.** A wedged syscall keeps a kernel slot and a
|
||||
//! user-space thread around for the rest of the process's life.
|
||||
//! Bounded by the number of independent rip/mux sessions, which is
|
||||
//! one per disc. The alternative — trapping the caller forever —
|
||||
//! defeats the entire purpose of `/api/stop`.
|
||||
//! - **Halt granularity ~250 ms.** Halt observation is not instant;
|
||||
//! it's the worst-case latency of the `recv_timeout` slice. Good
|
||||
//! enough for human-driven stop requests; not suitable for hard
|
||||
//! real-time deadlines.
|
||||
//!
|
||||
//! ## Single source of truth
|
||||
//!
|
||||
//! Do NOT inline this pattern. Every blocking-syscall wrapper in the
|
||||
//! rip + mux pipeline calls this helper, so changes (e.g. swapping the
|
||||
//! channel impl, adjusting the poll slice, adding metrics) land in one
|
||||
//! place.
|
||||
//!
|
||||
//! ## Platform
|
||||
//!
|
||||
//! Pure `std::thread` + `std::sync::mpsc`. No `cfg(target_os)` needed
|
||||
//! here — the helper itself is platform-agnostic. Callers that wrap
|
||||
//! Linux-only syscalls (`sync_file_range`) still need their own
|
||||
//! `#[cfg(target_os = "linux")]` gates; this helper does not.
|
||||
|
||||
use std::sync::mpsc::{RecvTimeoutError, sync_channel};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::halt::{Halt, POLL_INTERVAL};
|
||||
|
||||
/// Failure outcome from a bounded syscall wrapper.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum BoundedError {
|
||||
/// The user-visible halt token fired during the wait. The worker
|
||||
/// thread is intentionally leaked — the caller should fall back to
|
||||
/// a degraded code path rather than waiting on the syscall to
|
||||
/// return.
|
||||
Halted,
|
||||
/// The deadline elapsed before the syscall returned. Same leak
|
||||
/// semantics as `Halted`.
|
||||
Timeout,
|
||||
/// The worker thread panicked, the OS rejected the thread spawn,
|
||||
/// or its sender disconnected before sending a result. Treat as a
|
||||
/// benign no-op (callers usually log and continue) rather than a
|
||||
/// hard error — by definition no syscall observably ran to
|
||||
/// completion in this case. In the spawn-failure case no thread is
|
||||
/// leaked.
|
||||
WorkerLost,
|
||||
}
|
||||
|
||||
/// Run a (potentially-blocking) operation on a worker thread with a
|
||||
/// deadline and an optional cooperative halt-token poll. Returns the
|
||||
/// operation's result if it completes within `timeout`; otherwise one
|
||||
/// of [`BoundedError::Halted`] / [`BoundedError::Timeout`] /
|
||||
/// [`BoundedError::WorkerLost`].
|
||||
///
|
||||
/// On `Halted` / `Timeout` the worker thread is intentionally leaked:
|
||||
/// the syscall will unwind whenever the kernel decides, or when the
|
||||
/// process exits. The calling thread is never trapped inside a kernel
|
||||
/// call.
|
||||
///
|
||||
/// `halt` is polled at [`POLL_INTERVAL`] granularity. Pass `None` for
|
||||
/// callers that don't (yet) have a halt token plumbed through —
|
||||
/// behaviour degrades to deadline-only, matching the 0.20.5
|
||||
/// `wait_after_with_timeout` shape this helper generalises.
|
||||
///
|
||||
/// `op` returns `R: Send + 'static`. The closure must own everything
|
||||
/// it touches because it may outlive this call (timeout / halt cases).
|
||||
pub(crate) fn bounded_syscall<F, R>(
|
||||
halt: Option<&Halt>,
|
||||
timeout: Duration,
|
||||
op: F,
|
||||
) -> Result<R, BoundedError>
|
||||
where
|
||||
F: FnOnce() -> R + Send + 'static,
|
||||
R: Send + 'static,
|
||||
{
|
||||
// If the caller already requested halt, don't spawn (and leak) a
|
||||
// worker that would run `op` to completion in the background.
|
||||
if halt.is_some_and(|h| h.is_cancelled()) {
|
||||
return Err(BoundedError::Halted);
|
||||
}
|
||||
|
||||
// Rendezvous channel: the worker sends exactly one value (the
|
||||
// op's return) and then exits. Capacity-0 means the send blocks
|
||||
// until we receive — fine on the happy path; on the timeout /
|
||||
// halt path the receiver is dropped and the worker's send
|
||||
// returns Err, which the worker ignores.
|
||||
let (tx, rx) = sync_channel::<R>(0);
|
||||
let _ = thread::Builder::new()
|
||||
.name("freemkv-bounded-syscall".into())
|
||||
.spawn(move || {
|
||||
// Ignore the send error: if we time out (or get halted)
|
||||
// before the worker finishes, the receiver is dropped
|
||||
// and `tx.send` returns Err. Either way, the worker has
|
||||
// nothing more to do.
|
||||
let _ = tx.send(op());
|
||||
});
|
||||
|
||||
let deadline = Instant::now() + timeout;
|
||||
loop {
|
||||
let now = Instant::now();
|
||||
let remaining = deadline.saturating_duration_since(now);
|
||||
let slice = remaining.min(POLL_INTERVAL);
|
||||
match rx.recv_timeout(slice) {
|
||||
Ok(v) => return Ok(v),
|
||||
Err(RecvTimeoutError::Timeout) => {
|
||||
if let Some(h) = halt {
|
||||
if h.is_cancelled() {
|
||||
return Err(BoundedError::Halted);
|
||||
}
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(BoundedError::Timeout);
|
||||
}
|
||||
// Otherwise: another slice.
|
||||
}
|
||||
Err(RecvTimeoutError::Disconnected) => {
|
||||
// Worker thread spawn failed, or it panicked before
|
||||
// sending. Caller treats this as "no syscall ran" —
|
||||
// typically a no-op + log.
|
||||
return Err(BoundedError::WorkerLost);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
#[test]
|
||||
fn op_completes_quickly() {
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), || 42u32);
|
||||
assert!(matches!(r, Ok(42)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn op_exceeds_timeout() {
|
||||
// Op sleeps longer than the deadline → Timeout.
|
||||
let r = bounded_syscall(None, Duration::from_millis(300), || {
|
||||
thread::sleep(Duration::from_secs(2));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Timeout)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_fires_during_wait() {
|
||||
let halt = Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
// Flip the halt from a side thread after ~300 ms — long
|
||||
// enough that the receive loop has rolled at least one
|
||||
// 250 ms slice and is sitting in `recv_timeout` again when
|
||||
// the bit flips.
|
||||
thread::spawn(move || {
|
||||
thread::sleep(Duration::from_millis(300));
|
||||
halt2.cancel();
|
||||
});
|
||||
let r = bounded_syscall(Some(&halt), Duration::from_secs(5), || {
|
||||
thread::sleep(Duration::from_secs(5));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Halted)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_panics() {
|
||||
// Worker panics → sender drops without sending → recv sees
|
||||
// Disconnected → WorkerLost. We use an explicit panic in the
|
||||
// op closure rather than `panic!()` from inside the channel
|
||||
// machinery; the spawned thread's panic is contained (no
|
||||
// process abort) because we don't `.join()` it.
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), || -> u32 {
|
||||
panic!("intentional test panic");
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::WorkerLost)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_already_set_before_call_still_returns_halted() {
|
||||
// Halt observed on the very first poll slice. The op blocks
|
||||
// forever; we must not wait the full timeout to notice the
|
||||
// halt is already set.
|
||||
let halt = Halt::new();
|
||||
halt.cancel();
|
||||
let started = Instant::now();
|
||||
let r = bounded_syscall(Some(&halt), Duration::from_secs(10), || {
|
||||
thread::sleep(Duration::from_secs(10));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Halted)));
|
||||
// Should bail out within ~1 s; allow 2 s of slack for slow
|
||||
// CI hosts.
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(2),
|
||||
"halt-already-set took {:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ok_path_takes_no_halt_token() {
|
||||
// Sanity: the `None` halt path is the documented zero-config
|
||||
// form (matches the 0.20.5 `wait_after_with_timeout`
|
||||
// behaviour). Op returns immediately; we must observe Ok.
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let f2 = flag.clone();
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), move || {
|
||||
f2.store(true, Ordering::Relaxed);
|
||||
"ok"
|
||||
});
|
||||
assert!(matches!(r, Ok("ok")));
|
||||
assert!(flag.load(Ordering::Relaxed));
|
||||
}
|
||||
|
||||
// ── Added hardening tests ───────────────────────────────────────
|
||||
|
||||
/// Doc contract (lines 106-110): "If the caller already requested
|
||||
/// halt, don't spawn (and leak) a worker that would run `op`."
|
||||
/// When halt is pre-cancelled the op closure must NEVER run — the
|
||||
/// short-circuit returns Halted before spawning the worker. We
|
||||
/// prove the op did not execute by checking a side-effect flag.
|
||||
#[test]
|
||||
fn pre_cancelled_halt_never_runs_op() {
|
||||
let halt = Halt::new();
|
||||
halt.cancel();
|
||||
let ran = Arc::new(AtomicBool::new(false));
|
||||
let r2 = ran.clone();
|
||||
let r = bounded_syscall(Some(&halt), Duration::from_secs(2), move || {
|
||||
r2.store(true, Ordering::SeqCst);
|
||||
7u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Halted)));
|
||||
// The op closure must not have been scheduled at all.
|
||||
assert!(
|
||||
!ran.load(Ordering::SeqCst),
|
||||
"op ran despite pre-cancelled halt — short-circuit at line 108 broken"
|
||||
);
|
||||
}
|
||||
|
||||
/// Timeout boundary: with a tiny deadline and an op that sleeps
|
||||
/// much longer, the helper must return Timeout and must do so
|
||||
/// roughly at the deadline — NOT wait for the op to finish (that
|
||||
/// is the whole point of the bounded wrapper; the worker is
|
||||
/// leaked). Grounds the `Instant::now() >= deadline` arm (line 141)
|
||||
/// and the leak contract (doc lines 84-88).
|
||||
#[test]
|
||||
fn timeout_returns_near_deadline_not_after_op() {
|
||||
let started = Instant::now();
|
||||
let r = bounded_syscall(None, Duration::from_millis(100), || {
|
||||
thread::sleep(Duration::from_secs(3));
|
||||
0u32
|
||||
});
|
||||
let elapsed = started.elapsed();
|
||||
assert!(matches!(r, Err(BoundedError::Timeout)));
|
||||
// Must bail near the 100ms deadline (one POLL_INTERVAL slack at
|
||||
// most), not after the 3s op. Allow generous CI slack but stay
|
||||
// well under the op's 3s sleep.
|
||||
assert!(
|
||||
elapsed < Duration::from_millis(1500),
|
||||
"timeout did not return near deadline: {elapsed:?} (op should be leaked, not awaited)"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,559 @@
|
||||
//! `BytePrefetcher` — `std::io::Read` analogue of
|
||||
//! [`crate::sector::PrefetchedSectorSource`].
|
||||
//!
|
||||
//! Spawns a producer thread that fills a bounded pool of `Vec<u8>`
|
||||
//! chunks from the underlying reader and ships them through a
|
||||
//! channel; the consumer pulls filled chunks, uses them, and sends
|
||||
//! the empty `Vec<u8>` back through a recycle channel so the
|
||||
//! producer can re-fill in place. Result: zero allocations and zero
|
||||
//! cross-thread frees in the steady-state hot loop.
|
||||
//!
|
||||
//! This is the byte-stream half of the freemkv mux highway —
|
||||
//! `BytePrefetcher` feeds [`crate::mux::demux_thread::DemuxThread`]
|
||||
//! for `m2ts://` (the only in-tree caller today, via
|
||||
//! [`crate::mux::resolve`]), and works for any stream whose source is
|
||||
//! an `io::Read` rather than a `SectorSource`.
|
||||
|
||||
use crate::halt::{Halt, POLL_INTERVAL};
|
||||
use crossbeam_channel::{Receiver, RecvTimeoutError, SendTimeoutError, Sender, bounded};
|
||||
use std::io::Read;
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// Items flowing through the forward channel.
|
||||
pub type Batch = std::io::Result<Vec<u8>>;
|
||||
|
||||
/// Forward channel depth — how many filled buffers the producer can
|
||||
/// stay ahead by. Two is enough to absorb a moderate consumer stall
|
||||
/// without piling up bytes.
|
||||
const FORWARD_DEPTH: usize = 2;
|
||||
|
||||
/// Recycle channel depth = forward + 1 so the producer always has at
|
||||
/// least one buffer to fill while the consumer holds one.
|
||||
const RECYCLE_DEPTH: usize = FORWARD_DEPTH + 1;
|
||||
|
||||
/// Default chunk size — 16 MiB matches the ISO-mux sector batch and
|
||||
/// is large enough that per-chunk overhead is amortised; small
|
||||
/// enough that the in-flight memory footprint stays bounded.
|
||||
pub const DEFAULT_CHUNK_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// Returned from [`BytePrefetcher::into_channels`]. Owns the
|
||||
/// producer-thread join handle so dropping the shell joins the
|
||||
/// producer.
|
||||
///
|
||||
/// Drop blocks the calling thread until the producer exits. To
|
||||
/// guarantee a prompt exit, drop the forward receiver and the recycle
|
||||
/// sender first so the producer observes channel disconnection (or
|
||||
/// cancel the [`Halt`] passed to [`BytePrefetcher::new`], which the
|
||||
/// producer polls at [`POLL_INTERVAL`] granularity even while parked
|
||||
/// on a channel op).
|
||||
pub struct PrefetchShell {
|
||||
producer: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl Drop for PrefetchShell {
|
||||
fn drop(&mut self) {
|
||||
if let Some(h) = self.producer.take() {
|
||||
let _ = h.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawned byte prefetcher. Drop joins the producer thread.
|
||||
pub struct BytePrefetcher {
|
||||
rx: Option<Receiver<Batch>>,
|
||||
recycle_tx: Option<Sender<Vec<u8>>>,
|
||||
producer: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl BytePrefetcher {
|
||||
/// Spawn the producer thread. `reader` must be `Send` because it
|
||||
/// moves into the thread. `chunk_bytes` is the size of each
|
||||
/// recycled buffer; pick the natural batch size of the
|
||||
/// downstream demuxer (16 MiB for the BD-TS mux pipeline).
|
||||
pub fn new<R: Read + Send + 'static>(
|
||||
mut reader: R,
|
||||
chunk_bytes: usize,
|
||||
halt: Option<Halt>,
|
||||
) -> std::io::Result<Self> {
|
||||
// A zero-length chunk makes every recycled buffer an empty
|
||||
// slice; `reader.read(&mut [])` returns Ok(0), which the loop
|
||||
// below treats as EOF — the consumer would see a clean,
|
||||
// silent zero-byte stream. Callers pass the downstream
|
||||
// demuxer's batch size, which is always > 0.
|
||||
debug_assert!(chunk_bytes > 0, "BytePrefetcher chunk_bytes must be > 0");
|
||||
let (tx, rx) = bounded::<Batch>(FORWARD_DEPTH);
|
||||
let (recycle_tx, recycle_rx) = bounded::<Vec<u8>>(RECYCLE_DEPTH);
|
||||
|
||||
// Seed the recycle pool. Without these the first
|
||||
// `recycle_rx.recv()` would block forever (no consumer has
|
||||
// returned a buffer yet).
|
||||
for _ in 0..RECYCLE_DEPTH {
|
||||
let _ = recycle_tx.send(vec![0u8; chunk_bytes]);
|
||||
}
|
||||
|
||||
let producer = std::thread::Builder::new()
|
||||
.name("freemkv-byte-prefetch".into())
|
||||
.spawn(move || {
|
||||
// Wrap the feed loop in catch_unwind so a panic in the inner
|
||||
// `reader.read` (e.g. a decrypt-on-read slice/arith bug) is NOT
|
||||
// indistinguishable from a clean finish at the demux boundary. A
|
||||
// clean exit (EOF, halt, consumer disconnect) returns and drops
|
||||
// `tx` → the demux loop reads RecvError as EOF (correct). A PANIC
|
||||
// sends an explicit error sentinel first so the demux loop's
|
||||
// `Ok(Err(_))` arm fires and propagates a typed error instead of
|
||||
// converting the dropped channel into a clean `DemuxBatch::Eof`
|
||||
// that would finalize a TRUNCATED mux while reporting success.
|
||||
let body = std::panic::AssertUnwindSafe(|| {
|
||||
let cancelled = || halt.as_ref().map(|h| h.is_cancelled()).unwrap_or(false);
|
||||
// Liveness heartbeat: the producer blocks on the recycle and
|
||||
// forward channels; a stalled consumer or a wedged reader shows
|
||||
// up as the beat going silent. Total is unknown, so `pos` is
|
||||
// cumulative bytes read.
|
||||
let mut hb = crate::progress::Heartbeat::new("byte_prefetch");
|
||||
let mut produced_bytes: u64 = 0;
|
||||
loop {
|
||||
hb.tick(produced_bytes, 0);
|
||||
if cancelled() {
|
||||
return;
|
||||
}
|
||||
// Park on the recycle channel, but re-poll halt
|
||||
// every POLL_INTERVAL: a pure-AtomicBool Halt does
|
||||
// not disconnect the channel, so a blocking recv()
|
||||
// would never re-reach the cancel check.
|
||||
let mut buf = loop {
|
||||
match recycle_rx.recv_timeout(POLL_INTERVAL) {
|
||||
Ok(b) => break b,
|
||||
Err(RecvTimeoutError::Timeout) => {
|
||||
if cancelled() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Consumer dropped both channels.
|
||||
Err(RecvTimeoutError::Disconnected) => return,
|
||||
}
|
||||
};
|
||||
// Re-expose the full extent. After a short read the
|
||||
// prior iteration truncated to n < chunk_bytes, so
|
||||
// this regrows the length back to chunk_bytes
|
||||
// without reallocating (capacity was fixed at
|
||||
// construction and never shrinks).
|
||||
if buf.len() < chunk_bytes {
|
||||
buf.resize(chunk_bytes, 0);
|
||||
} else {
|
||||
// SAFETY: capacity is at least chunk_bytes
|
||||
// after construction.
|
||||
unsafe { buf.set_len(chunk_bytes) };
|
||||
}
|
||||
// Read up to one full chunk. Short reads are
|
||||
// valid and common — pipe `truncate` so the
|
||||
// consumer sees only the bytes that arrived.
|
||||
let n = match reader.read(&mut buf[..]) {
|
||||
Ok(0) => return, // EOF — drop tx, consumer sees RecvError
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
let _ = tx.send(Err(e));
|
||||
return;
|
||||
}
|
||||
};
|
||||
produced_bytes += n as u64;
|
||||
buf.truncate(n);
|
||||
// Hand off the filled buffer, re-polling halt on
|
||||
// each timeout slice so a cancel can interrupt a
|
||||
// producer parked on a saturated forward channel.
|
||||
let mut pending = Ok(buf);
|
||||
loop {
|
||||
match tx.send_timeout(pending, POLL_INTERVAL) {
|
||||
Ok(()) => break,
|
||||
Err(SendTimeoutError::Timeout(returned)) => {
|
||||
if cancelled() {
|
||||
return;
|
||||
}
|
||||
pending = returned;
|
||||
}
|
||||
// Consumer dropped.
|
||||
Err(SendTimeoutError::Disconnected(_)) => return,
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
if std::panic::catch_unwind(body).is_err() {
|
||||
// Producer panicked mid-stream — surface a typed terminal
|
||||
// error so the demux thread does NOT read the dropped channel
|
||||
// as a clean EOF and truncate output.
|
||||
let _ = tx.send(Err(crate::error::Error::DemuxThreadPanicked.into()));
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(Self {
|
||||
rx: Some(rx),
|
||||
recycle_tx: Some(recycle_tx),
|
||||
producer: Some(producer),
|
||||
})
|
||||
}
|
||||
|
||||
/// Peel off the channels for zero-copy pipeline consumption. The
|
||||
/// caller (typically [`crate::mux::demux_thread::DemuxThread`])
|
||||
/// drains `rx`, runs the demuxer in place on each filled buffer,
|
||||
/// and recycles back through `recycle_tx`.
|
||||
pub fn into_channels(self) -> (Receiver<Batch>, Sender<Vec<u8>>, PrefetchShell) {
|
||||
// MOVE the three fields out cleanly — never clone. Each of
|
||||
// `rx` and `recycle_tx` ends up with exactly ONE live copy:
|
||||
// the one in the returned tuple. The pre-1.0.0 implementation
|
||||
// cloned both and then `mem::forget`-ed `self`, leaking the
|
||||
// originals so an extra live receiver + sender survived
|
||||
// forever. That defeated the channel-disconnection shutdown:
|
||||
// when the demux consumer exited early (halt, or a `tx.send`
|
||||
// error in `demux_thread`), the producer's `recycle_rx.recv()`
|
||||
// and `tx.send()` never saw all-peers-dropped, so the producer
|
||||
// never returned and `PrefetchShell::drop`'s `join()` hung.
|
||||
//
|
||||
// `ManuallyDrop` + `ptr::read` reads each field out by value
|
||||
// and suppresses `self`'s own `Drop` (which would otherwise
|
||||
// double-`join`), leaving NO extra live endpoint behind. This
|
||||
// is the panic-free equivalent of the `Option::take` approach
|
||||
// and mirrors `sector::prefetched::into_channels`.
|
||||
let me = std::mem::ManuallyDrop::new(self);
|
||||
// SAFETY: `me` is `ManuallyDrop`, so none of these fields will
|
||||
// be dropped by `me`. Each `ptr::read` performs exactly one
|
||||
// bitwise move out; every field is read exactly once and never
|
||||
// touched again, so there are no double-frees and no aliasing.
|
||||
let producer = unsafe { std::ptr::read(&me.producer) };
|
||||
// SAFETY: `rx` and `recycle_tx` are always `Some` here —
|
||||
// `into_channels` is the only way to consume a live
|
||||
// `BytePrefetcher`; `Drop::drop` is suppressed by `ManuallyDrop`.
|
||||
let rx = unsafe { std::ptr::read(&me.rx) }.expect("rx always Some before drop");
|
||||
let recycle =
|
||||
unsafe { std::ptr::read(&me.recycle_tx) }.expect("recycle_tx always Some before drop");
|
||||
(rx, recycle, PrefetchShell { producer })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BytePrefetcher {
|
||||
fn drop(&mut self) {
|
||||
// Drop channel endpoints BEFORE joining the producer so the
|
||||
// producer observes SendTimeoutError::Disconnected (forward tx)
|
||||
// or RecvTimeoutError::Disconnected (recycle rx) and exits
|
||||
// promptly. Without this, a non-EOF source fills the depth-2
|
||||
// forward channel and then spins in send_timeout(POLL_INTERVAL)
|
||||
// forever because rx is never drained, causing join() to
|
||||
// deadlock.
|
||||
drop(self.rx.take());
|
||||
drop(self.recycle_tx.take());
|
||||
if let Some(h) = self.producer.take() {
|
||||
let _ = h.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Endless reader: every `read` fills the whole buffer and never
|
||||
/// hits EOF, so the producer keeps trying to push batches forward
|
||||
/// until the forward channel disconnects. Exactly the shape that
|
||||
/// wedged the pre-1.0.0 `clone + mem::forget` `into_channels`.
|
||||
struct EndlessReader;
|
||||
impl Read for EndlessReader {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
buf.fill(0);
|
||||
Ok(buf.len())
|
||||
}
|
||||
}
|
||||
|
||||
/// Run `f` on a helper thread and fail if it does not finish within
|
||||
/// `secs`. Turns a join-deadlock into a test failure instead of a
|
||||
/// hung CI run.
|
||||
fn within<F: FnOnce() + Send + 'static>(secs: u64, f: F) {
|
||||
let (done_tx, done_rx) = bounded::<()>(1);
|
||||
std::thread::spawn(move || {
|
||||
f();
|
||||
let _ = done_tx.send(());
|
||||
});
|
||||
assert!(
|
||||
done_rx
|
||||
.recv_timeout(std::time::Duration::from_secs(secs))
|
||||
.is_ok(),
|
||||
"operation did not complete within {secs}s (deadlock)"
|
||||
);
|
||||
}
|
||||
|
||||
/// The CRITICAL regression: after `into_channels`, dropping the
|
||||
/// returned forward receiver + recycle sender must let the producer
|
||||
/// observe disconnection and exit, so dropping the `PrefetchShell`
|
||||
/// (which joins the producer) returns promptly. With the old
|
||||
/// clone+forget the leaked endpoints kept the producer blocked and
|
||||
/// this join hung forever.
|
||||
#[test]
|
||||
fn into_channels_drop_releases_producer() {
|
||||
within(10, || {
|
||||
// Small chunk so the producer cycles quickly and fills the
|
||||
// forward channel without allocating much.
|
||||
let pf = BytePrefetcher::new(EndlessReader, 4096, None).expect("spawn");
|
||||
let (rx, recycle_tx, shell) = pf.into_channels();
|
||||
// Consumer goes away early (halt / abort analogue): drop
|
||||
// both channel endpoints without draining to EOF.
|
||||
drop(rx);
|
||||
drop(recycle_tx);
|
||||
// Joining the producer must not hang.
|
||||
drop(shell);
|
||||
});
|
||||
}
|
||||
|
||||
/// Same property via the halt path: cancel the token, then the
|
||||
/// producer must exit and the shell join must complete.
|
||||
#[test]
|
||||
fn halt_releases_producer() {
|
||||
within(10, || {
|
||||
let halt = Halt::new();
|
||||
let pf = BytePrefetcher::new(EndlessReader, 4096, Some(halt.clone())).expect("spawn");
|
||||
let (_rx, _recycle_tx, shell) = pf.into_channels();
|
||||
halt.cancel();
|
||||
drop(shell);
|
||||
});
|
||||
}
|
||||
|
||||
// ── Added hardening tests ───────────────────────────────────────
|
||||
|
||||
use std::io::Cursor;
|
||||
|
||||
/// Drain the forward channel, recycling every buffer, and
|
||||
/// reassemble the bytes. Returns the concatenation of every
|
||||
/// delivered chunk. Stops on RecvError (producer dropped tx == EOF)
|
||||
/// or on the first Err batch (which it returns separately).
|
||||
fn drain_to_vec(pf: BytePrefetcher) -> (Vec<u8>, Option<std::io::Error>) {
|
||||
let (rx, recycle_tx, shell) = pf.into_channels();
|
||||
let mut out = Vec::new();
|
||||
let mut err = None;
|
||||
while let Ok(batch) = rx.recv() {
|
||||
match batch {
|
||||
Ok(buf) => {
|
||||
out.extend_from_slice(&buf);
|
||||
// Recycle so the producer can refill. Ignore send
|
||||
// error (producer may have already exited at EOF).
|
||||
let _ = recycle_tx.send(buf);
|
||||
}
|
||||
Err(e) => {
|
||||
err = Some(e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(rx);
|
||||
drop(recycle_tx);
|
||||
drop(shell);
|
||||
(out, err)
|
||||
}
|
||||
|
||||
/// CORE CONTRACT: the prefetcher must deliver every source byte,
|
||||
/// in order, exactly once — never silently truncate or duplicate.
|
||||
/// Source is 5000 bytes; chunk size 1024 forces multiple chunks
|
||||
/// (4 full + 1 short of 904). The reassembled stream must equal the
|
||||
/// source. Mutation: replacing `buf.truncate(n)` (line 141) with a
|
||||
/// no-op would over-report bytes on the final short read and this
|
||||
/// fails.
|
||||
#[test]
|
||||
fn delivers_all_bytes_in_order_across_chunks() {
|
||||
within(10, || {
|
||||
let src: Vec<u8> = (0..5000u32).map(|i| (i & 0xff) as u8).collect();
|
||||
let pf = BytePrefetcher::new(Cursor::new(src.clone()), 1024, None).expect("spawn");
|
||||
let (got, err) = drain_to_vec(pf);
|
||||
assert!(err.is_none(), "unexpected error batch: {err:?}");
|
||||
assert_eq!(got, src, "prefetcher truncated or reordered bytes");
|
||||
});
|
||||
}
|
||||
|
||||
/// Short-read truncation: a reader that returns fewer bytes than
|
||||
/// requested per call must NOT leave stale tail bytes in the
|
||||
/// delivered chunk. Cursor over 10 bytes with a 4096 chunk yields a
|
||||
/// single 10-byte chunk; the consumer must see exactly 10 bytes,
|
||||
/// not 4096. Grounds `buf.truncate(n)` at line 141. Mutation:
|
||||
/// delete the truncate and the chunk would carry 4086 zero bytes of
|
||||
/// padding, failing the length assert.
|
||||
#[test]
|
||||
fn short_read_truncates_to_actual_length() {
|
||||
within(10, || {
|
||||
let src = vec![0xAB; 10];
|
||||
let pf = BytePrefetcher::new(Cursor::new(src.clone()), 4096, None).expect("spawn");
|
||||
let (got, err) = drain_to_vec(pf);
|
||||
assert!(err.is_none());
|
||||
assert_eq!(got.len(), 10, "delivered chunk padded past actual read");
|
||||
assert_eq!(got, src);
|
||||
});
|
||||
}
|
||||
|
||||
/// EOF semantics: an empty source (Cursor over `[]`) yields
|
||||
/// `read() == Ok(0)` on the first call, which the producer treats
|
||||
/// as EOF and returns, dropping tx. The consumer sees RecvError
|
||||
/// (zero batches), NOT an Err batch and NOT a zero-length Ok batch.
|
||||
/// Grounds the `Ok(0) => return` arm at line 134. Mutation:
|
||||
/// changing `Ok(0) => return` to `Ok(0) => continue` would spin
|
||||
/// forever (within() would time out).
|
||||
#[test]
|
||||
fn empty_source_yields_clean_eof_no_batches() {
|
||||
within(10, || {
|
||||
let pf = BytePrefetcher::new(Cursor::new(Vec::<u8>::new()), 4096, None).expect("spawn");
|
||||
let (rx, recycle_tx, shell) = pf.into_channels();
|
||||
// No Ok batch should ever arrive; first recv must be Err
|
||||
// (producer dropped tx at EOF).
|
||||
let first = rx.recv();
|
||||
assert!(
|
||||
first.is_err(),
|
||||
"empty source produced a batch instead of clean EOF: {first:?}"
|
||||
);
|
||||
drop(rx);
|
||||
drop(recycle_tx);
|
||||
drop(shell);
|
||||
});
|
||||
}
|
||||
|
||||
/// Error propagation: a reader that fails mid-stream must surface
|
||||
/// the io::Error as an `Err` batch on the forward channel (line
|
||||
/// 137), not swallow it. We deliver one good chunk then an error.
|
||||
/// The consumer must see the good bytes followed by the error.
|
||||
/// Mutation: changing `let _ = tx.send(Err(e)); return;` to a plain
|
||||
/// `return` would drop the error silently and this fails.
|
||||
#[test]
|
||||
fn read_error_is_propagated_as_err_batch() {
|
||||
within(10, || {
|
||||
struct OneThenError {
|
||||
served: bool,
|
||||
}
|
||||
impl Read for OneThenError {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
if !self.served {
|
||||
self.served = true;
|
||||
let n = buf.len().min(8);
|
||||
buf[..n].fill(0x11);
|
||||
Ok(n)
|
||||
} else {
|
||||
Err(std::io::Error::other("synthetic mid-stream read failure"))
|
||||
}
|
||||
}
|
||||
}
|
||||
let pf = BytePrefetcher::new(OneThenError { served: false }, 8, None).expect("spawn");
|
||||
let (got, err) = drain_to_vec(pf);
|
||||
assert_eq!(got, vec![0x11; 8], "good chunk lost");
|
||||
let err = err.expect("read error must surface as an Err batch");
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::Other);
|
||||
});
|
||||
}
|
||||
|
||||
/// PANIC propagation: a reader that PANICS mid-stream must NOT be read as a
|
||||
/// clean EOF at the demux boundary. The producer's catch_unwind sends an
|
||||
/// explicit `Err` sentinel before the thread unwinds, so the consumer sees
|
||||
/// the good bytes followed by an error batch — never a silent truncation.
|
||||
/// Without the catch_unwind the panic would just drop `tx`, the consumer
|
||||
/// would see RecvError (== clean EOF) and the partial output would be
|
||||
/// finalized as if complete.
|
||||
#[test]
|
||||
fn read_panic_surfaces_as_err_batch_not_clean_eof() {
|
||||
within(10, || {
|
||||
struct OneThenPanic {
|
||||
served: bool,
|
||||
}
|
||||
impl Read for OneThenPanic {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
if !self.served {
|
||||
self.served = true;
|
||||
let n = buf.len().min(8);
|
||||
buf[..n].fill(0x22);
|
||||
Ok(n)
|
||||
} else {
|
||||
panic!("synthetic mid-stream reader panic");
|
||||
}
|
||||
}
|
||||
}
|
||||
let pf = BytePrefetcher::new(OneThenPanic { served: false }, 8, None).expect("spawn");
|
||||
let (got, err) = drain_to_vec(pf);
|
||||
assert_eq!(got, vec![0x22; 8], "good chunk lost before the panic");
|
||||
assert!(
|
||||
err.is_some(),
|
||||
"a mid-stream producer PANIC must surface as an Err batch, \
|
||||
not a clean EOF (which would silently truncate the mux)"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// Recycle-buffer reuse must NOT leak stale bytes between chunks of
|
||||
/// different lengths. After a full chunk, a short read reuses the
|
||||
/// same recycled buffer; lines 123-129 regrow it to chunk_bytes
|
||||
/// before reading, then line 141 truncates to the short count. We
|
||||
/// verify the short chunk carries only fresh bytes by reassembling
|
||||
/// the full stream. Source: 8 bytes of 0xAA + 3 bytes of 0xBB, with
|
||||
/// chunk_bytes=8 → chunk0 = 8×0xAA, chunk1 = 3×0xBB.
|
||||
#[test]
|
||||
fn recycled_buffer_carries_no_stale_tail() {
|
||||
within(10, || {
|
||||
let mut src = vec![0xAA; 8];
|
||||
src.extend_from_slice(&[0xBB; 3]);
|
||||
let pf = BytePrefetcher::new(Cursor::new(src.clone()), 8, None).expect("spawn");
|
||||
let (got, err) = drain_to_vec(pf);
|
||||
assert!(err.is_none());
|
||||
assert_eq!(
|
||||
got, src,
|
||||
"stale bytes from recycled buffer leaked into short chunk"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// Exact-multiple boundary: when the source length is an exact
|
||||
/// multiple of chunk_bytes, the final non-empty chunk is followed
|
||||
/// by an `Ok(0)` EOF read, NOT a spurious empty Ok batch. 12 bytes
|
||||
/// with chunk_bytes=4 → three 4-byte chunks then clean EOF. Total
|
||||
/// bytes must equal 12 and no zero-length batch may appear.
|
||||
#[test]
|
||||
fn exact_multiple_length_no_trailing_empty_batch() {
|
||||
within(10, || {
|
||||
let src = vec![0x42u8; 12];
|
||||
let pf = BytePrefetcher::new(Cursor::new(src.clone()), 4, None).expect("spawn");
|
||||
let (rx, recycle_tx, shell) = pf.into_channels();
|
||||
let mut total = 0usize;
|
||||
let mut batch_count = 0usize;
|
||||
while let Ok(Ok(buf)) = rx.recv() {
|
||||
assert!(!buf.is_empty(), "producer emitted a zero-length batch");
|
||||
total += buf.len();
|
||||
batch_count += 1;
|
||||
let _ = recycle_tx.send(buf);
|
||||
}
|
||||
assert_eq!(total, 12);
|
||||
assert_eq!(batch_count, 3, "expected exactly 3 full chunks");
|
||||
drop(rx);
|
||||
drop(recycle_tx);
|
||||
drop(shell);
|
||||
});
|
||||
}
|
||||
|
||||
/// Dropping the BytePrefetcher directly (without into_channels)
|
||||
/// must join the producer cleanly when the source is finite. The
|
||||
/// producer reaches EOF, drops tx, and exits; Drop's join returns.
|
||||
/// Grounds the BytePrefetcher Drop impl (lines 202-208). Mutation:
|
||||
/// removing the `Ok(0) => return` EOF exit would hang this join.
|
||||
#[test]
|
||||
fn drop_finite_prefetcher_joins_cleanly() {
|
||||
within(10, || {
|
||||
let pf = BytePrefetcher::new(Cursor::new(vec![1u8; 100]), 4096, None).expect("spawn");
|
||||
// Drop without consuming — producer fills the forward
|
||||
// channel (capacity 2), reaches EOF on the third read since
|
||||
// 100 < 4096 (single chunk + EOF), drops tx, exits.
|
||||
drop(pf);
|
||||
});
|
||||
}
|
||||
|
||||
/// Regression: dropping a BytePrefetcher directly (without
|
||||
/// into_channels) with an ENDLESS source must not deadlock. Before
|
||||
/// the fix, Drop joined the producer while rx/recycle_tx were still
|
||||
/// alive (sibling field drop order), so the producer filled the
|
||||
/// depth-2 forward channel and then spun in send_timeout forever
|
||||
/// (rx never drained, halt=None). The fix drops rx+recycle_tx
|
||||
/// BEFORE the join so the producer sees SendTimeoutError::Disconnected
|
||||
/// and exits.
|
||||
#[test]
|
||||
fn drop_endless_prefetcher_joins_cleanly() {
|
||||
within(10, || {
|
||||
let pf = BytePrefetcher::new(EndlessReader, 4096, None).expect("spawn");
|
||||
// Drop without consuming — the old Drop deadlocked here.
|
||||
drop(pf);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
//! Linux read-side platform hooks: sequential-access hint at open +
|
||||
//! periodic page-cache eviction during streaming reads.
|
||||
//!
|
||||
//! ## Why both
|
||||
//!
|
||||
//! `POSIX_FADV_SEQUENTIAL` at open widens the kernel's readahead window
|
||||
//! so each pread aggregates into fewer NFS round-trips. `DONTNEED` on
|
||||
//! the consumed window (called periodically by the caller) drops the
|
||||
//! already-read pages from the page cache so an 85 GB streaming ISO
|
||||
//! read doesn't fill memory and starve concurrent writes (the MKV
|
||||
//! output during mux). Together they mirror the write-side
|
||||
//! WritebackPipeline's policy.
|
||||
//!
|
||||
//! ## History
|
||||
//!
|
||||
//! Pre-Phase-1 (0.20.7 baseline) had both. Phase 1's introduction of
|
||||
//! `FileSectorSource` silently dropped the read-side DONTNEED, and
|
||||
//! 0.21.2's revert of `SEQUENTIAL` (mistakenly attributing a regression
|
||||
//! to it) removed the hint. Net effect: 85 GB of ISO reads pinned in
|
||||
//! the page cache + no readahead widening → mux throughput collapse
|
||||
//! from 18 MB/s historical to 2.7-8 MB/s on 0.21.x. Restored in 0.21.6.
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
pub(super) fn hint_sequential(file: &File, _len_bytes: u64) {
|
||||
// Best-effort: return value ignored. A fadvise failure has no
|
||||
// user-observable consequence.
|
||||
unsafe {
|
||||
libc::posix_fadvise(file.as_raw_fd(), 0, 0, libc::POSIX_FADV_SEQUENTIAL);
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop pages in the half-open byte range `[start, start+len)` from
|
||||
/// the page cache. Called periodically by `read_sectors` to bound the
|
||||
/// read-side page cache pressure.
|
||||
pub(super) fn drop_window(file: &File, start: u64, len: u64) {
|
||||
unsafe {
|
||||
libc::posix_fadvise(
|
||||
file.as_raw_fd(),
|
||||
start as i64,
|
||||
len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Async-prefetch `len` bytes at `offset` into the page cache. The
|
||||
/// kernel `readahead(2)` syscall queues the I/O and returns
|
||||
/// immediately — it does NOT wait for completion. Called right after
|
||||
/// each consumed read so the next batch's I/O overlaps with the
|
||||
/// caller's processing of the current batch (decrypt + demux + mux).
|
||||
///
|
||||
/// Without this hint, with a synchronous demux consumer running at
|
||||
/// ~50 MB/s and a single-spindle disk capable of ~150 MB/s, the disk
|
||||
/// sits idle ~70% of each iteration because kernel readahead alone
|
||||
/// (capped at `/sys/block/<dev>/queue/read_ahead_kb`, default 128 KB)
|
||||
/// can only pre-stage a tiny slice of the next batch. An explicit
|
||||
/// `readahead()` of the same size as the current batch tells the
|
||||
/// kernel to queue the full next-batch read now.
|
||||
pub(super) fn prefetch(file: &File, offset: u64, len: u64) {
|
||||
unsafe {
|
||||
libc::readahead(file.as_raw_fd(), offset as i64, len as usize);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
//! macOS: hint the kernel to prefetch a generous chunk. macOS has no
|
||||
//! direct `POSIX_FADV_SEQUENTIAL` equivalent; the idiomatic hint is
|
||||
//! `fcntl(F_RDADVISE, &radvisory)` describing the byte range you
|
||||
//! intend to read soon. We point it at the whole file (clamped to a
|
||||
//! ceiling so a multi-TB ISO doesn't ask the kernel to prefetch
|
||||
//! everything at once).
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
/// Cap on the byte length we pass to `F_RDADVISE`. Asking for a
|
||||
/// multi-GB readahead window is counterproductive — the OS doesn't
|
||||
/// have that much cache to throw at one fd. 64 MiB is generous for
|
||||
/// our use case (sweep, mux) so the kernel's prefetch ≥ our app-level
|
||||
/// pipeline depth.
|
||||
const RDADVISE_MAX_BYTES: i64 = 64 * 1024 * 1024;
|
||||
|
||||
pub(super) fn hint_sequential(file: &File, len_bytes: u64) {
|
||||
let bytes = (len_bytes as i64).min(RDADVISE_MAX_BYTES);
|
||||
let mut ra = libc::radvisory {
|
||||
ra_offset: 0,
|
||||
ra_count: bytes as libc::c_int,
|
||||
};
|
||||
// Best-effort.
|
||||
unsafe {
|
||||
libc::fcntl(file.as_raw_fd(), libc::F_RDADVISE, &mut ra);
|
||||
}
|
||||
}
|
||||
|
||||
/// macOS has no direct `POSIX_FADV_DONTNEED` equivalent for a byte
|
||||
/// range. `fcntl(F_NOCACHE)` would disable caching globally on the fd
|
||||
/// (too coarse — we want the unread region to still benefit). Best
|
||||
/// approximation: no-op. macOS's unified buffer cache is generally
|
||||
/// less prone to the pin-everything pathology that triggers the
|
||||
/// regression on Linux NFS clients.
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
|
||||
/// Async-prefetch the byte range `[offset, offset+len)`. macOS uses
|
||||
/// the same `fcntl(F_RDADVISE, &radvisory)` primitive as the open-
|
||||
/// time sequential hint, just targeted at a moving window instead of
|
||||
/// the whole file. The kernel queues I/O for the requested range and
|
||||
/// returns immediately.
|
||||
pub(super) fn prefetch(file: &File, offset: u64, len: u64) {
|
||||
let bytes = (len as i64).min(RDADVISE_MAX_BYTES);
|
||||
let mut ra = libc::radvisory {
|
||||
ra_offset: offset as libc::off_t,
|
||||
ra_count: bytes as libc::c_int,
|
||||
};
|
||||
// Best-effort — kernel hint only.
|
||||
unsafe {
|
||||
libc::fcntl(file.as_raw_fd(), libc::F_RDADVISE, &mut ra);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,521 @@
|
||||
//! [`FileSectorSource`] — read 2048-byte sectors from an ISO file on
|
||||
//! disk via direct `seek + read_exact` (`pread`-equivalent) calls,
|
||||
//! letting the kernel's own readahead policy manage prefetch.
|
||||
//!
|
||||
//! ## Why no app-level buffer
|
||||
//!
|
||||
//! Pre-0.21.3 this source held a 32 MiB (later 4 MiB) read-ahead
|
||||
//! buffer to amortise per-sector NFS round-trips. Empirically that
|
||||
//! buffer hurt: 32 MiB refills bursted the NFS TCP connection hard
|
||||
//! enough to starve the concurrent writer, and even a 4 MiB window
|
||||
//! gave the kernel less freedom to pipeline reads with writes. Direct
|
||||
//! pread per call lets Linux's readahead widen as it detects the
|
||||
//! sequential pattern, and naturally interleaves with writeback.
|
||||
//!
|
||||
//! ## DONTNEED on the consumed window
|
||||
//!
|
||||
//! Without page-cache eviction an 85 GB streaming ISO read pins the
|
||||
//! entire file in memory, starves the concurrent writer, and collapses
|
||||
//! mux throughput (observed: 2.7 MB/s mux on 0.21.5 vs. 70 MB/s
|
||||
//! isolated NFS reads). Every [`READ_DROP_CHUNK_BYTES_DEFAULT`] of
|
||||
//! consumed bytes we call `posix_fadvise(DONTNEED)` over that window,
|
||||
//! mirroring the write-side [`crate::io::writeback::WritebackPipeline`]
|
||||
//! policy.
|
||||
//!
|
||||
//! The drop window is accounted by a monotonic forward byte counter,
|
||||
//! which matches the sequential streaming pattern the mux highway
|
||||
//! drives. Under random or backward access the dropped range no longer
|
||||
//! lines up with the bytes actually read — but `DONTNEED` is purely an
|
||||
//! advisory cache hint with no correctness impact, so this degrades to
|
||||
//! a slightly imprecise hint rather than a bug.
|
||||
//!
|
||||
//! ## Platform open hint
|
||||
//!
|
||||
//! On `open()` each platform issues its "sequential access expected"
|
||||
//! hint so OS-level readahead widens. The hint and the DONTNEED call
|
||||
//! live in per-OS sibling modules ([`linux::hint_sequential`] et al.)
|
||||
//! — no inline `#[cfg]` in this file.
|
||||
//!
|
||||
//! ## Read-ahead prefetch
|
||||
//!
|
||||
//! After every consumed read we issue an OS-level prefetch hint for
|
||||
//! the next equivalent-sized window (`platform::prefetch`). The
|
||||
//! kernel queues that I/O asynchronously and returns immediately, so
|
||||
//! the next batch's read overlaps with the caller's processing of
|
||||
//! the current batch (decrypt + demux + mux). Without this the disk
|
||||
//! sits idle ~70% of each iteration because kernel SEQUENTIAL
|
||||
//! readahead alone (capped at `read_ahead_kb`, default 128 KB) is
|
||||
//! far smaller than our 16 MiB app-level batch.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
mod other;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux as platform;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos as platform;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
use other as platform;
|
||||
#[cfg(target_os = "windows")]
|
||||
use windows as platform;
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::Path;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorSource;
|
||||
|
||||
use crate::consts::{SECTOR_BYTES, SECTOR_BYTES_U64};
|
||||
|
||||
/// Bytes-read threshold per `posix_fadvise(DONTNEED)` drop on the
|
||||
/// read side. Mirrors `WRITEBACK_CHUNK_BYTES` so the read-side page
|
||||
/// cache stays bounded the same way the write side does.
|
||||
///
|
||||
/// 32 MiB is the empirically tuned value on the rip1 test bed (single
|
||||
/// 7200rpm HDD via SATA): smaller windows (8 / 16 MiB) shorten the
|
||||
/// kernel-readahead overlap and slow the producer; larger windows
|
||||
/// (64 / 128 MiB) let the page cache pin enough of the ISO to
|
||||
/// pressure concurrent writes. Override via `FREEMKV_READ_DROP_CHUNK_MIB`.
|
||||
const READ_DROP_CHUNK_BYTES_DEFAULT: u64 = 32 * 1024 * 1024;
|
||||
|
||||
fn read_drop_chunk_bytes() -> u64 {
|
||||
std::env::var("FREEMKV_READ_DROP_CHUNK_MIB")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.filter(|&n| n > 0)
|
||||
.map(|n| n * 1024 * 1024)
|
||||
.unwrap_or(READ_DROP_CHUNK_BYTES_DEFAULT)
|
||||
}
|
||||
|
||||
/// SectorSource backed by a file (ISO image). Every `read_sectors`
|
||||
/// call is a direct `seek + read_exact` against the underlying file
|
||||
/// — kernel readahead handles prefetch, and every
|
||||
/// [`READ_DROP_CHUNK_BYTES_DEFAULT`] bytes of consumed data the
|
||||
/// platform's `DONTNEED` hook drops the consumed window from the
|
||||
/// page cache to bound memory pressure.
|
||||
pub struct FileSectorSource {
|
||||
file: File,
|
||||
/// Total file size in sectors. Constant after construction;
|
||||
/// surfaced via [`SectorSource::capacity_sectors`].
|
||||
capacity: u32,
|
||||
/// Bytes read since the last DONTNEED drop. Drives the per-
|
||||
/// [`read_drop_chunk_bytes`] page-cache eviction in read_sectors.
|
||||
bytes_read_since_drop: u64,
|
||||
/// File offset at which the current drop window starts. The next
|
||||
/// DONTNEED drops from `drop_window_start` for
|
||||
/// `bytes_read_since_drop` bytes. This advances monotonically with
|
||||
/// the byte count, so it tracks the actual reads only under the
|
||||
/// forward-sequential access the mux highway uses; under random
|
||||
/// access it degrades to a harmless, imprecise advisory hint.
|
||||
drop_window_start: u64,
|
||||
/// Cached drop chunk size (resolved from env once at open).
|
||||
drop_chunk_bytes: u64,
|
||||
}
|
||||
|
||||
impl FileSectorSource {
|
||||
/// Open an existing ISO file for reading. Capacity is derived
|
||||
/// from `metadata().len() / 2048`. Returns
|
||||
/// [`Error::IsoTooLarge`] if the file would exceed the 32-bit
|
||||
/// LBA address space (~8 TB).
|
||||
///
|
||||
/// Issues the platform's "sequential access expected" hint on the
|
||||
/// fd (Linux `posix_fadvise(SEQUENTIAL)`, macOS `fcntl(F_RDADVISE)`,
|
||||
/// Windows no-op) so the kernel's readahead widens.
|
||||
pub fn open(path: &Path) -> Result<Self> {
|
||||
let file = File::open(path).map_err(|e| Error::IoError { source: e })?;
|
||||
let len = file
|
||||
.metadata()
|
||||
.map_err(|e| Error::IoError { source: e })?
|
||||
.len();
|
||||
let sectors = len / SECTOR_BYTES_U64;
|
||||
if sectors > u32::MAX as u64 {
|
||||
return Err(Error::IsoTooLarge {
|
||||
path: path.to_string_lossy().into_owned(),
|
||||
});
|
||||
}
|
||||
let capacity = sectors as u32;
|
||||
|
||||
// Best-effort sequential hint. Ignored on platforms without
|
||||
// an equivalent primitive (or where the API exists but the
|
||||
// FS doesn't honour it).
|
||||
platform::hint_sequential(&file, len);
|
||||
|
||||
Ok(Self {
|
||||
file,
|
||||
capacity,
|
||||
bytes_read_since_drop: 0,
|
||||
drop_window_start: 0,
|
||||
drop_chunk_bytes: read_drop_chunk_bytes(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for FileSectorSource {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
out: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
let count = count as u32;
|
||||
let bytes = count as usize * SECTOR_BYTES;
|
||||
debug_assert!(
|
||||
out.len() >= bytes,
|
||||
"FileSectorSource::read_sectors: out len {} < requested {}",
|
||||
out.len(),
|
||||
bytes
|
||||
);
|
||||
if count == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
let offset = lba as u64 * SECTOR_BYTES_U64;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.read_exact(&mut out[..bytes])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
|
||||
// Queue the next batch's read with the kernel before the
|
||||
// caller starts processing what we just returned. readahead()
|
||||
// is non-blocking — it queues I/O and returns, so the kernel
|
||||
// pulls those pages into cache while the consumer (decrypt +
|
||||
// demux + mux) runs. Next read_sectors call hits a warm cache.
|
||||
platform::prefetch(&self.file, offset + bytes as u64, bytes as u64);
|
||||
|
||||
// Periodic page-cache eviction on the read side. Without
|
||||
// this, an 85 GB streaming ISO read pins the entire file in
|
||||
// the kernel page cache, which starves concurrent writes and
|
||||
// collapses mux throughput. Mirrors the write-side
|
||||
// WritebackPipeline's DONTNEED policy.
|
||||
self.bytes_read_since_drop += bytes as u64;
|
||||
if self.bytes_read_since_drop >= self.drop_chunk_bytes {
|
||||
let drop_start = self.drop_window_start;
|
||||
let drop_len = self.bytes_read_since_drop;
|
||||
platform::drop_window(&self.file, drop_start, drop_len);
|
||||
self.drop_window_start = drop_start + drop_len;
|
||||
self.bytes_read_since_drop = 0;
|
||||
}
|
||||
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
use tempfile::tempdir;
|
||||
|
||||
/// Build a deterministic ISO of `sectors` sectors where sector `n`
|
||||
/// is filled with the byte pattern `((n & 0xff) as u8)`. Lets us
|
||||
/// verify any sector by content alone.
|
||||
fn make_iso(path: &std::path::Path, sectors: u32) {
|
||||
let mut f = std::fs::File::create(path).unwrap();
|
||||
let mut chunk = vec![0u8; SECTOR_BYTES];
|
||||
for n in 0..sectors {
|
||||
let b = (n & 0xff) as u8;
|
||||
chunk.iter_mut().for_each(|c| *c = b);
|
||||
f.write_all(&chunk).unwrap();
|
||||
}
|
||||
f.flush().unwrap();
|
||||
}
|
||||
|
||||
/// Sectors used by spanning-boundary tests. Pick something that
|
||||
/// exercises multi-megabyte reads without making test ISOs huge.
|
||||
/// 8192 sectors = 16 MiB — large enough to cross any readahead
|
||||
/// chunk size we set the kernel hint to.
|
||||
const TEST_SPAN_SECTORS: u32 = 8192;
|
||||
|
||||
#[test]
|
||||
fn sequential_reads_match_file() {
|
||||
let total = TEST_SPAN_SECTORS * 2 + 17;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("seq.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), total);
|
||||
|
||||
let mut got = vec![0u8; SECTOR_BYTES];
|
||||
for lba in 0..total {
|
||||
src.read_sectors(lba, 1, &mut got, false).unwrap();
|
||||
let expected = (lba & 0xff) as u8;
|
||||
assert!(
|
||||
got.iter().all(|b| *b == expected),
|
||||
"sector {lba} content mismatch: expected 0x{expected:02x}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_sector_read_across_chunk_boundary() {
|
||||
let total = TEST_SPAN_SECTORS * 2;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("span.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
|
||||
let span_lba = TEST_SPAN_SECTORS - 2;
|
||||
let mut buf4 = vec![0u8; SECTOR_BYTES * 4];
|
||||
src.read_sectors(span_lba, 4, &mut buf4, false).unwrap();
|
||||
for i in 0..4 {
|
||||
let lba = span_lba + i as u32;
|
||||
let expected = (lba & 0xff) as u8;
|
||||
for b in &buf4[i * SECTOR_BYTES..(i + 1) * SECTOR_BYTES] {
|
||||
assert_eq!(*b, expected, "byte mismatch at sub-sector {i}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backward_seek_reads_correct_bytes() {
|
||||
// Read forward then jump back: the SectorSource contract is
|
||||
// byte-correctness regardless of access pattern.
|
||||
let total = TEST_SPAN_SECTORS * 2 + 5;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("back.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
let mut got = vec![0u8; SECTOR_BYTES];
|
||||
|
||||
src.read_sectors(TEST_SPAN_SECTORS + 1, 1, &mut got, false)
|
||||
.unwrap();
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
assert!(got.iter().all(|b| *b == 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_at_eof_returns_correct_bytes() {
|
||||
// File smaller than the readahead chunk — reads near EOF must
|
||||
// still return correct bytes.
|
||||
let total: u32 = 100;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("small.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), total);
|
||||
|
||||
let mut got = vec![0u8; SECTOR_BYTES];
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
src.read_sectors(total - 1, 1, &mut got, false).unwrap();
|
||||
let expected = ((total - 1) & 0xff) as u8;
|
||||
assert!(got.iter().all(|b| *b == expected));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_single_read() {
|
||||
// A multi-MB single read must work — the implementation has
|
||||
// no app-level chunking, so this just exercises the direct
|
||||
// pread path on a larger request.
|
||||
let total = TEST_SPAN_SECTORS + 100;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("big.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
let req = (TEST_SPAN_SECTORS + 1) as u16;
|
||||
let req_bytes = req as usize * SECTOR_BYTES;
|
||||
let mut big = vec![0u8; req_bytes];
|
||||
src.read_sectors(0, req, &mut big, false).unwrap();
|
||||
assert!(big[..SECTOR_BYTES].iter().all(|b| *b == 0));
|
||||
let last_lba = req as u32 - 1;
|
||||
let exp = (last_lba & 0xff) as u8;
|
||||
let last_off = (req as usize - 1) * SECTOR_BYTES;
|
||||
assert!(
|
||||
big[last_off..last_off + SECTOR_BYTES]
|
||||
.iter()
|
||||
.all(|b| *b == exp)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_chunk_size_env_override() {
|
||||
// Explicit 8 MiB via env var.
|
||||
// SAFETY: tests in this crate are single-threaded per the
|
||||
// default cargo test harness, but std::env::set_var is
|
||||
// declared `unsafe` since Rust 2024 (it can race with other
|
||||
// threads / TLS). For a test that runs in-process before any
|
||||
// FileSectorSource construction this is safe in practice.
|
||||
unsafe {
|
||||
std::env::set_var("FREEMKV_READ_DROP_CHUNK_MIB", "8");
|
||||
}
|
||||
assert_eq!(read_drop_chunk_bytes(), 8 * 1024 * 1024);
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("FREEMKV_READ_DROP_CHUNK_MIB");
|
||||
}
|
||||
assert_eq!(read_drop_chunk_bytes(), READ_DROP_CHUNK_BYTES_DEFAULT);
|
||||
|
||||
// Garbage env value falls back to default.
|
||||
unsafe {
|
||||
std::env::set_var("FREEMKV_READ_DROP_CHUNK_MIB", "not-a-number");
|
||||
}
|
||||
assert_eq!(read_drop_chunk_bytes(), READ_DROP_CHUNK_BYTES_DEFAULT);
|
||||
unsafe {
|
||||
std::env::remove_var("FREEMKV_READ_DROP_CHUNK_MIB");
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Additional coverage.
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
/// `count == 0` must short-circuit to Ok(0) WITHOUT seeking or
|
||||
/// reading, even at an out-of-range LBA — the early-return guard
|
||||
/// runs before any I/O. Grounding: `if count == 0 { return Ok(0) }`.
|
||||
#[test]
|
||||
fn zero_count_returns_zero_no_io() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("zc.iso");
|
||||
make_iso(&path, 4);
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
// LBA far past EOF — must not matter because count==0 returns early.
|
||||
let mut buf = [0u8; 1];
|
||||
let n = src.read_sectors(1_000_000, 0, &mut buf, false).unwrap();
|
||||
assert_eq!(n, 0);
|
||||
}
|
||||
|
||||
/// Reading past EOF must ERROR (read_exact's UnexpectedEof), never
|
||||
/// return a partial/short count. This is the core "never silently
|
||||
/// truncate / never return fewer bytes than declared" property of
|
||||
/// the SectorSource contract. Grounding: `self.file.read_exact(...)`
|
||||
/// — read_exact fails if the file can't supply the full span.
|
||||
#[test]
|
||||
fn read_past_eof_errors_not_truncates() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("eof.iso");
|
||||
make_iso(&path, 4); // 4 sectors only
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), 4);
|
||||
|
||||
// Request 2 sectors starting at LBA 3 → sector 4 doesn't exist.
|
||||
let mut buf = vec![0u8; 2 * SECTOR_BYTES];
|
||||
let r = src.read_sectors(3, 2, &mut buf, false);
|
||||
let err = r.expect_err("reading past EOF must error, not short-read");
|
||||
let io: std::io::Error = err.into();
|
||||
assert_eq!(
|
||||
io.kind(),
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
"partial read at EOF must surface read_exact's UnexpectedEof"
|
||||
);
|
||||
}
|
||||
|
||||
/// On a successful full read the returned count MUST equal
|
||||
/// `count * 2048` exactly — the declared byte count. Grounding:
|
||||
/// `Ok(bytes)` where `bytes = count * SECTOR_BYTES`.
|
||||
#[test]
|
||||
fn full_read_returns_exact_declared_bytes() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("exact.iso");
|
||||
make_iso(&path, 16);
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
let mut buf = vec![0u8; 5 * SECTOR_BYTES];
|
||||
let n = src.read_sectors(2, 5, &mut buf, false).unwrap();
|
||||
assert_eq!(n, 5 * SECTOR_BYTES, "must return exactly count*2048 bytes");
|
||||
}
|
||||
|
||||
/// Capacity is `file_len / 2048` (floor); trailing bytes that don't
|
||||
/// complete a sector are NOT counted. A file of 4 sectors + 100
|
||||
/// extra bytes reports capacity 4. Grounding: `len / SECTOR_BYTES`
|
||||
/// integer division in `open`.
|
||||
#[test]
|
||||
fn capacity_floors_partial_trailing_sector() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("partial.iso");
|
||||
make_iso(&path, 4);
|
||||
// Append 100 stray bytes (a torn final sector).
|
||||
{
|
||||
let mut f = std::fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&path)
|
||||
.unwrap();
|
||||
f.write_all(&[0xee; 100]).unwrap();
|
||||
f.flush().unwrap();
|
||||
}
|
||||
let src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(
|
||||
src.capacity_sectors(),
|
||||
4,
|
||||
"partial trailing bytes must not inflate the sector capacity"
|
||||
);
|
||||
}
|
||||
|
||||
/// An empty file opens cleanly with capacity 0. Grounding:
|
||||
/// `0 / 2048 == 0`, and the IsoTooLarge guard only fires for
|
||||
/// oversize files.
|
||||
#[test]
|
||||
fn empty_file_capacity_zero() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("empty.iso");
|
||||
std::fs::File::create(&path).unwrap();
|
||||
let src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), 0);
|
||||
}
|
||||
|
||||
/// Opening a nonexistent path returns an IoError (NotFound), not a
|
||||
/// panic. Grounding: `File::open(path).map_err(...)`.
|
||||
#[test]
|
||||
fn open_missing_file_errors() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("does-not-exist.iso");
|
||||
let err = match FileSectorSource::open(&path) {
|
||||
Ok(_) => panic!("missing file must error"),
|
||||
Err(e) => e,
|
||||
};
|
||||
let io: std::io::Error = err.into();
|
||||
assert_eq!(io.kind(), std::io::ErrorKind::NotFound);
|
||||
}
|
||||
|
||||
/// A DONTNEED drop crossing the chunk threshold must not corrupt or
|
||||
/// short subsequent reads — the eviction is a pure page-cache hint.
|
||||
/// We read past the DEFAULT 32 MiB drop chunk (16384 sectors) so the
|
||||
/// eviction block fires at least once, asserting every sector still
|
||||
/// reads correctly. (Avoids mutating FREEMKV_READ_DROP_CHUNK_MIB to
|
||||
/// sidestep a parallel-test env race with `drop_chunk_size_env_override`.)
|
||||
/// Grounding: the `bytes_read_since_drop >= drop_chunk_bytes`
|
||||
/// eviction block calls only `platform::drop_window` (advisory) and
|
||||
/// resets counters — no data effect.
|
||||
#[test]
|
||||
fn dontneed_eviction_does_not_affect_data() {
|
||||
// 32 MiB default chunk = 16384 sectors; read a bit past it.
|
||||
let total = (READ_DROP_CHUNK_BYTES_DEFAULT / SECTOR_BYTES_U64) as u32 + 64;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("drop.iso");
|
||||
make_iso(&path, total);
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
// Read in 16-sector batches to keep the loop fast while still
|
||||
// crossing the drop boundary by byte count.
|
||||
let batch = 16u16;
|
||||
let mut got = vec![0u8; batch as usize * SECTOR_BYTES];
|
||||
let mut lba = 0u32;
|
||||
while lba + batch as u32 <= total {
|
||||
src.read_sectors(lba, batch, &mut got, false).unwrap();
|
||||
for i in 0..batch as u32 {
|
||||
let expected = ((lba + i) & 0xff) as u8;
|
||||
let off = i as usize * SECTOR_BYTES;
|
||||
assert!(
|
||||
got[off..off + SECTOR_BYTES].iter().all(|x| *x == expected),
|
||||
"DONTNEED eviction corrupted sector {}",
|
||||
lba + i
|
||||
);
|
||||
}
|
||||
lba += batch as u32;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
//! Fallback for targets without a known sequential-readahead hint
|
||||
//! (BSDs, illumos, etc.). No-op — reads still work, they just don't
|
||||
//! get the OS-level prefetch widening.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn hint_sequential(_file: &File, _len_bytes: u64) {}
|
||||
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
|
||||
pub(super) fn prefetch(_file: &File, _offset: u64, _len: u64) {}
|
||||
@@ -0,0 +1,28 @@
|
||||
//! Windows: the canonical sequential-access hint is
|
||||
//! `FILE_FLAG_SEQUENTIAL_SCAN`, which must be passed to `CreateFile`
|
||||
//! at open time and cannot be set afterward via
|
||||
//! `SetFileInformationByHandle`. Since `FileSectorSource::open` uses a
|
||||
//! plain `File::open`, the hints in this module are no-op stubs.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
/// No-op stub. `FILE_FLAG_SEQUENTIAL_SCAN` can only be set at
|
||||
/// `CreateFile` open time, which the plain `File::open` path does not
|
||||
/// do, so there is no post-open hint to issue here.
|
||||
pub(super) fn hint_sequential(_file: &File, _len_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"FileSectorSource hint_sequential: windows no-op stub"
|
||||
);
|
||||
}
|
||||
|
||||
/// Windows page-cache eviction is not exposed via a posix_fadvise
|
||||
/// equivalent. The kernel does its own working-set management. No-op
|
||||
/// for now.
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
|
||||
/// Windows async-prefetch hint. With FILE_FLAG_SEQUENTIAL_SCAN at
|
||||
/// open the kernel already prefetches aggressively, so there's no
|
||||
/// per-range hint we'd add on top. No-op stub for parity with the
|
||||
/// posix platforms.
|
||||
pub(super) fn prefetch(_file: &File, _offset: u64, _len: u64) {}
|
||||
@@ -0,0 +1,92 @@
|
||||
//! Platform-aware crash-durability primitives.
|
||||
//!
|
||||
//! Two flush operations need OS-specific handling to make a write survive a
|
||||
//! crash / power loss:
|
||||
//!
|
||||
//! - [`dir`] — fsync a directory so a prior `rename(2)` into it is durable.
|
||||
//! After a crash a renamed file's dirent can otherwise be lost even though
|
||||
//! the rename returned, because it is still page-cache-only. This is a POSIX
|
||||
//! concept: on Windows std cannot even open a directory as a `File` (it does
|
||||
//! not set `FILE_FLAG_BACKUP_SEMANTICS`), and NTFS/ReFS commit the rename's
|
||||
//! dirent without an explicit directory flush — so it is a no-op there
|
||||
//! rather than a failed open that logs on every marker write.
|
||||
//!
|
||||
//! - [`file_durable`] — fsync a file's contents + metadata. Opens the file
|
||||
//! **read+write**: on Windows `File::sync_all` maps to `FlushFileBuffers`,
|
||||
//! which requires a handle with write access and returns
|
||||
//! `ERROR_ACCESS_DENIED` (os error 5) on a read-only handle. (A read-only
|
||||
//! `File::open` + `sync_all` is legal on POSIX, which is why that bug only
|
||||
//! bit Windows.) The open mode is platform-uniform, so this lives here with
|
||||
//! no dispatch.
|
||||
//!
|
||||
//! Per the crate convention (see [`crate::io::writeback_file`]), platform
|
||||
//! dispatch happens once here via cfg-gated `mod` decls — callers carry no
|
||||
//! inline `#[cfg(...)]`.
|
||||
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
#[cfg(not(windows))]
|
||||
mod posix;
|
||||
#[cfg(windows)]
|
||||
mod windows;
|
||||
|
||||
#[cfg(not(windows))]
|
||||
use posix as platform;
|
||||
#[cfg(windows)]
|
||||
use windows as platform;
|
||||
|
||||
/// fsync a directory so a prior `rename(2)` into it is durable. Best-effort:
|
||||
/// failures are logged and swallowed, never propagated — the renamed file's
|
||||
/// bytes are already synced and the caller's write itself succeeded. No-op on
|
||||
/// Windows (see module docs).
|
||||
pub fn dir(path: &Path) {
|
||||
platform::fsync_dir(path)
|
||||
}
|
||||
|
||||
/// Durably flush an existing file's contents + metadata to stable storage.
|
||||
///
|
||||
/// Opens the file read+write (not read-only) so the flush succeeds on every
|
||||
/// platform — see the module docs for the Windows `FlushFileBuffers` rationale.
|
||||
/// The file must already exist; its bytes are left intact (no create/truncate).
|
||||
pub fn file_durable(path: &Path) -> io::Result<()> {
|
||||
let f = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(path)?;
|
||||
f.sync_all()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// `file_durable` opens read+write (so the flush works on Windows) and
|
||||
/// syncs an existing file; a missing path surfaces as `Err` so the caller
|
||||
/// treats it as "not durably synced". Platform-uniform — same on
|
||||
/// unix/windows.
|
||||
#[test]
|
||||
fn file_durable_ok_for_existing_err_for_missing() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
let f = td.path().join("data.bin");
|
||||
std::fs::write(&f, b"durable").unwrap();
|
||||
assert!(
|
||||
file_durable(&f).is_ok(),
|
||||
"an existing file must open read+write and fsync cleanly"
|
||||
);
|
||||
assert!(
|
||||
file_durable(&td.path().join("absent.bin")).is_err(),
|
||||
"a missing file must surface the open failure as Err"
|
||||
);
|
||||
}
|
||||
|
||||
/// `dir` is best-effort: it must return normally for a real directory
|
||||
/// (POSIX fsyncs it, Windows no-ops) and must swallow — never panic on —
|
||||
/// a missing directory.
|
||||
#[test]
|
||||
fn dir_is_best_effort_never_panics() {
|
||||
let td = tempfile::tempdir().unwrap();
|
||||
dir(td.path());
|
||||
dir(&td.path().join("does-not-exist"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//! POSIX directory-fsync. Active on unix and any non-Windows fallback target
|
||||
//! (BSD, illumos, …) — all share the same `File::open(dir).sync_all()`
|
||||
//! semantics. The Windows no-op lives in the sibling `windows` module.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
pub(super) fn fsync_dir(dir: &Path) {
|
||||
match std::fs::File::open(dir) {
|
||||
Ok(f) => {
|
||||
if let Err(e) = f.sync_all() {
|
||||
tracing::warn!(path = %dir.display(), error = %e, "failed to fsync directory");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(path = %dir.display(), error = %e, "could not open directory to fsync");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
//! Windows directory-fsync: a no-op.
|
||||
//!
|
||||
//! Directory fsync is a POSIX concept. std cannot open a directory as a `File`
|
||||
//! on Windows (it does not set `FILE_FLAG_BACKUP_SEMANTICS`), so the POSIX impl
|
||||
//! could only ever fail the open and log a spurious warning on every marker /
|
||||
//! mapfile write. NTFS/ReFS commit a rename's directory entry without an
|
||||
//! explicit directory flush, so skipping it here is correct — not a durability
|
||||
//! regression. (File-content durability is handled platform-uniformly by
|
||||
//! [`super::file_durable`].)
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
pub(super) fn fsync_dir(_dir: &Path) {}
|
||||
@@ -0,0 +1,48 @@
|
||||
//! File I/O helpers that bound kernel cache pressure on big writes.
|
||||
//!
|
||||
//! `WritebackFile` is a drop-in wrapper around `std::fs::File` for any
|
||||
//! call site that performs large sequential writes (sweep, patch, mux,
|
||||
//! etc.). It implements `Write` and `Seek` so existing code paths can
|
||||
//! swap `File` for `WritebackFile` with no body changes. Internally it
|
||||
//! drives a `WritebackPipeline` that, on Linux, drains dirty pages
|
||||
//! continuously at 32 MB granularity to avoid the kernel's
|
||||
//! accumulate-then-burst flush behaviour. macOS and Windows use a
|
||||
//! no-op pipeline — their default cache policies have not been shown
|
||||
//! to exhibit the same pathology for this access pattern.
|
||||
//!
|
||||
//! `FileSectorSource` is the read-side dual — it implements
|
||||
//! [`crate::sector::SectorSource`] for an ISO file using direct
|
||||
//! `pread`-equivalent calls so the kernel's own readahead policy runs
|
||||
//! (which interleaves naturally with the concurrent writeback). It
|
||||
//! pairs that with periodic `posix_fadvise(DONTNEED)` drops on the
|
||||
//! consumed window so an 85 GB streaming ISO read doesn't fill the
|
||||
//! page cache and starve the concurrent MKV write.
|
||||
//!
|
||||
//! `Pipeline` + `Sink` is the generic producer/consumer primitive
|
||||
//! used by sweep, patch, and mux to overlap reads with writes via a
|
||||
//! bounded channel + dedicated consumer thread.
|
||||
//!
|
||||
//! `byte_prefetcher` is the read-ahead producer feeding the mux
|
||||
//! pipeline for `io::Read`-backed sources: a worker thread fills a
|
||||
//! recycled pool of buffers and ships them through a channel, exposing
|
||||
//! `BytePrefetcher` / `PrefetchShell`.
|
||||
|
||||
pub(crate) mod bounded;
|
||||
pub mod byte_prefetcher;
|
||||
pub mod file_sector_source;
|
||||
pub mod fsync;
|
||||
pub mod sink;
|
||||
mod writeback;
|
||||
mod writeback_file;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
pub(crate) mod platform_macos;
|
||||
|
||||
pub mod pipeline;
|
||||
|
||||
pub(crate) use writeback_file::WritebackFile;
|
||||
|
||||
pub use pipeline::{
|
||||
DEFAULT_PIPELINE_DEPTH, Flow, Pipeline, READ_PIPELINE_DEPTH, Sink, WRITE_PIPELINE_DEPTH,
|
||||
WRITE_THROUGH_DEPTH,
|
||||
};
|
||||
+1595
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,38 @@
|
||||
//! Shared macOS `fcntl(F_PREALLOCATE)` definitions.
|
||||
//!
|
||||
//! The `libc` crate doesn't expose these symbols across all macOS SDK
|
||||
//! versions, so we define them locally with values from
|
||||
//! `/usr/include/sys/fcntl.h`. Two call sites (
|
||||
//! [`crate::io::writeback_file`] and [`crate::io::sink::preallocate`])
|
||||
//! need the same constants and `fstore_t` layout — keeping a single
|
||||
//! source of truth here prevents the two copies from drifting.
|
||||
//!
|
||||
//! Module-level cfg gate lives in the parent (`io/mod.rs`); this file
|
||||
//! is only compiled on macOS, so no inner `#![cfg]` is needed.
|
||||
|
||||
/// `fcntl(F_PREALLOCATE)` command number from `sys/fcntl.h`.
|
||||
pub(crate) const F_PREALLOCATE: libc::c_int = 42;
|
||||
|
||||
/// Anchor preallocation at the current physical EOF.
|
||||
pub(crate) const F_PEOFPOSMODE: libc::c_int = 3;
|
||||
|
||||
/// Prefer a contiguous allocation. Try this first; on `EINVAL` (no
|
||||
/// contiguous run of that size), fall back to `F_ALLOCATEALL`.
|
||||
pub(crate) const F_ALLOCATECONTIG: libc::c_uint = 0x0000_0002;
|
||||
|
||||
/// Allow non-contiguous allocation. Stronger guarantee than just
|
||||
/// asking for `F_ALLOCATECONTIG` because the kernel will piece
|
||||
/// together fragments rather than failing.
|
||||
pub(crate) const F_ALLOCATEALL: libc::c_uint = 0x0000_0004;
|
||||
|
||||
/// `fstore_t` from `sys/fcntl.h`. `repr(C)` because we hand it to
|
||||
/// `fcntl(F_PREALLOCATE)` which writes through the pointer.
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct Fstore {
|
||||
pub fst_flags: libc::c_uint,
|
||||
pub fst_posmode: libc::c_int,
|
||||
pub fst_offset: libc::off_t,
|
||||
pub fst_length: libc::off_t,
|
||||
pub fst_bytesalloc: libc::off_t,
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
//! `LocalFileSink` — `BufWriter<File>` for the common local-disk case.
|
||||
//!
|
||||
//! Buffering: 4 MiB internal `BufWriter`. Sized to coalesce the small
|
||||
//! per-PES writes that come out of the muxer into kernel-page-aligned
|
||||
//! flushes without making the buffer big enough to matter for memory
|
||||
//! pressure on a single concurrent rip.
|
||||
//!
|
||||
//! `Seek` flushes the underlying `BufWriter` first; otherwise a seek
|
||||
//! could leapfrog buffered data and silently corrupt the file. This is
|
||||
//! the same shape `BufWriter` itself uses when it impls `Seek` in
|
||||
//! stdlib, and is necessary for MKV's seek-back operations (cluster
|
||||
//! size patch, Cues index, segment header backpatch) to land on the
|
||||
//! right offset.
|
||||
//!
|
||||
//! [`SequentialSink`](super::SequentialSink) is implemented explicitly
|
||||
//! (not via a blanket impl) so its `finish()` flushes the `BufWriter`
|
||||
//! and `fsync`s the file even when called through a `dyn` trait object;
|
||||
//! [`RandomAccessSink`](super::RandomAccessSink) is implemented over the
|
||||
//! `Seek` impl below.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, BufWriter, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use super::preallocate;
|
||||
use super::{RandomAccessSink, SequentialSink};
|
||||
|
||||
const BUFFER_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Random-access write sink for local disks.
|
||||
///
|
||||
/// Wraps a `BufWriter<File>` with a 4 MiB internal buffer and forwards
|
||||
/// `Write`/`Seek` so any call site that previously held a `File` or
|
||||
/// `WritebackFile` can drop this in. `finish()` flushes the buffer and
|
||||
/// runs `sync_all` on the underlying file so the caller can drop it
|
||||
/// without losing data.
|
||||
///
|
||||
/// Construction always opens the file `create + truncate + read +
|
||||
/// write`. `read` is enabled so the same handle can be reused for a
|
||||
/// verification re-read after the mux (the existing
|
||||
/// `FileSectorSink::create` pattern). On Linux, [`with_size_hint`]
|
||||
/// additionally calls `fallocate(FALLOC_FL_KEEP_SIZE)` to pre-reserve
|
||||
/// extents.
|
||||
///
|
||||
/// [`with_size_hint`]: Self::with_size_hint
|
||||
pub struct LocalFileSink {
|
||||
inner: BufWriter<File>,
|
||||
}
|
||||
|
||||
impl LocalFileSink {
|
||||
/// Open `path` for writing, truncating any existing contents.
|
||||
pub fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
Ok(Self {
|
||||
inner: BufWriter::with_capacity(BUFFER_BYTES, file),
|
||||
})
|
||||
}
|
||||
|
||||
/// Like [`Self::create`] but additionally calls the per-OS
|
||||
/// preallocate path with `size_bytes`. On Linux this is
|
||||
/// `fallocate(FALLOC_FL_KEEP_SIZE)` so the on-disk extents are
|
||||
/// reserved up front (reducing fragmentation for big sequential
|
||||
/// muxer output); on other OSes it is a no-op today. Failures
|
||||
/// from the preallocate call are non-fatal — the file is still
|
||||
/// returned, just without the size reservation.
|
||||
pub fn with_size_hint(path: &Path, size_bytes: u64) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
preallocate::preallocate(&file, size_bytes);
|
||||
Ok(Self {
|
||||
inner: BufWriter::with_capacity(BUFFER_BYTES, file),
|
||||
})
|
||||
}
|
||||
|
||||
/// Drain the internal buffer and `fsync` the underlying file.
|
||||
/// Idempotent with `Drop` (the `BufWriter` also flushes on drop;
|
||||
/// this call additionally surfaces fsync errors to the caller).
|
||||
/// [`SequentialSink::finish`](super::SequentialSink::finish)
|
||||
/// delegates here so the durable flush happens through a trait
|
||||
/// object too.
|
||||
pub fn sync_all(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()?;
|
||||
self.inner.get_ref().sync_all()
|
||||
}
|
||||
}
|
||||
|
||||
impl SequentialSink for LocalFileSink {
|
||||
/// Flush the 4 MiB `BufWriter` and `fsync` the file. Overriding the
|
||||
/// trait default is what makes a `dyn SequentialSink` / `dyn
|
||||
/// RandomAccessSink` `finish()` actually durable instead of a no-op.
|
||||
fn finish(&mut self) -> io::Result<()> {
|
||||
self.sync_all()
|
||||
}
|
||||
}
|
||||
|
||||
impl RandomAccessSink for LocalFileSink {}
|
||||
|
||||
impl Write for LocalFileSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.inner.write(buf)
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.inner.write_all(buf)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for LocalFileSink {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
// Flush before seeking so buffered bytes land at the offset
|
||||
// they were written for, not the new one.
|
||||
self.inner.flush()?;
|
||||
self.inner.get_mut().seek(from)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
|
||||
#[test]
|
||||
fn write_seek_roundtrip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("rt.bin");
|
||||
let mut s = LocalFileSink::create(&p).unwrap();
|
||||
s.write_all(b"AAAA").unwrap();
|
||||
s.write_all(b"BBBB").unwrap();
|
||||
// Seek back over the second word and overwrite.
|
||||
s.seek(SeekFrom::Start(4)).unwrap();
|
||||
s.write_all(b"CCCC").unwrap();
|
||||
s.sync_all().unwrap();
|
||||
drop(s);
|
||||
|
||||
let mut f = File::open(&p).unwrap();
|
||||
let mut got = Vec::new();
|
||||
f.read_to_end(&mut got).unwrap();
|
||||
assert_eq!(&got[..], b"AAAACCCC");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_flushes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("drop.bin");
|
||||
{
|
||||
let mut s = LocalFileSink::create(&p).unwrap();
|
||||
s.write_all(b"buffered").unwrap();
|
||||
// No explicit flush / sync_all — BufWriter drop runs the
|
||||
// flush and the file should land on disk.
|
||||
}
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..], b"buffered");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_size_hint_creates_writable_file() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("sz.bin");
|
||||
let mut s = LocalFileSink::with_size_hint(&p, 64 * 1024).unwrap();
|
||||
s.write_all(b"hint-ok").unwrap();
|
||||
s.sync_all().unwrap();
|
||||
drop(s);
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..], b"hint-ok");
|
||||
}
|
||||
|
||||
// ── Added hardening tests ───────────────────────────────────────
|
||||
|
||||
/// `create` must TRUNCATE an existing file (OpenOptions
|
||||
/// `.truncate(true)`, lines 52-58). Pre-seed a long file, recreate
|
||||
/// it via the sink, write a shorter payload — the old tail must be
|
||||
/// gone. Mutation: dropping `.truncate(true)` would leave the stale
|
||||
/// tail and the length assert fails.
|
||||
#[test]
|
||||
fn create_truncates_existing_file() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("trunc.bin");
|
||||
std::fs::write(&p, vec![0xFFu8; 4096]).unwrap();
|
||||
let mut s = LocalFileSink::create(&p).unwrap();
|
||||
s.write_all(b"short").unwrap();
|
||||
s.sync_all().unwrap();
|
||||
drop(s);
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(
|
||||
bytes.len(),
|
||||
5,
|
||||
"create must truncate the pre-existing 4096 bytes"
|
||||
);
|
||||
assert_eq!(&bytes, b"short");
|
||||
}
|
||||
|
||||
/// Seek must flush the BufWriter FIRST so buffered bytes land at
|
||||
/// their intended offset, not the post-seek one (lines 121-128, and
|
||||
/// the module doc's silent-corruption warning). We write into the
|
||||
/// buffer (no explicit flush), seek backward, write again, and
|
||||
/// confirm the first write stayed at offset 0. Mutation: removing
|
||||
/// the `self.inner.flush()?` in `seek` would flush the first 4
|
||||
/// bytes at the seeked offset, corrupting the file.
|
||||
#[test]
|
||||
fn seek_flushes_buffer_before_moving() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("seek-flush.bin");
|
||||
let mut s = LocalFileSink::create(&p).unwrap();
|
||||
// These bytes sit in the 4 MiB BufWriter, unflushed.
|
||||
s.write_all(b"HEAD").unwrap();
|
||||
// Seek forward to offset 10; the buffered HEAD must be flushed
|
||||
// to offset 0 BEFORE the position moves.
|
||||
s.seek(SeekFrom::Start(10)).unwrap();
|
||||
s.write_all(b"TAIL").unwrap();
|
||||
s.sync_all().unwrap();
|
||||
drop(s);
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(
|
||||
&bytes[0..4],
|
||||
b"HEAD",
|
||||
"buffered head landed at the wrong offset"
|
||||
);
|
||||
assert_eq!(&bytes[10..14], b"TAIL");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
//! Output-sink trait split for the buffering architecture.
|
||||
//!
|
||||
//! Two traits, one for each capability axis of an output destination:
|
||||
//!
|
||||
//! - [`SequentialSink`] — anything you can `Write` to in order. Sockets,
|
||||
//! pipes, append-only stores, plain files. Containers that don't need
|
||||
//! seek (M2TS, fMP4, HEVC elementary) target this.
|
||||
//! - [`RandomAccessSink`] — everything `SequentialSink` plus a working
|
||||
//! `Seek`. Local files, NFS files, anything with random-write
|
||||
//! semantics. Containers that need backpatch (MKV cluster sizes, Cues
|
||||
//! index, MP4 moov-at-end) target this.
|
||||
//!
|
||||
//! `RandomAccessSink: SequentialSink` — every random-access sink is
|
||||
//! also a valid sequential sink. The muxer is generic over which it
|
||||
//! requires (`MkvMux<S: RandomAccessSink>`, `M2tsMux<S: SequentialSink>`)
|
||||
//! so an attempt to mux MKV to a network socket is a compile error.
|
||||
//!
|
||||
//! Buffering policy belongs to the concrete sink, not to a wrapper at
|
||||
//! the call site. `LocalFileSink` wraps a `BufWriter<File>` with a
|
||||
//! 4 MiB buffer for the common local-disk case; `WritebackFile`
|
||||
//! (separate module) wraps a `File` with the adaptive-chunk
|
||||
//! `sync_file_range` machinery for the Linux+NFS case.
|
||||
|
||||
use std::io::{Seek, Write};
|
||||
|
||||
mod local_file;
|
||||
mod preallocate;
|
||||
mod socket;
|
||||
|
||||
pub use local_file::LocalFileSink;
|
||||
pub use socket::{SocketSink, UdpSocketSink};
|
||||
|
||||
/// Sequential-only write destination. Sockets, pipes, append-only
|
||||
/// stores. No seek. Implementations own their write buffering — the
|
||||
/// trait does not impose or hide any buffering of its own.
|
||||
///
|
||||
/// `finish` drains any internal buffering and signals end-of-stream to
|
||||
/// the underlying transport (close-write on a socket, flush + fsync on
|
||||
/// a buffered file, etc.). The default impl flushes via [`Write::flush`]
|
||||
/// — correct for an unbuffered destination — but every concrete sink in
|
||||
/// this module overrides it to drain its own buffer and run its
|
||||
/// transport-specific finalisation (socket `shutdown(Write)`, file
|
||||
/// `fsync`). There is deliberately NO blanket `impl SequentialSink for
|
||||
/// T`: a blanket impl would force the no-op-style default on every
|
||||
/// concrete sink (a blanket impl cannot be overridden per-type without a
|
||||
/// coherence conflict), so a `Box<dyn SequentialSink>` / `&mut dyn
|
||||
/// SequentialSink` `finish()` call would silently skip the flush and
|
||||
/// transport shutdown. With explicit per-type impls the vtable dispatches
|
||||
/// `finish` to the real implementation, so flush + durable-finish
|
||||
/// actually happen through a trait object.
|
||||
pub trait SequentialSink: Write + Send {
|
||||
fn finish(&mut self) -> std::io::Result<()> {
|
||||
self.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Random-access write destination. Local files, NFS files, anything
|
||||
/// with a working `Seek`. Inherits the `SequentialSink` contract — a
|
||||
/// random-access sink is always usable as a sequential sink.
|
||||
pub trait RandomAccessSink: SequentialSink + Seek {}
|
||||
|
||||
/// Pick the right `RandomAccessSink` impl for `dest` based on its
|
||||
/// filesystem type.
|
||||
///
|
||||
/// - Linux + NFS path → `WritebackFile` with its adaptive-chunk
|
||||
/// sync_file_range machinery and (when supported) `fallocate` size
|
||||
/// hint.
|
||||
/// - everything else → [`LocalFileSink`] over `BufWriter<File>`. On
|
||||
/// non-Linux there is no `WritebackFile` machinery to opt into, and
|
||||
/// on local Linux the kernel's default writeback policy is already
|
||||
/// fine.
|
||||
///
|
||||
/// `size_hint`, when present, is forwarded to the per-OS preallocate
|
||||
/// path (`fallocate(KEEP_SIZE)` on Linux, `F_PREALLOCATE` on macOS when
|
||||
/// implemented, no-op elsewhere).
|
||||
///
|
||||
/// Returns a boxed trait object so the call site (mux construction)
|
||||
/// stays agnostic of which concrete sink got picked.
|
||||
// Not yet wired into mux::resolve (follow-up commit). Kept `pub(crate)` until
|
||||
// then so an unfinished signature isn't frozen into the public 1.0 API.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn open_for_mkv(
|
||||
dest: &std::path::Path,
|
||||
size_hint: Option<u64>,
|
||||
) -> std::io::Result<Box<dyn RandomAccessSink>> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use crate::platform::fs_type::{FsType, detect};
|
||||
if detect(dest) == FsType::Nfs {
|
||||
let wf = match size_hint {
|
||||
Some(n) => crate::io::WritebackFile::create_with_size_hint(dest, n)?,
|
||||
None => crate::io::WritebackFile::create(dest)?,
|
||||
};
|
||||
return Ok(Box::new(wf));
|
||||
}
|
||||
}
|
||||
// Only Linux differentiates the sink by filesystem type (NFS gets
|
||||
// the WritebackFile machinery); every other OS always uses
|
||||
// `LocalFileSink`. Reference `detect` as a value (no call, no
|
||||
// `statfs` syscall) so it isn't flagged dead on non-Linux while
|
||||
// still avoiding the wasted probe whose result we'd discard.
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let _ = crate::platform::fs_type::detect;
|
||||
|
||||
let sink = match size_hint {
|
||||
Some(n) => LocalFileSink::with_size_hint(dest, n)?,
|
||||
None => LocalFileSink::create(dest)?,
|
||||
};
|
||||
Ok(Box::new(sink))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// Type-level assertion: the concrete sinks satisfy the trait
|
||||
// objects. These functions never run; they just have to type-check.
|
||||
fn _assert_is_sequential(_: &mut dyn SequentialSink) {}
|
||||
fn _assert_is_random_access(_: &mut dyn RandomAccessSink) {}
|
||||
|
||||
#[test]
|
||||
fn concrete_sinks_satisfy_traits() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
// `LocalFileSink` is a random-access (and thus sequential) sink.
|
||||
let mut s = LocalFileSink::create(&dir.path().join("b.bin")).unwrap();
|
||||
_assert_is_sequential(&mut s);
|
||||
_assert_is_random_access(&mut s);
|
||||
|
||||
// `WritebackFile` ditto, via its explicit per-type impls.
|
||||
let mut wf = crate::io::WritebackFile::create(&dir.path().join("c.bin")).unwrap();
|
||||
_assert_is_sequential(&mut wf);
|
||||
_assert_is_random_access(&mut wf);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_for_mkv_returns_a_random_access_sink() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("c.bin");
|
||||
let mut sink = open_for_mkv(&p, Some(64 * 1024)).unwrap();
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
sink.write_all(b"hello").unwrap();
|
||||
sink.seek(SeekFrom::Start(0)).unwrap();
|
||||
sink.finish().unwrap();
|
||||
drop(sink);
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..5], b"hello");
|
||||
}
|
||||
|
||||
/// finish() through a `dyn SequentialSink` trait object must
|
||||
/// dispatch to the concrete sink's override (flush + fsync), not a
|
||||
/// no-op default. This is the regression test for the silent-no-op
|
||||
/// finish() bug.
|
||||
#[test]
|
||||
fn finish_through_trait_object_flushes_local_file() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("trait-finish.bin");
|
||||
let sink = LocalFileSink::create(&p).unwrap();
|
||||
// Box as the trait object the production path uses.
|
||||
let mut boxed: Box<dyn SequentialSink> = Box::new(sink);
|
||||
boxed.write_all(b"buffered-tail").unwrap();
|
||||
// finish() through the vtable must drain the 4 MiB BufWriter and
|
||||
// fsync; the bytes must be visible to a separate reader BEFORE
|
||||
// we drop the sink (drop-flush must not be what saves us).
|
||||
boxed.finish().unwrap();
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..], b"buffered-tail");
|
||||
}
|
||||
|
||||
// ── Added hardening tests ───────────────────────────────────────
|
||||
|
||||
use std::io::{self, Write};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
/// A minimal `SequentialSink` that does NOT override `finish`, so it
|
||||
/// exercises the trait's DEFAULT impl (lines 51-55), which must call
|
||||
/// `Write::flush`. We record whether flush ran. This pins the
|
||||
/// documented contract that the default `finish` is "correct for an
|
||||
/// unbuffered destination" by flushing. Mutation: changing the
|
||||
/// default `finish` body from `self.flush()` to `Ok(())` would set
|
||||
/// `flushed=false` and fail.
|
||||
struct FlushTracker {
|
||||
flushed: Arc<AtomicBool>,
|
||||
bytes: Arc<AtomicUsize>,
|
||||
}
|
||||
impl Write for FlushTracker {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.bytes.fetch_add(buf.len(), Ordering::SeqCst);
|
||||
Ok(buf.len())
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.flushed.store(true, Ordering::SeqCst);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
// Uses the DEFAULT finish() — deliberately no override.
|
||||
impl SequentialSink for FlushTracker {}
|
||||
|
||||
#[test]
|
||||
fn default_finish_flushes() {
|
||||
let flushed = Arc::new(AtomicBool::new(false));
|
||||
let bytes = Arc::new(AtomicUsize::new(0));
|
||||
let mut sink = FlushTracker {
|
||||
flushed: flushed.clone(),
|
||||
bytes: bytes.clone(),
|
||||
};
|
||||
sink.write_all(b"abc").unwrap();
|
||||
assert!(
|
||||
!flushed.load(Ordering::SeqCst),
|
||||
"flush should not run before finish"
|
||||
);
|
||||
sink.finish().unwrap();
|
||||
assert!(
|
||||
flushed.load(Ordering::SeqCst),
|
||||
"default SequentialSink::finish must call Write::flush"
|
||||
);
|
||||
assert_eq!(bytes.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
|
||||
/// `open_for_mkv` with `None` size hint must still produce a working
|
||||
/// random-access sink (the `match size_hint { None => ... }` arm,
|
||||
/// lines 103-106). Round-trip a seek-back patch through it to prove
|
||||
/// both Write and Seek dispatch. Mutation: if the None arm returned
|
||||
/// a sequential-only sink the seek would not compile / would fail.
|
||||
#[test]
|
||||
fn open_for_mkv_without_size_hint_is_random_access() {
|
||||
use std::io::{Seek, SeekFrom};
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("nohint.bin");
|
||||
let mut sink = open_for_mkv(&p, None).unwrap();
|
||||
sink.write_all(b"AAAABBBB").unwrap();
|
||||
sink.seek(SeekFrom::Start(4)).unwrap();
|
||||
sink.write_all(b"CCCC").unwrap();
|
||||
sink.finish().unwrap();
|
||||
drop(sink);
|
||||
assert_eq!(std::fs::read(&p).unwrap(), b"AAAACCCC");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//! Linux `fallocate(FALLOC_FL_KEEP_SIZE)` preallocation.
|
||||
//!
|
||||
//! `KEEP_SIZE` reserves extents without changing the apparent file
|
||||
//! length, which matches the muxer's expectation that writes still grow
|
||||
//! the file naturally.
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
pub(super) fn preallocate_impl(file: &File, size_bytes: u64) {
|
||||
let fd = file.as_raw_fd();
|
||||
// Clamp to the signed `off_t` range fallocate expects; an unchecked
|
||||
// `as i64` cast would wrap a >= 2^63 size to a negative length that
|
||||
// fallocate rejects with EINVAL (silent no-op).
|
||||
let len = i64::try_from(size_bytes).unwrap_or(i64::MAX);
|
||||
// FALLOC_FL_KEEP_SIZE = 0x01.
|
||||
let rc = unsafe { libc::fallocate(fd, libc::FALLOC_FL_KEEP_SIZE, 0, len) };
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink fallocate size_hint={size_bytes} rc={rc} ok={}",
|
||||
rc == 0
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
//! macOS `F_PREALLOCATE` extent reservation.
|
||||
//!
|
||||
//! `fcntl(F_PREALLOCATE)` with `F_ALLOCATECONTIG | F_ALLOCATEALL` first
|
||||
//! (prefer a contiguous run but accept scattered extents to satisfy the
|
||||
//! full length) and fall back to `F_ALLOCATEALL` alone on failure.
|
||||
//! Reported file size is unchanged — the muxer's writes still grow it.
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
use crate::io::platform_macos::{
|
||||
F_ALLOCATEALL, F_ALLOCATECONTIG, F_PEOFPOSMODE, F_PREALLOCATE, Fstore,
|
||||
};
|
||||
|
||||
pub(super) fn preallocate_impl(file: &File, size_bytes: u64) {
|
||||
let fd = file.as_raw_fd();
|
||||
// Clamp to the signed `off_t` range; an unchecked `as off_t` cast
|
||||
// would wrap a >= 2^63 size to a negative length.
|
||||
let len = i64::try_from(size_bytes).unwrap_or(i64::MAX) as libc::off_t;
|
||||
let mut store = Fstore {
|
||||
// Prefer a contiguous run but accept scattered extents to
|
||||
// satisfy the full length. Without F_ALLOCATEALL the first
|
||||
// attempt is best-effort and can return rc=0 with a partial
|
||||
// allocation, so the fallback below would never fire. Matches
|
||||
// writeback_file/macos.rs.
|
||||
fst_flags: F_ALLOCATECONTIG | F_ALLOCATEALL,
|
||||
fst_posmode: F_PEOFPOSMODE,
|
||||
fst_offset: 0,
|
||||
fst_length: len,
|
||||
fst_bytesalloc: 0,
|
||||
};
|
||||
let mut rc = unsafe { libc::fcntl(fd, F_PREALLOCATE, &mut store as *mut Fstore) };
|
||||
if rc == -1 {
|
||||
// Fall back to non-contiguous only.
|
||||
store.fst_flags = F_ALLOCATEALL;
|
||||
rc = unsafe { libc::fcntl(fd, F_PREALLOCATE, &mut store as *mut Fstore) };
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink F_PREALLOCATE size_hint={size_bytes} rc={rc} bytesalloc={}",
|
||||
store.fst_bytesalloc
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! Per-OS extent preallocation. Best-effort; failures are logged at
|
||||
//! debug and otherwise swallowed because the file is still usable
|
||||
//! without the size reservation — only large-file fragmentation gets
|
||||
//! marginally worse.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
mod other;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux::preallocate_impl;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos::preallocate_impl;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
use other::preallocate_impl;
|
||||
|
||||
/// Reserve `size_bytes` of disk space for `file`'s on-disk extents.
|
||||
/// Reported file size is unchanged — writes still grow the file
|
||||
/// naturally; only the allocator's extent map is primed.
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
preallocate_impl(file, size_bytes);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
//! Fallback preallocate impl. No-op.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn preallocate_impl(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink preallocate size_hint={size_bytes} skipped (no platform impl)"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
//! TCP / UDP socket sinks (sequential-only).
|
||||
//!
|
||||
//! [`SocketSink`] wraps a `TcpStream` in a 1 MiB `BufWriter`. Constructor
|
||||
//! tunes `SO_SNDBUF` to a caller hint when provided. `finish()` flushes
|
||||
//! the buffer then `shutdown(Write)`s the socket so the peer sees clean
|
||||
//! end-of-stream.
|
||||
//!
|
||||
//! [`UdpSocketSink`] wraps a connected `UdpSocket`. Each `write` call
|
||||
//! emits exactly one datagram — the caller is responsible for packetizing
|
||||
//! to a reasonable MTU (188 × 7 = 1316 bytes for MPEG-TS-over-UDP is the
|
||||
//! conventional choice). `finish()` is a no-op; UDP has no end-of-stream
|
||||
//! marker.
|
||||
//!
|
||||
//! Both types implement [`SequentialSink`] explicitly so their
|
||||
//! `finish()` dispatches correctly through a `dyn SequentialSink` trait
|
||||
//! object (the `SocketSink` override drains the buffer and
|
||||
//! `shutdown(Write)`s; the `UdpSocketSink` override flushes only).
|
||||
//! Neither implements `Seek`, so neither satisfies [`RandomAccessSink`]
|
||||
//! — using one with `MkvMux` is a compile error, which is the design
|
||||
//! intent.
|
||||
//!
|
||||
//! [`SequentialSink`]: super::SequentialSink
|
||||
//! [`RandomAccessSink`]: super::RandomAccessSink
|
||||
|
||||
use std::io::{self, BufWriter, Write};
|
||||
use std::net::{Shutdown, SocketAddr, TcpStream, ToSocketAddrs, UdpSocket};
|
||||
|
||||
use super::SequentialSink;
|
||||
|
||||
/// `BufWriter` capacity for [`SocketSink`]. 1 MiB matches the typical
|
||||
/// kernel send-buffer ceiling and keeps small-write amplification from
|
||||
/// containers (TS = 188-byte packets, fMP4 fragment headers = ~100 bytes)
|
||||
/// from translating into syscall storms.
|
||||
const TCP_BUF_CAPACITY: usize = 1024 * 1024;
|
||||
|
||||
/// Sequential-only sink over a TCP connection.
|
||||
///
|
||||
/// Wraps a `BufWriter<TcpStream>`; the inner `TcpStream` is kept as a
|
||||
/// clone so [`finish`](Self::finish) can call `shutdown(Write)` after
|
||||
/// flushing the buffer (the buffered writer doesn't expose the socket
|
||||
/// directly).
|
||||
pub struct SocketSink {
|
||||
/// Buffered write half. All payload bytes go through this.
|
||||
buf: BufWriter<TcpStream>,
|
||||
/// Shutdown handle — clone of the socket inside `buf`. Used only by
|
||||
/// `finish()` for `shutdown(Write)`; never read or written through.
|
||||
shutdown_handle: TcpStream,
|
||||
}
|
||||
|
||||
impl SocketSink {
|
||||
/// Open a TCP connection to `addr` and wrap it for sequential
|
||||
/// writing. `sndbuf_bytes`, when present, is forwarded to
|
||||
/// `setsockopt(SO_SNDBUF)` as a kernel hint — the OS may clamp it.
|
||||
///
|
||||
/// `addr` accepts anything `ToSocketAddrs` does: `"192.0.2.1:1234"`,
|
||||
/// `("host", 1234)`, a `SocketAddr`, etc.
|
||||
pub fn connect<A: ToSocketAddrs>(addr: A, sndbuf_bytes: Option<usize>) -> io::Result<Self> {
|
||||
let stream = TcpStream::connect(addr)?;
|
||||
// `set_nodelay(true)` keeps small writes (TS packet trains, fMP4
|
||||
// moof headers) from sitting in Nagle's algorithm until the buffer
|
||||
// fills. The BufWriter already absorbs syscall overhead; Nagle
|
||||
// would just add latency without coalescing more. It is a latency
|
||||
// hint, not a correctness requirement, so a platform that rejects
|
||||
// TCP_NODELAY must not fail the connect — demote the error.
|
||||
let _ = stream.set_nodelay(true);
|
||||
if let Some(n) = sndbuf_bytes {
|
||||
set_send_buffer(&stream, n)?;
|
||||
}
|
||||
let shutdown_handle = stream.try_clone()?;
|
||||
Ok(Self {
|
||||
buf: BufWriter::with_capacity(TCP_BUF_CAPACITY, stream),
|
||||
shutdown_handle,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for SocketSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.buf.write(buf)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.buf.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl SequentialSink for SocketSink {
|
||||
/// Drain the BufWriter and `shutdown(Write)` the underlying socket
|
||||
/// so the peer sees a clean EOF. Overriding the trait default is
|
||||
/// what makes a `dyn SequentialSink` `finish()` send the buffered
|
||||
/// tail and the EOF instead of silently dropping them.
|
||||
fn finish(&mut self) -> io::Result<()> {
|
||||
self.buf.flush()?;
|
||||
// `shutdown(Write)` signals clean EOF to the peer. Errors here
|
||||
// are non-fatal — the connection may have already been torn down
|
||||
// by the peer — but we surface them so callers can log.
|
||||
self.shutdown_handle.shutdown(Shutdown::Write)
|
||||
}
|
||||
}
|
||||
|
||||
/// Sequential-only sink over a connected UDP socket.
|
||||
///
|
||||
/// Each [`write`](Write::write) call sends exactly one datagram. The
|
||||
/// caller is responsible for splitting payload at packet boundaries —
|
||||
/// for MPEG-TS this means 7 × 188 = 1316 bytes per datagram, the
|
||||
/// industry standard for MPEG-TS-over-UDP. No buffering happens here;
|
||||
/// adding it would silently merge datagrams.
|
||||
///
|
||||
/// `finish()` is a no-op: UDP has no end-of-stream marker. Closing the
|
||||
/// socket happens on drop.
|
||||
pub struct UdpSocketSink {
|
||||
socket: UdpSocket,
|
||||
}
|
||||
|
||||
impl UdpSocketSink {
|
||||
/// Bind a local UDP socket to an ephemeral port and `connect` it to
|
||||
/// `peer`. `connect` doesn't open a connection — it just fixes the
|
||||
/// peer address so subsequent `send` calls don't need to repeat it,
|
||||
/// and so receive-side filtering rejects packets from other sources.
|
||||
///
|
||||
/// `sndbuf_bytes`, when present, is a hint to `SO_SNDBUF`.
|
||||
pub fn connect<A: ToSocketAddrs>(peer: A, sndbuf_bytes: Option<usize>) -> io::Result<Self> {
|
||||
// Resolve the peer first so the local bind matches its address
|
||||
// family. Binding `0.0.0.0:0` (IPv4) and then connecting to an
|
||||
// IPv6 peer fails with EAFNOSUPPORT, so pick the wildcard that
|
||||
// matches the resolved family.
|
||||
let peer_addr = peer
|
||||
.to_socket_addrs()?
|
||||
.next()
|
||||
.ok_or_else(|| io::Error::from(io::ErrorKind::AddrNotAvailable))?;
|
||||
let bind_addr = match peer_addr {
|
||||
SocketAddr::V4(_) => "0.0.0.0:0",
|
||||
SocketAddr::V6(_) => "[::]:0",
|
||||
};
|
||||
// Bind to the matching wildcard / any port. The kernel picks an
|
||||
// ephemeral source port and the source IP at first send.
|
||||
let socket = UdpSocket::bind(bind_addr)?;
|
||||
socket.connect(peer_addr)?;
|
||||
if let Some(n) = sndbuf_bytes {
|
||||
set_udp_send_buffer(&socket, n)?;
|
||||
}
|
||||
Ok(Self { socket })
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for UdpSocketSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
// `send` writes the entire datagram or fails — no partial sends
|
||||
// for UDP. Match `Write::write`'s contract by reporting bytes
|
||||
// accepted.
|
||||
self.socket.send(buf)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl SequentialSink for UdpSocketSink {
|
||||
/// UDP has no end-of-stream marker, so there is nothing to shut
|
||||
/// down; `write` already sent each datagram unbuffered. Flush is a
|
||||
/// no-op but kept explicit so the trait-object `finish()` matches
|
||||
/// the concrete behaviour.
|
||||
fn finish(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ── Platform `SO_SNDBUF` tuning ────────────────────────────────────────────
|
||||
//
|
||||
// std's `TcpStream` / `UdpSocket` don't expose `SO_SNDBUF`. We drop to
|
||||
// libc on Linux + macOS (the libc-dep targets in Cargo.toml). On other
|
||||
// targets the hint is silently ignored — the socket still works, the
|
||||
// kernel just picks its own send-buffer size.
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn set_send_buffer(stream: &TcpStream, bytes: usize) -> io::Result<()> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
setsockopt_sndbuf(stream.as_raw_fd(), bytes)
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn set_udp_send_buffer(socket: &UdpSocket, bytes: usize) -> io::Result<()> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
setsockopt_sndbuf(socket.as_raw_fd(), bytes)
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
fn set_send_buffer(_stream: &TcpStream, _bytes: usize) -> io::Result<()> {
|
||||
// Non-Linux-non-macOS targets aren't in Cargo.toml's libc dep list;
|
||||
// silently ignore the hint rather than failing the connect. Callers
|
||||
// can detect via the lack of an explicit "sndbuf applied" signal
|
||||
// (not provided, intentionally — this is a hint, not a guarantee).
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
fn set_udp_send_buffer(_socket: &UdpSocket, _bytes: usize) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn setsockopt_sndbuf(fd: std::os::unix::io::RawFd, bytes: usize) -> io::Result<()> {
|
||||
// Clamp into c_int range; SO_SNDBUF takes an `int` argument.
|
||||
let want: libc::c_int = bytes.try_into().unwrap_or(libc::c_int::MAX);
|
||||
let ret = unsafe {
|
||||
libc::setsockopt(
|
||||
fd,
|
||||
libc::SOL_SOCKET,
|
||||
libc::SO_SNDBUF,
|
||||
&want as *const _ as *const libc::c_void,
|
||||
std::mem::size_of::<libc::c_int>() as libc::socklen_t,
|
||||
)
|
||||
};
|
||||
if ret != 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
use std::net::{TcpListener, UdpSocket};
|
||||
use std::thread;
|
||||
|
||||
/// Bind a listener, accept on a thread, return (listener_addr,
|
||||
/// accepted-bytes future via JoinHandle).
|
||||
#[test]
|
||||
fn socket_sink_round_trips_bytes() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let accept = thread::spawn(move || {
|
||||
let (mut sock, _) = listener.accept().unwrap();
|
||||
let mut buf = Vec::new();
|
||||
sock.read_to_end(&mut buf).unwrap();
|
||||
buf
|
||||
});
|
||||
|
||||
let mut sink = SocketSink::connect(addr, Some(256 * 1024)).unwrap();
|
||||
// Write enough to overflow the BufWriter at least once, then a
|
||||
// tail that lives in the buffer until `finish` flushes.
|
||||
let big: Vec<u8> = (0..(2 * TCP_BUF_CAPACITY))
|
||||
.map(|i| (i & 0xff) as u8)
|
||||
.collect();
|
||||
sink.write_all(&big).unwrap();
|
||||
sink.write_all(b"tail\n").unwrap();
|
||||
sink.finish().unwrap();
|
||||
drop(sink);
|
||||
|
||||
let received = accept.join().unwrap();
|
||||
assert_eq!(received.len(), big.len() + 5);
|
||||
assert_eq!(&received[..big.len()], &big[..]);
|
||||
assert_eq!(&received[big.len()..], b"tail\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn socket_sink_is_sequential_only() {
|
||||
// Compile-time assertion via dyn — if this ever started
|
||||
// satisfying `RandomAccessSink`, the trait split would be broken.
|
||||
fn _assert_seq(_: &mut dyn super::super::SequentialSink) {}
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let _accept = thread::spawn(move || {
|
||||
let _ = listener.accept();
|
||||
});
|
||||
let mut sink = SocketSink::connect(addr, None).unwrap();
|
||||
_assert_seq(&mut sink);
|
||||
// The negative is harder to assert directly (no `is_not<T>`),
|
||||
// but `SocketSink` does not impl `Seek`, so it can't unify with
|
||||
// `RandomAccessSink`'s super-bound. The Phase 2 blanket impl
|
||||
// `impl<T: SequentialSink + Seek> RandomAccessSink for T {}` thus
|
||||
// excludes it by construction.
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn udp_socket_sink_delivers_datagrams() {
|
||||
let receiver = UdpSocket::bind("127.0.0.1:0").unwrap();
|
||||
receiver
|
||||
.set_read_timeout(Some(std::time::Duration::from_secs(2)))
|
||||
.unwrap();
|
||||
let addr = receiver.local_addr().unwrap();
|
||||
let mut sink = UdpSocketSink::connect(addr, Some(128 * 1024)).unwrap();
|
||||
|
||||
sink.write_all(&[1, 2, 3, 4, 5]).unwrap();
|
||||
sink.write_all(&[9, 9, 9]).unwrap();
|
||||
sink.finish().unwrap();
|
||||
|
||||
let mut buf = [0u8; 64];
|
||||
let n1 = receiver.recv(&mut buf).unwrap();
|
||||
assert_eq!(&buf[..n1], &[1, 2, 3, 4, 5]);
|
||||
let n2 = receiver.recv(&mut buf).unwrap();
|
||||
assert_eq!(&buf[..n2], &[9, 9, 9]);
|
||||
}
|
||||
|
||||
// ── Added hardening tests ───────────────────────────────────────
|
||||
|
||||
/// `SocketSink::finish` must signal a clean EOF to the peer via
|
||||
/// `shutdown(Write)` (lines 91-97). The receiving side's
|
||||
/// `read_to_end` only returns when it observes that EOF — if
|
||||
/// `finish` merely flushed without the shutdown, `read_to_end`
|
||||
/// would block forever (the socket stays half-open). We assert the
|
||||
/// receiver completes promptly AND sees the buffered tail.
|
||||
/// Mutation: replacing the `shutdown(Write)` line with `Ok(())`
|
||||
/// makes the accept thread hang and the join times out.
|
||||
#[test]
|
||||
fn finish_signals_eof_to_peer() {
|
||||
use std::sync::mpsc;
|
||||
use std::time::Duration;
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
thread::spawn(move || {
|
||||
let (mut sock, _) = listener.accept().unwrap();
|
||||
let mut buf = Vec::new();
|
||||
// Returns only when the peer half-closes (shutdown Write).
|
||||
sock.read_to_end(&mut buf).unwrap();
|
||||
let _ = tx.send(buf);
|
||||
});
|
||||
|
||||
let mut sink = SocketSink::connect(addr, None).unwrap();
|
||||
sink.write_all(b"unflushed-tail").unwrap();
|
||||
sink.finish().unwrap();
|
||||
|
||||
// read_to_end must complete because finish() shut down writes.
|
||||
let received = rx
|
||||
.recv_timeout(Duration::from_secs(3))
|
||||
.expect("peer never saw EOF — finish() did not shutdown(Write)");
|
||||
assert_eq!(received, b"unflushed-tail");
|
||||
}
|
||||
|
||||
/// UDP `write` must emit ONE datagram per call carrying exactly the
|
||||
/// bytes passed — no buffering, no coalescing (doc lines 100-108).
|
||||
/// Two writes of different lengths must arrive as two separate
|
||||
/// datagrams of those exact lengths, in order. Mutation: adding a
|
||||
/// BufWriter to UdpSocketSink (the doc explicitly forbids it) would
|
||||
/// merge these into one datagram and the second `recv` would time
|
||||
/// out.
|
||||
#[test]
|
||||
fn udp_write_is_one_datagram_per_call() {
|
||||
let receiver = UdpSocket::bind("127.0.0.1:0").unwrap();
|
||||
receiver
|
||||
.set_read_timeout(Some(std::time::Duration::from_secs(2)))
|
||||
.unwrap();
|
||||
let addr = receiver.local_addr().unwrap();
|
||||
let mut sink = UdpSocketSink::connect(addr, None).unwrap();
|
||||
|
||||
// Distinct lengths so a merge would be detectable.
|
||||
let n_a = sink.write(&[0xAA; 10]).unwrap();
|
||||
let n_b = sink.write(&[0xBB; 20]).unwrap();
|
||||
assert_eq!(n_a, 10);
|
||||
assert_eq!(n_b, 20);
|
||||
|
||||
let mut buf = [0u8; 256];
|
||||
let first = receiver.recv(&mut buf).unwrap();
|
||||
assert_eq!(first, 10, "first datagram must be exactly 10 bytes");
|
||||
assert!(buf[..first].iter().all(|&b| b == 0xAA));
|
||||
let second = receiver.recv(&mut buf).unwrap();
|
||||
assert_eq!(second, 20, "second datagram must be exactly 20 bytes");
|
||||
assert!(buf[..second].iter().all(|&b| b == 0xBB));
|
||||
}
|
||||
|
||||
/// UDP `finish` is a documented no-op (lines 157-165): there is no
|
||||
/// EOF marker for UDP. Calling it must not error and must not
|
||||
/// affect prior datagrams. Mutation: if `finish` tried to
|
||||
/// `shutdown` the UDP socket it could error or close it
|
||||
/// prematurely; here it must just return Ok.
|
||||
#[test]
|
||||
fn udp_finish_is_noop_ok() {
|
||||
let receiver = UdpSocket::bind("127.0.0.1:0").unwrap();
|
||||
let addr = receiver.local_addr().unwrap();
|
||||
let mut sink = UdpSocketSink::connect(addr, None).unwrap();
|
||||
assert!(sink.finish().is_ok());
|
||||
// A second finish is equally harmless.
|
||||
assert!(sink.finish().is_ok());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
//! Per-platform writeback pipeline. On Linux, drains dirty pages
|
||||
//! continuously at chunk granularity to keep the kernel's writeback
|
||||
//! queue bounded. On macOS and Windows, a no-op stub.
|
||||
//!
|
||||
//! The platform decision lives entirely in this file (the cfg-gated
|
||||
//! `pub use` below). Callers — and `DiskWriter` itself — are
|
||||
//! platform-independent.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
mod noop;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) use linux::WritebackPipeline;
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub(super) use noop::WritebackPipeline;
|
||||
@@ -0,0 +1,626 @@
|
||||
//! Linux writeback pipeline using `sync_file_range` + `posix_fadvise`.
|
||||
//!
|
||||
//! Pathology this fixes: the kernel's default `vm.dirty_ratio` (~20 %
|
||||
//! of RAM) lets dirty pages accumulate to hundreds of MB during a
|
||||
//! big sequential write, then bursts a flush at 99 % disk utilisation.
|
||||
//! While the burst runs, app writes block on the writeback queue —
|
||||
//! observed empirically as instantaneous speed dropping from ~15 MB/s
|
||||
//! to ~1 MB/s every ~30 s during a Pass 1 sweep.
|
||||
//!
|
||||
//! Strategy: every `chunk_bytes` of new sequential output, kick async
|
||||
//! writeback (`SYNC_FILE_RANGE_WRITE`) on the just-completed chunk and
|
||||
//! finalise the *previous* chunk via `WAIT_AFTER` + `posix_fadvise
|
||||
//! (DONTNEED)`. By the time we finalise, that previous chunk has had
|
||||
//! a full chunk's worth of work to flush — the wait is near-instant.
|
||||
//! Dirty cache stays bounded at ~2 × `chunk_bytes` and writes drain
|
||||
//! continuously instead of in bursts.
|
||||
//!
|
||||
//! The chunk size is adaptive: we measure the elapsed time of the
|
||||
//! `WAIT_AFTER` call over a rolling window of the last 16 chunks and
|
||||
//! resize the chunk based on the p95. Slow storage (NFS, network
|
||||
//! shares, HDD) sees larger chunks to amortise per-chunk overhead;
|
||||
//! fast storage (NVMe) sees smaller chunks to keep cache pressure
|
||||
//! tight. Bounds: [4 MiB, 256 MiB].
|
||||
//!
|
||||
//! ## NFS escape hatch
|
||||
//!
|
||||
//! `sync_file_range(WAIT_AFTER)` on an NFS-mounted file can block
|
||||
//! indefinitely waiting for the server's commit ack. If the server
|
||||
//! never acks (network partition, server-side hang, slow commit), the
|
||||
//! syscall never returns and the consumer thread is stuck inside the
|
||||
//! kernel — `/api/stop` can't reach it because halt is cooperative.
|
||||
//!
|
||||
//! When `fstatfs` reports the file lives on an NFS mount
|
||||
//! (`f_type == NFS_SUPER_MAGIC`), the pipeline skips the WAIT_AFTER +
|
||||
//! `posix_fadvise(DONTNEED)` dance entirely. NFS clients have their
|
||||
//! own buffering and commit semantics that handle dirty-page bounds
|
||||
//! without us forcing the issue. The async `SYNC_FILE_RANGE_WRITE`
|
||||
//! kickoff still runs (non-blocking by spec) so writeback still gets
|
||||
//! a nudge.
|
||||
//!
|
||||
//! ## Defence in depth: WAIT_AFTER timeout
|
||||
//!
|
||||
//! Even on local storage, a degraded disk or odd filesystem driver
|
||||
//! could in principle wedge inside WAIT_AFTER. Each WAIT_AFTER call
|
||||
//! runs on a worker thread with a 30s recv_timeout on its result
|
||||
//! channel. On timeout we log a loud error, set a `degraded` flag,
|
||||
//! and from then on skip WAIT_AFTER + DONTNEED for the rest of the
|
||||
//! pipeline's life (same shape as the NFS path). The worker thread
|
||||
//! is intentionally leaked — it unwinds whenever the syscall
|
||||
//! eventually returns or the process exits. The mux continues; the
|
||||
//! original dirty-burst pathology re-emerges but the rip can still
|
||||
//! finish instead of freezing.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::{AsRawFd, RawFd};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const ADAPTIVE_WINDOW: usize = 16;
|
||||
const CHUNK_BYTES_MIN: u64 = 4 * 1024 * 1024;
|
||||
const CHUNK_BYTES_MAX: u64 = 256 * 1024 * 1024;
|
||||
const ADAPTIVE_GROW_MS: u64 = 200;
|
||||
const ADAPTIVE_SHRINK_MS: u64 = 20;
|
||||
/// Every N chunks, emit a `debug!` snapshot of the current chunk
|
||||
/// size so operators tailing the log can see where the autoscaler
|
||||
/// settled.
|
||||
const SIZE_LOG_INTERVAL: u64 = 32;
|
||||
/// Hard upper bound on a single `sync_file_range(WAIT_AFTER)` call.
|
||||
/// Beyond this we declare the pipeline degraded and stop calling
|
||||
/// WAIT_AFTER for the rest of its life.
|
||||
const WAIT_AFTER_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
pub(crate) struct WritebackPipeline {
|
||||
/// Aliases the wrapping `WritebackFile::file`. Only valid for the
|
||||
/// lifetime of that struct — moving the `File` independently
|
||||
/// would silently UAF this fd. The pipeline is a private field of
|
||||
/// `WritebackFile` and never exposed outside that wrapper, which
|
||||
/// is what keeps the alias sound.
|
||||
fd: RawFd,
|
||||
/// An owned clone of the file descriptor, held so that any
|
||||
/// leaked WAIT_AFTER worker thread retains a valid reference to
|
||||
/// the underlying file description for the duration of its
|
||||
/// syscall — even if the original `WritebackFile` is closed first
|
||||
/// and the OS reuses its fd number. `None` only when `try_clone`
|
||||
/// failed at construction (rare); the pipeline falls back to the
|
||||
/// pre-clone `fd` integer in that case, which carries the original
|
||||
/// fd-reuse risk but is no worse than the previous behaviour.
|
||||
wait_file: Option<File>,
|
||||
chunk_bytes: u64,
|
||||
last_flush_pos: u64,
|
||||
pending: Option<(u64, u64)>,
|
||||
/// Rolling window of recent `WAIT_AFTER` elapsed_ms measurements.
|
||||
wait_after_window: VecDeque<u64>,
|
||||
/// Count of chunks emitted (used to space out periodic
|
||||
/// `debug!` size snapshots).
|
||||
chunk_count: u64,
|
||||
/// True when the underlying file is on an NFS mount. NFS makes
|
||||
/// WAIT_AFTER unsafe (can block forever on missing server ack), so
|
||||
/// we skip it entirely and let the NFS client handle commit on
|
||||
/// close.
|
||||
is_nfs: bool,
|
||||
/// Set the first time WAIT_AFTER exceeds [`WAIT_AFTER_TIMEOUT`].
|
||||
/// Once set, behaviour matches the NFS path for the rest of the
|
||||
/// pipeline's life. A plain `AtomicBool`: the flag is only ever
|
||||
/// touched on the owning thread (the spawned WAIT_AFTER worker never
|
||||
/// reads or writes it). `AtomicBool` over `bool` only because the
|
||||
/// load/store sites read cleanly; no sharing is needed today.
|
||||
degraded: AtomicBool,
|
||||
}
|
||||
|
||||
impl WritebackPipeline {
|
||||
/// Construct a pipeline aliasing `file`'s file descriptor. The
|
||||
/// returned `WritebackPipeline` MUST be dropped before `file`
|
||||
/// itself, or kept inside the same struct that owns `file` — the
|
||||
/// alias is unchecked.
|
||||
pub(crate) fn new(file: &File, start_pos: u64, chunk_bytes: u64) -> Self {
|
||||
let fd = file.as_raw_fd();
|
||||
let is_nfs = detect_nfs(fd);
|
||||
// Clone the fd so any leaked WAIT_AFTER worker thread keeps the
|
||||
// file description alive. Log but continue on clone failure.
|
||||
let wait_file = match file.try_clone() {
|
||||
Ok(f) => Some(f),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
target: "mux",
|
||||
"WritebackPipeline fd={fd}: try_clone failed ({e}), WAIT_AFTER workers \
|
||||
will use raw fd (fd-reuse risk on timeout)"
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
tracing::info!(
|
||||
target: "mux",
|
||||
"WritebackPipeline fd={fd} is_nfs={is_nfs} chunk_bytes={chunk_bytes} strategy={}",
|
||||
if is_nfs { "nfs-skip-wait" } else { "wait+dontneed" }
|
||||
);
|
||||
Self {
|
||||
fd,
|
||||
wait_file,
|
||||
chunk_bytes,
|
||||
last_flush_pos: start_pos,
|
||||
pending: None,
|
||||
wait_after_window: VecDeque::with_capacity(ADAPTIVE_WINDOW),
|
||||
chunk_count: 0,
|
||||
is_nfs,
|
||||
degraded: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
/// True if we should bypass the WAIT_AFTER + DONTNEED finalisation
|
||||
/// step. NFS always bypasses; local storage bypasses once the
|
||||
/// pipeline has flipped to degraded after a WAIT_AFTER timeout.
|
||||
#[inline]
|
||||
fn skip_wait(&self) -> bool {
|
||||
self.is_nfs || self.degraded.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Produce a fresh per-call `File` clone for the WAIT_AFTER worker.
|
||||
///
|
||||
/// Each call to `wait_after_with_timeout` needs its own owned clone
|
||||
/// so the worker thread keeps the file description alive for the
|
||||
/// duration of the syscall. We clone from `self.wait_file` (itself a
|
||||
/// clone taken at construction) rather than from the original file.
|
||||
///
|
||||
/// Returns `None` only if `wait_file` is `None` (construction
|
||||
/// try_clone failed) or if the second-level try_clone fails — both
|
||||
/// rare; the fallback raw-fd path in `wait_after_with_timeout`
|
||||
/// handles that case.
|
||||
#[inline]
|
||||
fn clone_for_worker(&self) -> Option<File> {
|
||||
self.wait_file.as_ref().and_then(|f| f.try_clone().ok())
|
||||
}
|
||||
|
||||
/// Caller advanced the file position to `pos`. If a chunk boundary
|
||||
/// was crossed, kick async writeback for the just-completed chunk
|
||||
/// and finalise the previous one.
|
||||
pub(crate) fn note_progress(&mut self, pos: u64) {
|
||||
if pos < self.last_flush_pos.saturating_add(self.chunk_bytes) {
|
||||
return;
|
||||
}
|
||||
// Byte offsets are unsigned throughout; the signed cast happens
|
||||
// only at the libc call boundary where the kernel ABI requires
|
||||
// `i64`. `saturating_sub` documents and hardens the line-above
|
||||
// guard that `pos >= last_flush_pos`.
|
||||
let chunk_off: u64 = self.last_flush_pos;
|
||||
let chunk_len: u64 = pos.saturating_sub(self.last_flush_pos);
|
||||
let mut wait_ms: u64 = 0;
|
||||
let mut fadvise_ms: u64 = 0;
|
||||
// Async kickoff for the just-completed chunk runs on every
|
||||
// path (NFS, degraded, normal) — it's nominally non-blocking
|
||||
// by spec and gives the kernel an early hint that this range
|
||||
// is ready to flush.
|
||||
let kickoff_rc = unsafe {
|
||||
libc::sync_file_range(
|
||||
self.fd,
|
||||
chunk_off as i64,
|
||||
chunk_len as i64,
|
||||
libc::SYNC_FILE_RANGE_WRITE,
|
||||
)
|
||||
};
|
||||
if kickoff_rc != 0 {
|
||||
// Non-fatal: the async write-out hint failed, but the data is
|
||||
// still in the page cache and will be flushed by later fsync /
|
||||
// kernel writeback. Surface it for diagnosability.
|
||||
tracing::warn!(
|
||||
target: "freemkv::io",
|
||||
errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0),
|
||||
"sync_file_range(WRITE) kickoff failed"
|
||||
);
|
||||
}
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
if self.skip_wait() {
|
||||
// NFS branch (or degraded fallback after a prior
|
||||
// timeout): the WAIT_AFTER + DONTNEED dance is what
|
||||
// hangs on NFS — skip it. We still advance `pending`
|
||||
// so the next call has a stable cycle.
|
||||
} else {
|
||||
// Normal local-storage branch with belt-and-braces
|
||||
// timeout. If WAIT_AFTER hangs > WAIT_AFTER_TIMEOUT
|
||||
// we mark the pipeline degraded, log a loud error,
|
||||
// and fall through to the skip path on subsequent
|
||||
// calls.
|
||||
match wait_after_with_timeout(self.clone_for_worker(), self.fd, prev_off, prev_len)
|
||||
{
|
||||
Some(ms) => {
|
||||
wait_ms = ms;
|
||||
let t_fadv = Instant::now();
|
||||
unsafe {
|
||||
libc::posix_fadvise(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
}
|
||||
fadvise_ms = t_fadv.elapsed().as_millis() as u64;
|
||||
self.record_wait(wait_ms);
|
||||
}
|
||||
None => {
|
||||
// Timeout branch: switch to NFS-style skip
|
||||
// for the rest of the pipeline's life. Do
|
||||
// NOT call DONTNEED — if WAIT_AFTER hasn't
|
||||
// returned, the pages aren't safely flushed.
|
||||
self.degraded.store(true, Ordering::Relaxed);
|
||||
// Once degraded we skip DONTNEED, so every subsequent
|
||||
// chunk's pages stay resident until close — the same
|
||||
// page-cache exposure profile as NFS. Shrink to the
|
||||
// floor so that exposure window is as small as the NFS
|
||||
// path keeps it, instead of whatever the adaptive sizing
|
||||
// had grown chunk_bytes to (up to 256 MiB).
|
||||
self.chunk_bytes = CHUNK_BYTES_MIN;
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackPipeline WAIT_AFTER timed out after {}s on chunk off={} len={}, marking writeback degraded (subsequent chunks will skip WAIT_AFTER + DONTNEED, chunk_bytes lowered to floor)",
|
||||
WAIT_AFTER_TIMEOUT.as_secs(),
|
||||
prev_off,
|
||||
prev_len
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.pending = Some((chunk_off, chunk_len));
|
||||
self.last_flush_pos = pos;
|
||||
self.chunk_count += 1;
|
||||
tracing::trace!(
|
||||
target: "mux",
|
||||
"WritebackPipeline chunk off={} len={} wait_after_ms={wait_ms} fadvise_ms={fadvise_ms} chunk_bytes={} skip_wait={}",
|
||||
chunk_off,
|
||||
chunk_len,
|
||||
self.chunk_bytes,
|
||||
self.skip_wait(),
|
||||
);
|
||||
if self.chunk_count % SIZE_LOG_INTERVAL == 0 {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackPipeline chunk_bytes={} after {} chunks is_nfs={} degraded={}",
|
||||
self.chunk_bytes,
|
||||
self.chunk_count,
|
||||
self.is_nfs,
|
||||
self.degraded.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Push a new `WAIT_AFTER` measurement into the rolling window
|
||||
/// and, if the window is full, adapt `chunk_bytes` based on p95.
|
||||
fn record_wait(&mut self, wait_ms: u64) {
|
||||
if self.wait_after_window.len() == ADAPTIVE_WINDOW {
|
||||
self.wait_after_window.pop_front();
|
||||
}
|
||||
self.wait_after_window.push_back(wait_ms);
|
||||
if self.wait_after_window.len() < ADAPTIVE_WINDOW {
|
||||
return;
|
||||
}
|
||||
// p95 index, derived from the window size so it stays valid if
|
||||
// ADAPTIVE_WINDOW changes (a hard-coded `[14]` would panic OOB
|
||||
// for a window <= 14). For the default 16 this is index 15
|
||||
// (ceil(16 * 95 / 100) - 1 = 15), i.e. the top sample.
|
||||
let mut sorted: Vec<u64> = self.wait_after_window.iter().copied().collect();
|
||||
sorted.sort_unstable();
|
||||
let p95_idx = (ADAPTIVE_WINDOW * 95).div_ceil(100).min(ADAPTIVE_WINDOW) - 1;
|
||||
let p95 = sorted[p95_idx];
|
||||
let old = self.chunk_bytes;
|
||||
let new = if p95 > ADAPTIVE_GROW_MS && self.chunk_bytes < CHUNK_BYTES_MAX {
|
||||
(self.chunk_bytes * 2).min(CHUNK_BYTES_MAX)
|
||||
} else if p95 < ADAPTIVE_SHRINK_MS && self.chunk_bytes > CHUNK_BYTES_MIN {
|
||||
(self.chunk_bytes / 2).max(CHUNK_BYTES_MIN)
|
||||
} else {
|
||||
self.chunk_bytes
|
||||
};
|
||||
if new != old {
|
||||
self.chunk_bytes = new;
|
||||
tracing::info!(
|
||||
target: "mux",
|
||||
"WritebackPipeline adaptive chunk_bytes {} -> {} p95_ms={p95}",
|
||||
old,
|
||||
new
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Caller is about to seek away from the current write region.
|
||||
/// Drain any in-flight chunk and reset tracking.
|
||||
pub(crate) fn handle_seek(&mut self, new_pos: u64) {
|
||||
self.finalize();
|
||||
self.last_flush_pos = new_pos;
|
||||
}
|
||||
|
||||
/// Drain any in-flight chunk. Idempotent. Call before `sync_all()`
|
||||
/// or when discarding the pipeline.
|
||||
pub(crate) fn finalize(&mut self) {
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackPipeline finalize chunk off={prev_off} len={prev_len} skip_wait={} is_nfs={} degraded={}",
|
||||
self.skip_wait(),
|
||||
self.is_nfs,
|
||||
self.degraded.load(Ordering::Relaxed),
|
||||
);
|
||||
if self.skip_wait() {
|
||||
// NFS / degraded: skip WAIT_AFTER + DONTNEED. close()
|
||||
// / sync_all() handle commit through their normal
|
||||
// paths.
|
||||
return;
|
||||
}
|
||||
match wait_after_with_timeout(self.clone_for_worker(), self.fd, prev_off, prev_len) {
|
||||
Some(_ms) => unsafe {
|
||||
libc::posix_fadvise(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
},
|
||||
None => {
|
||||
self.degraded.store(true, Ordering::Relaxed);
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackPipeline finalize WAIT_AFTER timed out after {}s on chunk off={prev_off} len={prev_len}, marking writeback degraded",
|
||||
WAIT_AFTER_TIMEOUT.as_secs(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe whether `fd` lives on an NFS mount. Thin wrapper around
|
||||
/// [`crate::platform::fs_type::detect_fd`] so writeback policy and
|
||||
/// general-purpose fs-type classification stay in sync (same magic
|
||||
/// numbers, same musl-vs-glibc cast handling).
|
||||
///
|
||||
/// Fails open: any classification other than NFS counts as "not NFS"
|
||||
/// (including `Unknown` on `fstatfs` error) — better to run the
|
||||
/// normal local-storage path on a misdetected NFS mount and surface
|
||||
/// the freeze loudly via [`WAIT_AFTER_TIMEOUT`] than to needlessly
|
||||
/// disable writeback bounding on every local file because of a
|
||||
/// transient stat error.
|
||||
fn detect_nfs(fd: RawFd) -> bool {
|
||||
matches!(
|
||||
crate::platform::fs_type::detect_fd(fd),
|
||||
crate::platform::fs_type::FsType::Nfs
|
||||
)
|
||||
}
|
||||
|
||||
/// Run `sync_file_range(WAIT_AFTER)` on a worker thread and wait up
|
||||
/// to [`WAIT_AFTER_TIMEOUT`] for it to return. `Some(elapsed_ms)` on
|
||||
/// success; `None` on timeout. On timeout the worker thread is
|
||||
/// intentionally leaked — it unwinds whenever the syscall eventually
|
||||
/// returns or the process exits.
|
||||
///
|
||||
/// This delegates to [`crate::io::bounded::bounded_syscall`], the
|
||||
/// generic worker-thread + `recv_timeout` primitive, and just adapts it
|
||||
/// to the WAIT_AFTER call shape: it returns `elapsed_ms` instead of the
|
||||
/// syscall's `()`, and treats `WorkerLost` as a benign no-op to match
|
||||
/// the original semantics.
|
||||
///
|
||||
/// ## fd lifetime / fd-reuse safety
|
||||
///
|
||||
/// `worker_file` is an *owned* `File` (produced by `File::try_clone` at
|
||||
/// pipeline construction). It is moved into the worker closure so the
|
||||
/// file description stays alive for exactly as long as the worker thread
|
||||
/// lives — even if the original `WritebackFile` is closed and the OS
|
||||
/// reuses its fd number before the worker's syscall returns.
|
||||
///
|
||||
/// `fallback_fd` is used only when `worker_file` is `None` (i.e. the
|
||||
/// `try_clone` at construction failed). In that case the worker captures
|
||||
/// the raw fd integer, which carries the original fd-reuse risk but is
|
||||
/// no worse than the pre-fix behaviour.
|
||||
fn wait_after_with_timeout(
|
||||
worker_file: Option<File>,
|
||||
fallback_fd: RawFd,
|
||||
off: u64,
|
||||
len: u64,
|
||||
) -> Option<u64> {
|
||||
let started = Instant::now();
|
||||
let result = if let Some(owned) = worker_file {
|
||||
// Happy path: the closure owns a cloned File that keeps the
|
||||
// file description alive until the worker drops it.
|
||||
crate::io::bounded::bounded_syscall(None, WAIT_AFTER_TIMEOUT, move || unsafe {
|
||||
let fd = owned.as_raw_fd();
|
||||
libc::sync_file_range(fd, off as i64, len as i64, libc::SYNC_FILE_RANGE_WAIT_AFTER);
|
||||
// `owned` drops here, closing the cloned fd.
|
||||
})
|
||||
} else {
|
||||
// Fallback: try_clone failed at construction; use the raw fd.
|
||||
// This carries the pre-fix fd-reuse risk on timeout, but is no
|
||||
// regression from the original behaviour.
|
||||
crate::io::bounded::bounded_syscall(None, WAIT_AFTER_TIMEOUT, move || unsafe {
|
||||
libc::sync_file_range(
|
||||
fallback_fd,
|
||||
off as i64,
|
||||
len as i64,
|
||||
libc::SYNC_FILE_RANGE_WAIT_AFTER,
|
||||
);
|
||||
})
|
||||
};
|
||||
match result {
|
||||
Ok(()) => Some(started.elapsed().as_millis() as u64),
|
||||
Err(crate::io::bounded::BoundedError::Timeout)
|
||||
| Err(crate::io::bounded::BoundedError::Halted) => None,
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => {
|
||||
// Worker thread spawn failed or panicked before sending.
|
||||
// Treat as a benign success (no syscall ran) rather than
|
||||
// a degrade trigger — falling through with elapsed_ms=0
|
||||
// matches the no-op behaviour.
|
||||
Some(0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
/// Helper: build a `WritebackPipeline` over a local tempfile. On
|
||||
/// every test rig (linux dev box, CI) the tempfile lives on a
|
||||
/// local FS, so `is_nfs=false` and `skip_wait` returns false until
|
||||
/// we explicitly mark the pipeline degraded.
|
||||
fn local_pipeline(chunk_bytes: u64) -> (NamedTempFile, WritebackPipeline) {
|
||||
let f = NamedTempFile::new().expect("tempfile create");
|
||||
let pipeline = WritebackPipeline::new(f.as_file(), 0, chunk_bytes);
|
||||
(f, pipeline)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_pipeline_starts_active() {
|
||||
let (_f, p) = local_pipeline(32 * 1024 * 1024);
|
||||
assert!(!p.is_nfs, "local tempfile must not classify as NFS");
|
||||
assert!(!p.degraded.load(Ordering::Relaxed));
|
||||
assert!(!p.skip_wait(), "fresh local pipeline must not skip wait");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn degraded_flag_short_circuits_wait() {
|
||||
let (_f, p) = local_pipeline(32 * 1024 * 1024);
|
||||
assert!(!p.skip_wait());
|
||||
p.degraded.store(true, Ordering::Relaxed);
|
||||
assert!(
|
||||
p.skip_wait(),
|
||||
"degraded flag must force the wait+dontneed bypass"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_wait_grows_chunk_on_high_p95() {
|
||||
let (_f, mut p) = local_pipeline(16 * 1024 * 1024);
|
||||
// Fill the window with samples above the grow threshold.
|
||||
for _ in 0..ADAPTIVE_WINDOW {
|
||||
p.record_wait(ADAPTIVE_GROW_MS + 50);
|
||||
}
|
||||
assert!(
|
||||
p.chunk_bytes > 16 * 1024 * 1024,
|
||||
"chunk should have grown; got {}",
|
||||
p.chunk_bytes
|
||||
);
|
||||
assert!(p.chunk_bytes <= CHUNK_BYTES_MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_wait_shrinks_chunk_on_low_p95() {
|
||||
let (_f, mut p) = local_pipeline(64 * 1024 * 1024);
|
||||
for _ in 0..ADAPTIVE_WINDOW {
|
||||
p.record_wait(1); // well under ADAPTIVE_SHRINK_MS
|
||||
}
|
||||
assert!(
|
||||
p.chunk_bytes < 64 * 1024 * 1024,
|
||||
"chunk should have shrunk; got {}",
|
||||
p.chunk_bytes
|
||||
);
|
||||
assert!(p.chunk_bytes >= CHUNK_BYTES_MIN);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_wait_no_op_below_window_fill() {
|
||||
let (_f, mut p) = local_pipeline(16 * 1024 * 1024);
|
||||
let initial = p.chunk_bytes;
|
||||
// Only push a few samples; window not full → no adaptation.
|
||||
for _ in 0..(ADAPTIVE_WINDOW - 1) {
|
||||
p.record_wait(ADAPTIVE_GROW_MS + 100);
|
||||
}
|
||||
assert_eq!(
|
||||
p.chunk_bytes, initial,
|
||||
"chunk must not change before window is full"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_wait_clamps_to_chunk_bounds() {
|
||||
// Grow past the max.
|
||||
let (_f, mut p) = local_pipeline(CHUNK_BYTES_MAX);
|
||||
for _ in 0..ADAPTIVE_WINDOW {
|
||||
p.record_wait(ADAPTIVE_GROW_MS + 1000);
|
||||
}
|
||||
assert_eq!(p.chunk_bytes, CHUNK_BYTES_MAX, "must clamp to MAX");
|
||||
|
||||
// Shrink past the min.
|
||||
let (_f, mut p) = local_pipeline(CHUNK_BYTES_MIN);
|
||||
for _ in 0..ADAPTIVE_WINDOW {
|
||||
p.record_wait(0);
|
||||
}
|
||||
assert_eq!(p.chunk_bytes, CHUNK_BYTES_MIN, "must clamp to MIN");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_nfs_local_file_is_false() {
|
||||
// Local tempfile must not classify as NFS. This locks in the
|
||||
// consolidation through `crate::platform::fs_type::detect_fd`.
|
||||
let f = NamedTempFile::new().expect("tempfile create");
|
||||
use std::os::unix::io::AsRawFd;
|
||||
assert!(!detect_nfs(f.as_file().as_raw_fd()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn note_progress_below_chunk_is_noop() {
|
||||
let (_f, mut p) = local_pipeline(32 * 1024 * 1024);
|
||||
// No-op return before crossing the first chunk boundary.
|
||||
let before = p.chunk_count;
|
||||
p.note_progress(1024); // < 32 MiB
|
||||
assert_eq!(p.chunk_count, before);
|
||||
assert!(p.pending.is_none());
|
||||
}
|
||||
|
||||
// ── Bug-fix regression tests ────────────────────────────────────────
|
||||
|
||||
/// Regression for the fd-reuse / use-after-close fix. Verifies that
|
||||
/// `WritebackPipeline::new` successfully clones the fd into
|
||||
/// `wait_file` (i.e. `try_clone` doesn't fail for a normal
|
||||
/// tempfile) and that `clone_for_worker` returns `Some` — meaning
|
||||
/// the WAIT_AFTER worker will capture an owned `File` rather than a
|
||||
/// raw fd integer.
|
||||
///
|
||||
/// A deterministic test for the actual fd-reuse race is not clean to
|
||||
/// write (it would require simultaneously closing the original File
|
||||
/// and re-opening a new one to steal the fd number while the worker
|
||||
/// is mid-syscall, which is inherently racy). This test instead pins
|
||||
/// the structural invariant: on a normal local file, the pipeline
|
||||
/// holds a valid clone and will give the worker an owned File.
|
||||
#[test]
|
||||
fn wait_file_clone_is_present_for_local_tempfile() {
|
||||
let (_f, p) = local_pipeline(32 * 1024 * 1024);
|
||||
assert!(
|
||||
p.wait_file.is_some(),
|
||||
"wait_file must be Some for a normal local tempfile (try_clone should not fail)"
|
||||
);
|
||||
// clone_for_worker must return Some — the worker will get an
|
||||
// owned File, not fall through to the raw-fd fallback.
|
||||
let worker_clone = p.clone_for_worker();
|
||||
assert!(
|
||||
worker_clone.is_some(),
|
||||
"clone_for_worker must return Some when wait_file is Some"
|
||||
);
|
||||
}
|
||||
|
||||
/// Structural: the worker `File` clone returned by `clone_for_worker`
|
||||
/// is a distinct file descriptor (different fd number) that refers to
|
||||
/// the same underlying file. Closing the original tempfile must not
|
||||
/// affect the clone's validity — the OS keeps the file description
|
||||
/// alive until all file descriptors referring to it are closed.
|
||||
///
|
||||
/// We verify "distinct fd number" and "still usable as a raw fd"
|
||||
/// without actually racing a syscall.
|
||||
#[test]
|
||||
fn worker_clone_has_distinct_fd_from_original() {
|
||||
let f = NamedTempFile::new().expect("tempfile create");
|
||||
let original_fd = f.as_file().as_raw_fd();
|
||||
let pipeline = WritebackPipeline::new(f.as_file(), 0, 32 * 1024 * 1024);
|
||||
|
||||
let clone = pipeline
|
||||
.clone_for_worker()
|
||||
.expect("clone_for_worker returned None");
|
||||
let clone_fd = clone.as_raw_fd();
|
||||
|
||||
// The clone must have a different fd number — it is a separate
|
||||
// open file description (dup'd by try_clone).
|
||||
assert_ne!(
|
||||
clone_fd, original_fd,
|
||||
"worker clone must have a distinct fd number from the original"
|
||||
);
|
||||
// The clone fd must be valid (non-negative on Unix).
|
||||
assert!(clone_fd >= 0, "clone fd must be non-negative");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
//! No-op writeback pipeline for non-Linux targets. macOS and Windows
|
||||
//! page cache policies have not been shown to exhibit the Linux
|
||||
//! accumulate-then-burst flush pathology for our access pattern.
|
||||
//! If that changes, replace this stub with a real implementation
|
||||
//! (e.g. `F_NOCACHE` on macOS, `FILE_FLAG_WRITE_THROUGH` on Windows).
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(crate) struct WritebackPipeline;
|
||||
|
||||
impl WritebackPipeline {
|
||||
pub(crate) fn new(_file: &File, _start_pos: u64, _chunk_bytes: u64) -> Self {
|
||||
Self
|
||||
}
|
||||
pub(crate) fn note_progress(&mut self, _pos: u64) {}
|
||||
pub(crate) fn handle_seek(&mut self, _new_pos: u64) {}
|
||||
pub(crate) fn finalize(&mut self) {}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
//! Linux platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! - `preallocate`: `fallocate(FALLOC_FL_KEEP_SIZE)` — reserve extents
|
||||
//! without growing the reported file size. Reduces extent
|
||||
//! fragmentation on large sequential writes (mux output on NFS in
|
||||
//! particular).
|
||||
//! - `durable_sync`: `fsync` wrapped in
|
||||
//! [`crate::io::bounded::bounded_syscall`] with a 60 s deadline so a
|
||||
//! wedged NFS server can't trap the calling thread indefinitely.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Pre-reserve extents for `size_bytes` of upcoming sequential writes.
|
||||
/// Best-effort: a non-zero rc is logged but not propagated, since the
|
||||
/// caller would just continue with the unreserved file anyway.
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
// FALLOC_FL_KEEP_SIZE = 0x01 — keep the reported file size at 0
|
||||
// (writes grow it normally) while still pre-reserving the extents.
|
||||
// Clamp to the signed `off_t` range; an unchecked `as i64` cast
|
||||
// would wrap a >= 2^63 size to a negative length (EINVAL no-op).
|
||||
let len = i64::try_from(size_bytes).unwrap_or(i64::MAX);
|
||||
let rc = unsafe { libc::fallocate(file.as_raw_fd(), libc::FALLOC_FL_KEEP_SIZE, 0, len) };
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile fallocate size_hint={size_bytes} rc={rc} ok={}",
|
||||
rc == 0
|
||||
);
|
||||
}
|
||||
|
||||
/// Run `fsync` on `file` with a 60 s deadline. On timeout — and
|
||||
/// likewise on halt or a lost worker — we log and return `Ok(())`: the
|
||||
/// kernel will still flush on close, so the data is best-effort durable.
|
||||
/// The alternative (trap the thread for the rest of the rip, or return
|
||||
/// an error that aborts an otherwise-complete mux) is worse, so all
|
||||
/// three fallbacks return `Ok(())`. `Ok(())` from these paths is NOT a
|
||||
/// durability barrier — the durable flush did not complete; only the
|
||||
/// hang is bounded.
|
||||
///
|
||||
/// ## fd-reuse safety
|
||||
///
|
||||
/// The `fsync` runs on a bounded worker thread that may be leaked on
|
||||
/// timeout. To avoid the leaked worker's syscall hitting a recycled fd
|
||||
/// number after the original `File` is closed, we `try_clone` an owned
|
||||
/// `File` and move it into the closure. The clone keeps the underlying
|
||||
/// file description alive for as long as the worker thread lives.
|
||||
/// On `try_clone` failure (rare) we fall back to the raw fd integer —
|
||||
/// no worse than the previous behaviour.
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
// Clone so a leaked worker thread retains a valid fd even after the
|
||||
// original File is closed and its fd number is reused.
|
||||
let owned = match file.try_clone() {
|
||||
Ok(f) => Some(f),
|
||||
Err(e) => {
|
||||
let fd = file.as_raw_fd();
|
||||
tracing::warn!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all fd={fd}: try_clone failed ({e}), fsync worker will use raw fd (fd-reuse risk on timeout)"
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
let fallback_fd = file.as_raw_fd();
|
||||
match crate::io::bounded::bounded_syscall(
|
||||
None,
|
||||
Duration::from_secs(60),
|
||||
move || -> io::Result<()> {
|
||||
let fd = owned.as_ref().map(|f| f.as_raw_fd()).unwrap_or(fallback_fd);
|
||||
let rc = unsafe { libc::fsync(fd) };
|
||||
// `owned` (if Some) drops here, releasing the cloned fd.
|
||||
if rc == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::last_os_error())
|
||||
}
|
||||
},
|
||||
) {
|
||||
Ok(inner) => inner,
|
||||
Err(crate::io::bounded::BoundedError::Timeout) => {
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all fsync timed out after 60s; kernel will flush on close (best-effort)"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(crate::io::bounded::BoundedError::Halted) => {
|
||||
tracing::warn!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all fsync skipped (halt requested); data not durably flushed, kernel will flush on close"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => {
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all fsync worker lost before completion; data not durably flushed, kernel will flush on close"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(target_os = "linux")]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
/// Regression for the fd-reuse / use-after-close fix in `durable_sync`.
|
||||
///
|
||||
/// Verifies the structural invariant: `try_clone` succeeds for a normal
|
||||
/// local tempfile, and the cloned `File` has a distinct fd number from
|
||||
/// the original. This pins the property that a leaked fsync worker thread
|
||||
/// captures an owned `File` (and thus keeps the file description alive)
|
||||
/// rather than a bare fd integer that can be reused after the original
|
||||
/// `File` closes.
|
||||
///
|
||||
/// The actual fd-reuse race is non-deterministic and not cleanly
|
||||
/// testable without coordinating a simultaneous close + re-open on
|
||||
/// another thread. A structural test is the accepted substitute.
|
||||
#[test]
|
||||
fn durable_sync_worker_uses_owned_clone_with_distinct_fd() {
|
||||
let f = NamedTempFile::new().expect("tempfile create");
|
||||
let original_fd = f.as_file().as_raw_fd();
|
||||
|
||||
// try_clone must succeed for a normal local file.
|
||||
let owned = f
|
||||
.as_file()
|
||||
.try_clone()
|
||||
.expect("try_clone must succeed for a local tempfile");
|
||||
let clone_fd = owned.as_raw_fd();
|
||||
|
||||
// The clone must be a distinct fd (dup'd, not aliased).
|
||||
assert_ne!(
|
||||
clone_fd, original_fd,
|
||||
"owned clone must have a distinct fd number — not an alias of the original"
|
||||
);
|
||||
assert!(clone_fd >= 0, "clone fd must be a valid non-negative fd");
|
||||
|
||||
// durable_sync must complete without error on the local tempfile.
|
||||
durable_sync(f.as_file()).expect("durable_sync must return Ok on a local tempfile");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
//! macOS platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! - `preallocate`: `fcntl(F_PREALLOCATE)` — macOS's fallocate-equiv.
|
||||
//! First attempt requests `F_ALLOCATECONTIG | F_ALLOCATEALL` (prefer a
|
||||
//! contiguous run but accept scattered extents to satisfy the full
|
||||
//! length), falling back to `F_ALLOCATEALL` alone on failure.
|
||||
//! `F_PREALLOCATE` never advances EOF regardless of the flags — only
|
||||
//! `ftruncate`/writes grow the file — so the reported file size is
|
||||
//! unchanged; `F_ALLOCATEALL` governs the contiguity fallback, not size.
|
||||
//! - `durable_sync`: `fcntl(F_FULLFSYNC)` wrapped in
|
||||
//! [`crate::io::bounded::bounded_syscall`] with a 60 s deadline.
|
||||
//! F_FULLFSYNC is HFS+/APFS's true-fsync (flushes the disk's own
|
||||
//! write cache) — what `fsync` should have been on macOS. Falls back
|
||||
//! to plain `fsync` if F_FULLFSYNC returns ENOTSUP.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::io::platform_macos::{
|
||||
F_ALLOCATEALL, F_ALLOCATECONTIG, F_PEOFPOSMODE, F_PREALLOCATE, Fstore,
|
||||
};
|
||||
|
||||
/// `fcntl(F_FULLFSYNC)` opcode. Documented in `man 2 fcntl` on macOS;
|
||||
/// not in the `libc` crate as a named constant.
|
||||
const F_FULLFSYNC: libc::c_int = 51;
|
||||
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
// Clamp to the signed `off_t` range; an unchecked `as off_t` cast
|
||||
// would wrap a >= 2^63 size to a negative length.
|
||||
let len = i64::try_from(size_bytes).unwrap_or(i64::MAX) as libc::off_t;
|
||||
let mut fst = Fstore {
|
||||
fst_flags: F_ALLOCATECONTIG | F_ALLOCATEALL,
|
||||
fst_posmode: F_PEOFPOSMODE,
|
||||
fst_offset: 0,
|
||||
fst_length: len,
|
||||
fst_bytesalloc: 0,
|
||||
};
|
||||
// First attempt: contiguous.
|
||||
let mut rc = unsafe { libc::fcntl(file.as_raw_fd(), F_PREALLOCATE, &mut fst) };
|
||||
if rc == -1 {
|
||||
// Fall back: drop the contiguous hint, allow scattered extents.
|
||||
fst.fst_flags = F_ALLOCATEALL;
|
||||
rc = unsafe { libc::fcntl(file.as_raw_fd(), F_PREALLOCATE, &mut fst) };
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile F_PREALLOCATE size_hint={size_bytes} rc={rc} bytes_allocated={} ok={}",
|
||||
fst.fst_bytesalloc,
|
||||
rc != -1
|
||||
);
|
||||
}
|
||||
|
||||
/// ## fd-reuse safety
|
||||
///
|
||||
/// The F_FULLFSYNC / fsync runs on a bounded worker thread that may be
|
||||
/// leaked on timeout. To avoid the leaked worker's syscall hitting a
|
||||
/// recycled fd number after the original `File` is closed, we
|
||||
/// `try_clone` an owned `File` and move it into the closure. The clone
|
||||
/// keeps the underlying file description alive for as long as the worker
|
||||
/// thread lives. On `try_clone` failure (rare) we fall back to the raw
|
||||
/// fd integer — no worse than the previous behaviour.
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
// Clone so a leaked worker thread retains a valid fd even after the
|
||||
// original File is closed and its fd number is reused.
|
||||
let owned = match file.try_clone() {
|
||||
Ok(f) => Some(f),
|
||||
Err(e) => {
|
||||
let fd = file.as_raw_fd();
|
||||
tracing::warn!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all fd={fd}: try_clone failed ({e}), F_FULLFSYNC worker will use raw fd (fd-reuse risk on timeout)"
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
let fallback_fd = file.as_raw_fd();
|
||||
match crate::io::bounded::bounded_syscall(
|
||||
None,
|
||||
Duration::from_secs(60),
|
||||
move || -> io::Result<()> {
|
||||
let fd = owned.as_ref().map(|f| f.as_raw_fd()).unwrap_or(fallback_fd);
|
||||
// Try F_FULLFSYNC first. If it isn't supported on this
|
||||
// filesystem (older HFS, some network mounts) fall back to
|
||||
// plain fsync — better than nothing.
|
||||
let rc = unsafe { libc::fcntl(fd, F_FULLFSYNC, 0) };
|
||||
if rc == 0 {
|
||||
// `owned` (if Some) drops here, releasing the cloned fd.
|
||||
return Ok(());
|
||||
}
|
||||
let err = io::Error::last_os_error();
|
||||
if err.raw_os_error() == Some(libc::ENOTSUP) {
|
||||
let rc = unsafe { libc::fsync(fd) };
|
||||
// `owned` drops here.
|
||||
if rc == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::last_os_error())
|
||||
}
|
||||
} else {
|
||||
// `owned` drops here.
|
||||
Err(err)
|
||||
}
|
||||
},
|
||||
) {
|
||||
Ok(inner) => inner,
|
||||
Err(crate::io::bounded::BoundedError::Timeout) => {
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all F_FULLFSYNC timed out after 60s; kernel will flush on close (best-effort)"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(crate::io::bounded::BoundedError::Halted) => Ok(()),
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(target_os = "macos")]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
/// Regression for the fd-reuse / use-after-close fix in `durable_sync`.
|
||||
///
|
||||
/// Verifies the structural invariant: `try_clone` succeeds for a normal
|
||||
/// local tempfile, and the cloned `File` has a distinct fd number from
|
||||
/// the original. This pins the property that a leaked F_FULLFSYNC/fsync
|
||||
/// worker thread captures an owned `File` (keeping the file description
|
||||
/// alive) rather than a bare fd integer that can be reused after the
|
||||
/// original `File` closes.
|
||||
///
|
||||
/// The actual fd-reuse race is non-deterministic; a structural test is
|
||||
/// the accepted substitute.
|
||||
#[test]
|
||||
fn durable_sync_worker_uses_owned_clone_with_distinct_fd() {
|
||||
let f = NamedTempFile::new().expect("tempfile create");
|
||||
let original_fd = f.as_file().as_raw_fd();
|
||||
|
||||
// try_clone must succeed for a normal local file.
|
||||
let owned = f
|
||||
.as_file()
|
||||
.try_clone()
|
||||
.expect("try_clone must succeed for a local tempfile");
|
||||
let clone_fd = owned.as_raw_fd();
|
||||
|
||||
// The clone must be a distinct fd (dup'd, not aliased).
|
||||
assert_ne!(
|
||||
clone_fd, original_fd,
|
||||
"owned clone must have a distinct fd number — not an alias of the original"
|
||||
);
|
||||
assert!(clone_fd >= 0, "clone fd must be a valid non-negative fd");
|
||||
|
||||
// durable_sync must complete without error on the local tempfile.
|
||||
durable_sync(f.as_file()).expect("durable_sync must return Ok on a local tempfile");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,646 @@
|
||||
//! `WritebackFile` — a `File` wrapper whose reason for existing is the
|
||||
//! bounded-cache writeback pipeline.
|
||||
//!
|
||||
//! Why: large sequential writes (sweep, patch, mux on UHD-scale output)
|
||||
//! left to the kernel's default writeback policy accumulate hundreds of
|
||||
//! megabytes of dirty pages and then burst-flush, stalling subsequent
|
||||
//! writes for seconds at a time. `WritebackFile` drives a continuous
|
||||
//! [`super::writeback::WritebackPipeline`] that on Linux issues
|
||||
//! incremental `sync_file_range` + `posix_fadvise(DONTNEED)` calls at
|
||||
//! 32 MB granularity so dirty pages drain at the same rate they're
|
||||
//! produced. macOS and Windows fall through to a no-op pipeline — their
|
||||
//! default cache policies have not been shown to exhibit the same
|
||||
//! pathology for this access pattern.
|
||||
//!
|
||||
//! It implements `Write` and `Seek` so any call site that wrote to a
|
||||
//! plain `File` through those traits (sweep, patch, mux) can swap in
|
||||
//! `WritebackFile` without touching the body of the loop. The wrapper
|
||||
//! also tracks the current file position to feed the pipeline with
|
||||
//! progress + seek boundaries.
|
||||
//!
|
||||
//! See `super::writeback::linux` for the underlying pathology and the
|
||||
//! strategy.
|
||||
//!
|
||||
//! ## Platform split
|
||||
//!
|
||||
//! The platform-specific pieces of this wrapper — extent preallocation
|
||||
//! (Linux `fallocate(KEEP_SIZE)`, macOS `F_PREALLOCATE`, Windows no-op
|
||||
//! today) and the durable-flush primitive (Linux/macOS
|
||||
//! `fsync`/`F_FULLFSYNC` wrapped in a bounded syscall; Windows plain
|
||||
//! `FlushFileBuffers`, unbounded) — live in per-OS sibling modules. The
|
||||
//! dispatch happens once at the bottom of this file via cfg-gated `mod`
|
||||
//! decls. No inline `#[cfg(target_os = "...")]` in the business-logic
|
||||
//! above.
|
||||
//!
|
||||
//! ## Write path
|
||||
//!
|
||||
//! Writes are direct passthrough to the underlying `File` (no writer
|
||||
//! thread, no ring, no batching). Empirically a writer-thread
|
||||
//! architecture introduced a ~60% mux throughput regression on NFS
|
||||
//! bidirectional workloads; the direct-passthrough write path is faster.
|
||||
//! The writeback pipeline still runs (it's called inline from `write` /
|
||||
//! `write_all` / `seek`) so the bounded-cache invariant on Linux is
|
||||
//! preserved.
|
||||
//!
|
||||
//! ## Halt-safety
|
||||
//!
|
||||
//! `sync_all` runs the per-OS durable-flush primitive. On Linux/macOS
|
||||
//! it is wrapped in [`crate::io::bounded::bounded_syscall`] with a 60 s
|
||||
//! deadline, so a wedged NFS server cannot trap the muxer indefinitely
|
||||
//! on the final fsync. Windows is a known deviation: its `durable_sync`
|
||||
//! calls `File::sync_all` (`FlushFileBuffers`) directly and is NOT
|
||||
//! bounded — a wedged UNC/SMB share can block the final flush there.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
mod other;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux as platform;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos as platform;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
use other as platform;
|
||||
#[cfg(target_os = "windows")]
|
||||
use windows as platform;
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use super::writeback::WritebackPipeline;
|
||||
|
||||
/// Granularity at which the Linux writeback pipeline issues
|
||||
/// `sync_file_range` pairs. 32 MiB is the empirically best value on a
|
||||
/// 1 GbE NFS mount backed by a single spinning disk: 8 MiB / 64 MiB /
|
||||
/// 128 MiB all measured worse. Override via `FREEMKV_WRITEBACK_CHUNK_MIB`
|
||||
/// — faster backends (NVMe, RAID) may tolerate larger windows.
|
||||
const WRITEBACK_CHUNK_BYTES_DEFAULT: u64 = 32 * 1024 * 1024;
|
||||
|
||||
/// Upper bound (in MiB) accepted from `FREEMKV_WRITEBACK_CHUNK_MIB`.
|
||||
/// 64 GiB — far above `CHUNK_BYTES_MAX` (256 MiB), generous for any
|
||||
/// real backend, and small enough that `n * 1024 * 1024` cannot wrap
|
||||
/// `u64`. Out-of-range values fall back to the default.
|
||||
const WRITEBACK_CHUNK_MIB_MAX: u64 = 64 * 1024;
|
||||
|
||||
fn writeback_chunk_bytes() -> u64 {
|
||||
std::env::var("FREEMKV_WRITEBACK_CHUNK_MIB")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.filter(|&n| n > 0 && n <= WRITEBACK_CHUNK_MIB_MAX)
|
||||
.map(|n| n * 1024 * 1024)
|
||||
.unwrap_or(WRITEBACK_CHUNK_BYTES_DEFAULT)
|
||||
}
|
||||
|
||||
pub(crate) struct WritebackFile {
|
||||
file: File,
|
||||
pipeline: WritebackPipeline,
|
||||
pos: u64,
|
||||
/// Count of position-moving seeks (for the finalize summary). The MKV muxer
|
||||
/// seeks back occasionally (cluster size patching, Cues, Segment header
|
||||
/// backpatch); the per-seek DEBUG line is trace-level now, and this rolls
|
||||
/// the total into one finalize summary.
|
||||
seek_count: u64,
|
||||
/// Sum of |delta| over all position-moving seeks, in bytes.
|
||||
seek_bytes: u64,
|
||||
}
|
||||
|
||||
impl WritebackFile {
|
||||
/// Wrap an open `File`. The current OS file position is queried
|
||||
/// once so the pipeline starts tracking from wherever the file
|
||||
/// already is (typically 0 for fresh files; non-zero for resumed
|
||||
/// or appended files).
|
||||
pub(crate) fn new(mut file: File) -> io::Result<Self> {
|
||||
let pos = file.stream_position()?;
|
||||
let pipeline = WritebackPipeline::new(&file, pos, writeback_chunk_bytes());
|
||||
Ok(Self {
|
||||
file,
|
||||
pipeline,
|
||||
pos,
|
||||
seek_count: 0,
|
||||
seek_bytes: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new file at `path` (truncating any existing contents)
|
||||
/// and wrap it. Convenience for the common
|
||||
/// `File::create(path)` + `WritebackFile::new(file)` pair so callers
|
||||
/// don't have to assemble a `File` first.
|
||||
///
|
||||
/// Callers that know the target output size should prefer
|
||||
/// [`Self::create_with_size_hint`] so the kernel can pre-reserve
|
||||
/// extents.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = File::create(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Like [`Self::create`] but pre-reserves `size_bytes` of disk
|
||||
/// space via the platform's extent-preallocation primitive (Linux
|
||||
/// `fallocate(KEEP_SIZE)`, macOS `F_PREALLOCATE`, Windows
|
||||
/// `SetFileValidData` stub). The reported file size is unchanged
|
||||
/// (writes still grow the file naturally) — only the on-disk extent
|
||||
/// allocation is preallocated, which reduces extent fragmentation
|
||||
/// on large sequential writes (mux output, especially on slow
|
||||
/// storage / NFS).
|
||||
///
|
||||
/// On platforms without an extent-preallocation primitive this is
|
||||
/// equivalent to `create` — the size hint is dropped after a debug
|
||||
/// log.
|
||||
pub(crate) fn create_with_size_hint(path: &Path, size_bytes: u64) -> io::Result<Self> {
|
||||
let file = File::create(path)?;
|
||||
platform::preallocate(&file, size_bytes);
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Open an existing file at `path` for writing (no truncation) and
|
||||
/// wrap it. Mirrors `File::open` semantics for the writable case
|
||||
/// — used by patch / resume paths that mutate an existing ISO in
|
||||
/// place.
|
||||
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new().write(true).open(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Drain in-flight writeback then issue a full fsync. Use this in
|
||||
/// place of `File::sync_all`.
|
||||
///
|
||||
/// The final durable flush is wrapped in
|
||||
/// [`crate::io::bounded::bounded_syscall`] (per the per-OS module)
|
||||
/// with a 60 s deadline on Linux/macOS — a wedged NFS server cannot
|
||||
/// trap the calling thread indefinitely. On timeout the page cache
|
||||
/// is left to the kernel's normal flush-on-close path — best
|
||||
/// effort, but bounded.
|
||||
///
|
||||
/// IMPORTANT: on Linux/macOS a successful `Ok(())` does NOT
|
||||
/// guarantee the data is durable if the bounded fsync timed out or
|
||||
/// was halted — only the hang is bounded, the fsync may not have
|
||||
/// completed. Callers needing crash-consistency (e.g. mux-finish
|
||||
/// then external commit/DB update) must not treat `Ok(())` as a
|
||||
/// durability barrier.
|
||||
pub(crate) fn sync_all(&mut self) -> io::Result<()> {
|
||||
if self.seek_count > 0 {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile finalize: {} seeks, {} bytes seeked total",
|
||||
self.seek_count,
|
||||
self.seek_bytes
|
||||
);
|
||||
}
|
||||
self.pipeline.finalize();
|
||||
platform::durable_sync(&self.file)
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for WritebackFile {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let n = self.file.write(buf)?;
|
||||
self.pos += n as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.file.write_all(buf)?;
|
||||
self.pos += buf.len() as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for WritebackFile {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
let p = self.file.seek(from)?;
|
||||
// Only treat seeks that actually move the position as
|
||||
// boundaries — sweep does a redundant `seek(Current(pos))`
|
||||
// before every write, and we don't want that to drain the
|
||||
// pipeline on every iteration.
|
||||
if p != self.pos {
|
||||
// Diagnostic for the NFS mux hang: the MKV format requires
|
||||
// the muxer to seek back occasionally (cluster size
|
||||
// patching, Cues index write, Segment header backpatch).
|
||||
// Each such seek invalidates the writeback chunk tracking
|
||||
// and forces a finalize → WAIT_AFTER on the in-flight
|
||||
// chunk. Logging the seek delta lets us correlate hang
|
||||
// offsets with specific muxer operations.
|
||||
let from_pos = self.pos;
|
||||
let to_pos = p;
|
||||
let delta: i64 = (to_pos as i64).wrapping_sub(from_pos as i64);
|
||||
// Per-seek detail is trace-level (L4) — benign and high-frequency.
|
||||
// The aggregate (count + total bytes) is logged once at finalize.
|
||||
tracing::trace!(
|
||||
target: "mux",
|
||||
"WritebackFile seek from={from_pos} to={to_pos} delta={delta}"
|
||||
);
|
||||
self.seek_count += 1;
|
||||
self.seek_bytes += delta.unsigned_abs();
|
||||
self.pipeline.handle_seek(p);
|
||||
self.pos = p;
|
||||
}
|
||||
Ok(p)
|
||||
}
|
||||
}
|
||||
|
||||
impl super::sink::SequentialSink for WritebackFile {
|
||||
/// Drain the writeback pipeline and run the bounded durable flush —
|
||||
/// the same work [`Self::sync_all`] does. Implemented explicitly (no
|
||||
/// blanket impl) so a `dyn SequentialSink` / `dyn RandomAccessSink`
|
||||
/// `finish()` actually finalises + fsyncs instead of hitting a no-op
|
||||
/// default. Note the bounded-fsync caveat from [`Self::sync_all`]
|
||||
/// applies: `Ok(())` is not a durability barrier if the fsync timed
|
||||
/// out or was halted.
|
||||
fn finish(&mut self) -> io::Result<()> {
|
||||
self.sync_all()
|
||||
}
|
||||
}
|
||||
|
||||
impl super::sink::RandomAccessSink for WritebackFile {}
|
||||
|
||||
impl Drop for WritebackFile {
|
||||
fn drop(&mut self) {
|
||||
// Run the pipeline's tail finalize so the last in-flight chunk
|
||||
// gets its `WAIT_AFTER` + `posix_fadvise(DONTNEED)`. Without
|
||||
// this, callers that drop a `WritebackFile` without calling
|
||||
// `sync_all` (panic, early-return, idiomatic `let _ = w;`)
|
||||
// leave the trailing chunk in cache; the kernel still flushes
|
||||
// on close, but the bounded-cache invariant fails at the tail.
|
||||
// We deliberately do *not* call `self.file.sync_all()` here —
|
||||
// close already triggers a flush, and an `fsync` from `Drop`
|
||||
// would silently swallow its `io::Error` anyway. `finalize` is
|
||||
// idempotent so an explicit `sync_all` followed by drop is
|
||||
// still safe.
|
||||
self.pipeline.finalize();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
|
||||
fn read_back(path: &Path) -> Vec<u8> {
|
||||
let mut f = File::open(path).unwrap();
|
||||
let mut v = Vec::new();
|
||||
f.read_to_end(&mut v).unwrap();
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_then_drop_persists_bytes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("a.bin");
|
||||
{
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"hello world").unwrap();
|
||||
// Drop drains the pipeline tail.
|
||||
}
|
||||
assert_eq!(read_back(&p), b"hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_all_drains_and_flushes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("b.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
for _ in 0..32 {
|
||||
w.write_all(&[0x5au8; 1024]).unwrap();
|
||||
}
|
||||
// After sync_all, the bytes MUST be visible to a separate
|
||||
// reader. The pipeline has been finalised and durable-sync has
|
||||
// run.
|
||||
w.sync_all().unwrap();
|
||||
let bytes = read_back(&p);
|
||||
assert_eq!(bytes.len(), 32 * 1024);
|
||||
assert!(bytes.iter().all(|&b| b == 0x5a));
|
||||
drop(w);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seek_then_patch_roundtrip() {
|
||||
// Write A; seek back; patch with B; read back; the patch lands
|
||||
// at the right offset.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("c.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
let big = vec![b'A'; 4096];
|
||||
w.write_all(&big).unwrap();
|
||||
// Seek back to offset 1000 and overwrite 8 bytes.
|
||||
w.seek(SeekFrom::Start(1000)).unwrap();
|
||||
w.write_all(b"PATCHED!").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
let bytes = read_back(&p);
|
||||
assert_eq!(bytes.len(), 4096);
|
||||
assert_eq!(&bytes[1000..1008], b"PATCHED!");
|
||||
// Bytes outside the patch are still 'A'.
|
||||
assert_eq!(bytes[999], b'A');
|
||||
assert_eq!(bytes[1008], b'A');
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flush_is_observed_in_order() {
|
||||
// `Write::flush` should not panic or reorder; verify the bytes
|
||||
// land in order through interleaved flushes.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("f.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"one").unwrap();
|
||||
w.flush().unwrap();
|
||||
w.write_all(b"two").unwrap();
|
||||
w.flush().unwrap();
|
||||
w.write_all(b"three").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
assert_eq!(read_back(&p), b"onetwothree");
|
||||
}
|
||||
|
||||
/// finish() through a `dyn RandomAccessSink` trait object must
|
||||
/// dispatch to WritebackFile's override (finalize + durable_sync),
|
||||
/// not a no-op default. Bytes must be visible to a separate reader
|
||||
/// before drop.
|
||||
#[test]
|
||||
fn finish_through_trait_object_persists() {
|
||||
use crate::io::sink::RandomAccessSink;
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("finish-dyn.bin");
|
||||
let w = WritebackFile::create(&p).unwrap();
|
||||
let mut boxed: Box<dyn RandomAccessSink> = Box::new(w);
|
||||
boxed.write_all(b"durable-tail").unwrap();
|
||||
boxed.finish().unwrap();
|
||||
assert_eq!(read_back(&p), b"durable-tail");
|
||||
}
|
||||
|
||||
// ── Added hardening tests ───────────────────────────────────────
|
||||
|
||||
/// `write` (not write_all) must return the count the inner File
|
||||
/// reported and advance `pos` by exactly that count (lines
|
||||
/// 185-189). For a regular file a single `write` of a small buffer
|
||||
/// writes all of it. We verify the returned count equals the buffer
|
||||
/// length AND that a subsequent seek reports the right position.
|
||||
/// Mutation: changing `self.pos += n` to `self.pos += buf.len()`
|
||||
/// (lines 187 vs a hypothetical bug) would desync on a partial
|
||||
/// write; here they coincide, but `Seek(Current(0))` reflecting `n`
|
||||
/// still guards the count return value.
|
||||
#[test]
|
||||
fn write_returns_byte_count_and_advances_pos() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("wc.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
let n = w.write(b"twelve bytes").unwrap();
|
||||
assert_eq!(n, 12, "write must report bytes written");
|
||||
// pos is private; observe it via the public Seek impl's
|
||||
// stream_position (which resolves to seek(Current(0))).
|
||||
let pos = w.stream_position().unwrap();
|
||||
assert_eq!(pos, 12, "pos not advanced by write count");
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
assert_eq!(read_back(&p), b"twelve bytes");
|
||||
}
|
||||
|
||||
/// Redundant seek to the CURRENT position must be a no-op for the
|
||||
/// pipeline (lines 211-228 only act when `p != self.pos`). This is
|
||||
/// the documented sweep optimisation: sweep does
|
||||
/// `seek(Current(pos))` before every write and we must not treat it
|
||||
/// as a boundary. We can only observe the public effect: the seek
|
||||
/// returns the same offset and writes continue contiguously.
|
||||
/// Mutation: removing the `if p != self.pos` guard (line 211) would
|
||||
/// call handle_seek on every redundant seek — on the noop pipeline
|
||||
/// (macOS) this stays correct for data, but the contiguity +
|
||||
/// returned-offset invariant still must hold and is asserted here.
|
||||
#[test]
|
||||
fn seek_to_current_position_is_noop_for_data() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("noop-seek.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"AAAA").unwrap();
|
||||
// Seek to the current end (offset 4) — a no-move seek.
|
||||
let off = w.seek(SeekFrom::Start(4)).unwrap();
|
||||
assert_eq!(off, 4);
|
||||
w.write_all(b"BBBB").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
assert_eq!(
|
||||
read_back(&p),
|
||||
b"AAAABBBB",
|
||||
"redundant seek corrupted contiguous write"
|
||||
);
|
||||
}
|
||||
|
||||
/// `open` (no-truncate) must preserve existing file contents and
|
||||
/// allow in-place patching from offset 0 — distinct from `create`
|
||||
/// which truncates (lines 157-160 use OpenOptions write-only, no
|
||||
/// truncate). We pre-seed a file, reopen with `open`, overwrite the
|
||||
/// first bytes, and confirm the tail survives. Mutation: if `open`
|
||||
/// used `File::create` (truncate) the tail would be lost.
|
||||
#[test]
|
||||
fn open_preserves_existing_contents() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("reopen.bin");
|
||||
std::fs::write(&p, b"ORIGINAL-CONTENT").unwrap();
|
||||
let mut w = WritebackFile::open(&p).unwrap();
|
||||
// open() does NOT truncate; pos starts at 0. Overwrite the
|
||||
// first 8 bytes only.
|
||||
w.write_all(b"PATCHED!").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
// First 8 bytes overwritten; the rest of ORIGINAL-CONTENT
|
||||
// ("-CONTENT") survives because there was no truncation.
|
||||
assert_eq!(read_back(&p), b"PATCHED!-CONTENT");
|
||||
}
|
||||
|
||||
/// `open` on a file whose position is queried must start tracking
|
||||
/// from the file's current offset. `WritebackFile::new` calls
|
||||
/// `stream_position()` (line 112); a freshly `open`ed file is at
|
||||
/// offset 0. After writing, seeking Current(0) must reflect the
|
||||
/// bytes written from 0. Mutation: if `new` hardcoded pos=0 instead
|
||||
/// of querying, a non-zero starting offset would desync — covered
|
||||
/// indirectly; here we assert the offset is exactly the write size.
|
||||
#[test]
|
||||
fn new_tracks_initial_position() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("pos-init.bin");
|
||||
std::fs::write(&p, b"0123456789").unwrap();
|
||||
let mut w = WritebackFile::open(&p).unwrap();
|
||||
let start = w.stream_position().unwrap();
|
||||
assert_eq!(start, 0, "freshly opened file should start at offset 0");
|
||||
w.write_all(b"XY").unwrap();
|
||||
let after = w.stream_position().unwrap();
|
||||
assert_eq!(after, 2, "pos must advance by written length");
|
||||
}
|
||||
|
||||
/// Seek past EOF then write must create a sparse hole that reads
|
||||
/// back as zeros — standard POSIX file semantics that the wrapper
|
||||
/// must not break (it forwards seek to the inner File at line 205).
|
||||
/// Mutation: if `seek` clamped or mishandled the offset, the hole
|
||||
/// size/zero-fill would be wrong.
|
||||
#[test]
|
||||
fn seek_past_eof_creates_zero_hole() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("hole.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"head").unwrap(); // bytes 0..4
|
||||
w.seek(SeekFrom::Start(20)).unwrap(); // jump past EOF
|
||||
w.write_all(b"tail").unwrap(); // bytes 20..24
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
let bytes = read_back(&p);
|
||||
assert_eq!(
|
||||
bytes.len(),
|
||||
24,
|
||||
"file should extend to the last written byte"
|
||||
);
|
||||
assert_eq!(&bytes[0..4], b"head");
|
||||
// The 4..20 gap must read back as zeros (sparse hole).
|
||||
assert!(bytes[4..20].iter().all(|&b| b == 0), "hole not zero-filled");
|
||||
assert_eq!(&bytes[20..24], b"tail");
|
||||
}
|
||||
|
||||
/// `SeekFrom::End` must resolve against the actual file length.
|
||||
/// After writing 10 bytes, `seek(End(-2))` lands at offset 8;
|
||||
/// overwriting 2 bytes there patches the tail. Mutation: forwarding
|
||||
/// the wrong SeekFrom variant would land at the wrong offset.
|
||||
#[test]
|
||||
fn seek_from_end_resolves_against_length() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("end-seek.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"0123456789").unwrap();
|
||||
let landed = w.seek(SeekFrom::End(-2)).unwrap();
|
||||
assert_eq!(landed, 8, "End(-2) of a 10-byte file is offset 8");
|
||||
w.write_all(b"XY").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
assert_eq!(read_back(&p), b"01234567XY");
|
||||
}
|
||||
|
||||
/// `create_with_size_hint` must produce a normal, writable file
|
||||
/// whose *reported size* tracks bytes written (the hint only
|
||||
/// reserves extents, per the doc lines 137-145 — it must NOT
|
||||
/// pre-grow the logical file length). We write 5 bytes against a
|
||||
/// 1 MiB hint and the file must be exactly 5 bytes long.
|
||||
/// Mutation: if the hint path truncated/extended to size_bytes the
|
||||
/// length would be 1 MiB and this fails.
|
||||
#[test]
|
||||
fn create_with_size_hint_does_not_inflate_logical_length() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("hint-len.bin");
|
||||
let mut w = WritebackFile::create_with_size_hint(&p, 1024 * 1024).unwrap();
|
||||
w.write_all(b"hello").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
let bytes = read_back(&p);
|
||||
assert_eq!(bytes.len(), 5, "size hint must not inflate logical length");
|
||||
assert_eq!(&bytes, b"hello");
|
||||
}
|
||||
|
||||
/// `sync_all` is idempotent: calling it twice (and then Drop, which
|
||||
/// also finalizes) must not corrupt data or panic. Doc lines
|
||||
/// 256-262: `finalize` is idempotent so explicit sync_all then drop
|
||||
/// is safe. Mutation: a finalize that double-freed or advanced a
|
||||
/// cursor would corrupt on the second call.
|
||||
#[test]
|
||||
fn double_sync_all_is_idempotent() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("double-sync.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"idempotent").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
w.sync_all().unwrap(); // second call must be safe
|
||||
drop(w); // Drop also finalizes
|
||||
assert_eq!(read_back(&p), b"idempotent");
|
||||
}
|
||||
|
||||
/// Env-var chunk override parsing (`writeback_chunk_bytes`, lines
|
||||
/// 91-98). Out-of-range / unparseable values must fall back to the
|
||||
/// 32 MiB default; valid in-range values are converted MiB→bytes.
|
||||
/// We can't safely mutate process env in parallel tests for the
|
||||
/// default-path branch, but we CAN assert the pure boundary logic
|
||||
/// the function encodes by reconstructing it: the filter accepts
|
||||
/// `0 < n <= WRITEBACK_CHUNK_MIB_MAX`. This pins the constants and
|
||||
/// the MiB→byte multiply. Mutation: changing `* 1024 * 1024` to a
|
||||
/// single `* 1024` would break this equality.
|
||||
#[test]
|
||||
fn writeback_chunk_constants_and_conversion() {
|
||||
// Default is exactly 32 MiB.
|
||||
assert_eq!(WRITEBACK_CHUNK_BYTES_DEFAULT, 32 * 1024 * 1024);
|
||||
// Max MiB bound is 64 GiB expressed in MiB, and the byte value
|
||||
// it maps to must not overflow u64.
|
||||
assert_eq!(WRITEBACK_CHUNK_MIB_MAX, 64 * 1024);
|
||||
let max_bytes = (WRITEBACK_CHUNK_MIB_MAX as u128) * 1024 * 1024;
|
||||
assert!(
|
||||
max_bytes <= u64::MAX as u128,
|
||||
"max chunk MiB * 1MiB must fit in u64"
|
||||
);
|
||||
}
|
||||
|
||||
/// Env-var override parsing for `writeback_chunk_bytes` (lines
|
||||
/// 91-98). All four branches in ONE test to avoid the data race of
|
||||
/// several parallel tests mutating the same process-global env var.
|
||||
///
|
||||
/// Branches: (1) valid in-range value → MiB→byte conversion; (2)
|
||||
/// zero → `n > 0` filter rejects → default; (3) garbage → parse
|
||||
/// fails → default; (4) over-max → `n <= MAX` filter rejects →
|
||||
/// default.
|
||||
///
|
||||
/// Mutations: `* 1024 * 1024` → `* 1024` breaks (1); dropping
|
||||
/// `n > 0` breaks (2); `unwrap()` on parse panics (3); dropping
|
||||
/// `n <= MAX` breaks (4).
|
||||
#[test]
|
||||
fn writeback_chunk_env_override_branches() {
|
||||
// SAFETY: this is the only test touching this env var, and it
|
||||
// sets+reads+clears synchronously within its own body.
|
||||
let set = |v: &str| unsafe { std::env::set_var("FREEMKV_WRITEBACK_CHUNK_MIB", v) };
|
||||
let clear = || unsafe { std::env::remove_var("FREEMKV_WRITEBACK_CHUNK_MIB") };
|
||||
|
||||
set("8");
|
||||
assert_eq!(
|
||||
writeback_chunk_bytes(),
|
||||
8 * 1024 * 1024,
|
||||
"in-range mis-converted"
|
||||
);
|
||||
|
||||
set("0");
|
||||
assert_eq!(
|
||||
writeback_chunk_bytes(),
|
||||
WRITEBACK_CHUNK_BYTES_DEFAULT,
|
||||
"zero must fall back (n > 0 filter)"
|
||||
);
|
||||
|
||||
set("not-a-number");
|
||||
assert_eq!(
|
||||
writeback_chunk_bytes(),
|
||||
WRITEBACK_CHUNK_BYTES_DEFAULT,
|
||||
"unparseable must fall back"
|
||||
);
|
||||
|
||||
// One past the max: WRITEBACK_CHUNK_MIB_MAX + 1.
|
||||
set(&(WRITEBACK_CHUNK_MIB_MAX + 1).to_string());
|
||||
assert_eq!(
|
||||
writeback_chunk_bytes(),
|
||||
WRITEBACK_CHUNK_BYTES_DEFAULT,
|
||||
"over-max must fall back (n <= MAX filter)"
|
||||
);
|
||||
|
||||
// Exactly at the max boundary is accepted (inclusive bound).
|
||||
set(&WRITEBACK_CHUNK_MIB_MAX.to_string());
|
||||
assert_eq!(
|
||||
writeback_chunk_bytes(),
|
||||
WRITEBACK_CHUNK_MIB_MAX * 1024 * 1024,
|
||||
"max boundary must be accepted (inclusive)"
|
||||
);
|
||||
|
||||
clear();
|
||||
// With the var cleared, the default is returned.
|
||||
assert_eq!(writeback_chunk_bytes(), WRITEBACK_CHUNK_BYTES_DEFAULT);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//! Fallback platform impl for [`super::WritebackFile`] on targets
|
||||
//! without a dedicated implementation (BSDs, illumos, etc.).
|
||||
//!
|
||||
//! - `preallocate` is a logged no-op.
|
||||
//! - `durable_sync` calls `File::sync_all` directly (no bounded-syscall
|
||||
//! wrapper — the wrapper depends on Linux/macOS unix idioms that
|
||||
//! aren't universally portable). If a future BSD impl needs the
|
||||
//! 60-s deadline, it should land in its own per-OS file rather than
|
||||
//! bloat this fallback.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
|
||||
pub(super) fn preallocate(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile preallocate size_hint={size_bytes} skipped (no impl on this target)"
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
file.sync_all()
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
//! Windows platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! Current behaviour:
|
||||
//!
|
||||
//! - `preallocate` is a debug-logged no-op. Windows has no
|
||||
//! `fallocate`-equivalent that keeps the reported size, so extent
|
||||
//! reservation is not wired up.
|
||||
//! - `durable_sync` delegates to the std `File::sync_all`, which on
|
||||
//! Windows maps to `FlushFileBuffers`. Unlike the Linux/macOS impls
|
||||
//! this is NOT wrapped in the bounded-syscall primitive (that would
|
||||
//! need an `unsafe impl Send` for `RawHandle`, which cannot be
|
||||
//! validated without a Windows test env), so a wedged UNC/SMB share
|
||||
//! can block the final flush. This deviation is documented on
|
||||
//! [`super::WritebackFile::sync_all`] and the parent module's
|
||||
//! Halt-safety section.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
|
||||
pub(super) fn preallocate(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile preallocate size_hint={size_bytes} skipped (no-op on windows)"
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
// `File::sync_all` on Windows is `FlushFileBuffers`. Not wrapped in
|
||||
// the bounded-syscall primitive (see the module doc) — unbounded.
|
||||
file.sync_all()
|
||||
}
|
||||
-182
@@ -1,182 +0,0 @@
|
||||
//! KEYDB.cfg updater — HTTP GET, unzip, verify, save.
|
||||
//!
|
||||
//! Zero external HTTP dependencies. Raw TCP for HTTP GET.
|
||||
//! Uses `zip` and `flate2` (already in deps) for extraction.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpStream;
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Standard keydb storage path.
|
||||
pub fn default_path() -> Result<PathBuf> {
|
||||
let home = std::env::var("HOME")
|
||||
.or_else(|_| std::env::var("USERPROFILE"))
|
||||
.map_err(|_| Error::KeydbParse)?;
|
||||
Ok(PathBuf::from(home)
|
||||
.join(".config")
|
||||
.join("freemkv")
|
||||
.join("keydb.cfg"))
|
||||
}
|
||||
|
||||
/// Download a KEYDB from a URL, verify, save to the standard path.
|
||||
pub fn update(url: &str) -> Result<UpdateResult> {
|
||||
let body = http_get(url)?;
|
||||
save(&body)
|
||||
}
|
||||
|
||||
/// Verify and save raw keydb bytes (plain text, .zip, or .gz).
|
||||
pub fn save(data: &[u8]) -> Result<UpdateResult> {
|
||||
let text = if data.starts_with(b"PK\x03\x04") {
|
||||
extract_zip(data)?
|
||||
} else if data.starts_with(&[0x1f, 0x8b]) {
|
||||
let mut dec = flate2::read::GzDecoder::new(data);
|
||||
let mut out = String::new();
|
||||
dec.read_to_string(&mut out)
|
||||
.map_err(|_| Error::KeydbParse)?;
|
||||
out
|
||||
} else {
|
||||
String::from_utf8(data.to_vec()).map_err(|_| Error::KeydbParse)?
|
||||
};
|
||||
|
||||
let entries = text
|
||||
.lines()
|
||||
.filter(|l| {
|
||||
let t = l.trim();
|
||||
t.starts_with("0x")
|
||||
|| t.starts_with("| DK")
|
||||
|| t.starts_with("| PK")
|
||||
|| t.starts_with("| HC")
|
||||
})
|
||||
.count();
|
||||
|
||||
if entries == 0 {
|
||||
return Err(Error::KeydbInvalid);
|
||||
}
|
||||
|
||||
let path = default_path()?;
|
||||
if let Some(dir) = path.parent() {
|
||||
std::fs::create_dir_all(dir).map_err(|_| Error::KeydbWrite {
|
||||
path: path.display().to_string(),
|
||||
})?;
|
||||
}
|
||||
std::fs::write(&path, &text).map_err(|_| Error::KeydbWrite {
|
||||
path: path.display().to_string(),
|
||||
})?;
|
||||
|
||||
Ok(UpdateResult {
|
||||
path,
|
||||
entries,
|
||||
bytes: text.len(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Result of a KEYDB update -- path written, entry count, and byte size.
|
||||
#[derive(Debug)]
|
||||
pub struct UpdateResult {
|
||||
pub path: PathBuf,
|
||||
pub entries: usize,
|
||||
pub bytes: usize,
|
||||
}
|
||||
|
||||
fn http_get(url: &str) -> Result<Vec<u8>> {
|
||||
let (mut host, mut port, mut path) = parse_url(url)?;
|
||||
|
||||
for _ in 0..5 {
|
||||
let addr = format!("{host}:{port}");
|
||||
let mut stream =
|
||||
TcpStream::connect(&addr).map_err(|_| Error::KeydbConnect { host: host.clone() })?;
|
||||
stream
|
||||
.set_read_timeout(Some(std::time::Duration::from_secs(30)))
|
||||
.ok();
|
||||
|
||||
let request = format!(
|
||||
"GET {path} HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\nAccept-Encoding: identity\r\n\r\n"
|
||||
);
|
||||
stream
|
||||
.write_all(request.as_bytes())
|
||||
.map_err(|_| Error::KeydbConnect { host: host.clone() })?;
|
||||
|
||||
let mut response = Vec::new();
|
||||
stream
|
||||
.take(100 * 1024 * 1024)
|
||||
.read_to_end(&mut response)
|
||||
.map_err(|_| Error::KeydbConnect { host: host.clone() })?;
|
||||
|
||||
let header_end = find_header_end(&response).ok_or(Error::KeydbParse)?;
|
||||
let headers = std::str::from_utf8(&response[..header_end]).unwrap_or("");
|
||||
let body = &response[header_end + 4..];
|
||||
|
||||
if let Some(location) = extract_header(headers, "Location") {
|
||||
let parsed = parse_url(&location)?;
|
||||
host = parsed.0;
|
||||
port = parsed.1;
|
||||
path = parsed.2;
|
||||
continue;
|
||||
}
|
||||
|
||||
let status = parse_status(headers);
|
||||
if status != 200 {
|
||||
return Err(Error::KeydbHttp { status });
|
||||
}
|
||||
|
||||
return Ok(body.to_vec());
|
||||
}
|
||||
|
||||
Err(Error::KeydbHttp { status: 302 })
|
||||
}
|
||||
|
||||
fn parse_url(url: &str) -> Result<(String, u16, String)> {
|
||||
let url = url.strip_prefix("http://").ok_or(Error::KeydbParse)?;
|
||||
let (host_port, path) = match url.find('/') {
|
||||
Some(i) => (&url[..i], &url[i..]),
|
||||
None => (url, "/"),
|
||||
};
|
||||
let (host, port) = match host_port.find(':') {
|
||||
Some(i) => (&host_port[..i], host_port[i + 1..].parse().unwrap_or(80)),
|
||||
None => (host_port, 80u16),
|
||||
};
|
||||
Ok((host.to_string(), port, path.to_string()))
|
||||
}
|
||||
|
||||
fn parse_status(headers: &str) -> u16 {
|
||||
headers
|
||||
.lines()
|
||||
.next()
|
||||
.and_then(|l| l.split_whitespace().nth(1))
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn find_header_end(data: &[u8]) -> Option<usize> {
|
||||
data.windows(4).position(|w| w == b"\r\n\r\n")
|
||||
}
|
||||
|
||||
fn extract_header(headers: &str, name: &str) -> Option<String> {
|
||||
for line in headers.lines() {
|
||||
if line.len() > name.len() + 2
|
||||
&& line[..name.len()].eq_ignore_ascii_case(name)
|
||||
&& line.as_bytes()[name.len()] == b':'
|
||||
{
|
||||
return Some(line[name.len() + 1..].trim().to_string());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn extract_zip(data: &[u8]) -> Result<String> {
|
||||
let cursor = std::io::Cursor::new(data);
|
||||
let mut archive = zip::ZipArchive::new(cursor).map_err(|_| Error::KeydbParse)?;
|
||||
|
||||
for i in 0..archive.len() {
|
||||
let mut file = archive.by_index(i).map_err(|_| Error::KeydbParse)?;
|
||||
if file.name().ends_with(".cfg") || file.name().ends_with(".CFG") {
|
||||
let mut text = String::new();
|
||||
file.read_to_string(&mut text)
|
||||
.map_err(|_| Error::KeydbParse)?;
|
||||
return Ok(text);
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::KeydbInvalid)
|
||||
}
|
||||
@@ -0,0 +1,624 @@
|
||||
//! Key sources — the layer that hands libfreemkv a disc's terminal Unit Keys.
|
||||
//!
|
||||
//! libfreemkv performs NO key lookup. An application resolves a disc's keys
|
||||
//! through one or more [`KeySource`]s, each an adapter over a backing store (a
|
||||
//! keydb file, a key server, the mapfile cache). A source's job is to return the
|
||||
//! disc's terminal **Unit Keys** ([`crate::aacs::UnitKey`]). It knows what
|
||||
//! material it holds (a DK / MK / VUK / pre-decrypted UK) and what it must fetch
|
||||
//! from the disc (VID, MKB, encrypted title keys, content samples) to get there;
|
||||
//! it orchestrates the derivation by calling libfreemkv's own boil-down crypto
|
||||
//! primitives ([`crate::aacs::mk_from_dk`] / [`crate::aacs::vuk_from_mk`] /
|
||||
//! [`crate::aacs::uk_from_vuk`]) through the [`ResolveCtx`] handed to it.
|
||||
//!
|
||||
//! libfreemkv still OWNS the crypto: the boil-down primitives and the AES live
|
||||
//! here. A source owns only PATH ORCHESTRATION — deciding which primitive to
|
||||
//! call with what input for the material it happens to hold. Source
|
||||
//! implementations are published in the companion `freemkv-keysources` crate,
|
||||
//! keeping key *policy* (which store, which order, online vs local) out of the
|
||||
//! library.
|
||||
|
||||
use crate::aacs::{HostCert, UnitKey, Vid};
|
||||
use crate::disc::Key;
|
||||
use crate::error::Error;
|
||||
|
||||
/// The public AACS inputs a key source needs to look a disc up. Captured at
|
||||
/// scan; contains no secrets — only the disc identity and the on-disc AACS
|
||||
/// structures a source or key server may key on.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DiscInputs {
|
||||
/// SHA-1 of `Unit_Key_RO.inf`, `0x`-prefixed hex. The value a keydb keys
|
||||
/// its per-disc entries by, and a key server identifies the disc with.
|
||||
pub disc_hash: String,
|
||||
/// Volume ID (16 bytes). `[0u8; 16]` when no authenticated handshake ran
|
||||
/// (e.g. an ISO/mapfile flow), which disables VID-keyed lookups.
|
||||
pub volume_id: [u8; 16],
|
||||
/// Raw MKB bytes. Empty when not captured.
|
||||
pub mkb: Vec<u8>,
|
||||
/// Raw `Unit_Key_RO.inf` bytes. Empty when not captured.
|
||||
pub unit_key_ro: Vec<u8>,
|
||||
/// Encrypted on-disc content sample units (each a 6144-byte aligned unit),
|
||||
/// for sources that validate a key server-side against real ciphertext
|
||||
/// (e.g. an online key service). Empty for sources that don't need them
|
||||
/// (a local keydb). Populated by the application — reading content requires
|
||||
/// the disc reader, which the library's scan does not retain — so
|
||||
/// [`crate::Disc::inputs`] leaves it empty for the caller to fill.
|
||||
pub samples: Vec<Vec<u8>>,
|
||||
/// The disc's human title — the UDF/ISO volume identifier (e.g.
|
||||
/// `TITLE_2024`), falling back to the BDMV `<di:name>` when present.
|
||||
/// `None` when not captured. Identity only, no secret; a key service may
|
||||
/// record it (keyed by `disc_hash`) to build a hash→title catalog. Not used
|
||||
/// in any AACS derivation.
|
||||
pub volume_label: Option<String>,
|
||||
}
|
||||
|
||||
/// A lazy view of a disc's AACS material, handed to [`KeySource::get_uk`] so a
|
||||
/// source can drive the derivation chain without holding the disc reader.
|
||||
///
|
||||
/// "Lazy" by contract: each accessor returns only what the source asks for, so a
|
||||
/// source that already holds terminal Unit Keys never touches the MKB or
|
||||
/// samples. (Today the backing [`DiscInputsCtx`] is eagerly populated from a
|
||||
/// scan-time [`DiscInputs`]; the trait keeps the lazy signature so a future
|
||||
/// implementation can fetch on demand without a source-API break.)
|
||||
pub trait ResolveCtx {
|
||||
/// SHA-1 of `Unit_Key_RO.inf`, `0x`-prefixed hex — the per-disc lookup key.
|
||||
fn disc_hash(&self) -> &str;
|
||||
/// The disc's human title (UDF/ISO volume identifier), when captured.
|
||||
fn title(&self) -> Option<&str>;
|
||||
/// Volume ID, or `None` when no authenticated handshake ran (the all-zero
|
||||
/// sentinel) — VID-dependent derivation (`MK → VUK`) is then impossible.
|
||||
fn vid(&self) -> Option<Vid>;
|
||||
/// Raw MKB bytes (may be empty when not captured).
|
||||
fn mkb(&self) -> Result<&[u8], Error>;
|
||||
/// The disc's encrypted title keys, parsed from `Unit_Key_RO.inf` the same
|
||||
/// way the library's resolver parses them ([`crate::aacs::parse_unit_key_ro`]),
|
||||
/// in on-disc order. Feed straight into [`crate::aacs::uk_from_vuk`].
|
||||
fn enc_title_keys(&self) -> Result<&[[u8; 16]], Error>;
|
||||
/// Up to `n` encrypted on-disc content sample units, for a source that
|
||||
/// validates a candidate server-side against real ciphertext.
|
||||
fn samples(&self, n: usize) -> Result<Vec<Vec<u8>>, Error>;
|
||||
/// Raw `Unit_Key_RO.inf` bytes, verbatim. Most sources derive locally from
|
||||
/// the parsed [`Self::enc_title_keys`]; a source that forwards the on-disc
|
||||
/// structure to a server doing its OWN derivation (an online key service)
|
||||
/// needs the unparsed blob. Empty when not captured. Defaults to empty so
|
||||
/// existing/foreign `ResolveCtx` impls keep compiling unchanged.
|
||||
fn unit_key_ro(&self) -> &[u8] {
|
||||
&[]
|
||||
}
|
||||
}
|
||||
|
||||
/// [`ResolveCtx`] over a scan-time [`DiscInputs`].
|
||||
///
|
||||
/// Pre-parses the encrypted title keys at construction (so `enc_title_keys` can
|
||||
/// hand back a borrowed slice) at the version-appropriate `Unit_Key_RO.inf`
|
||||
/// stride — `version_u8` is the disc's AACS major (1 → 48-byte V10 stride, else
|
||||
/// 64-byte V20/V21 stride), matching the library resolver's dispatch.
|
||||
pub struct DiscInputsCtx<'a> {
|
||||
inner: &'a DiscInputs,
|
||||
enc_keys: Vec<[u8; 16]>,
|
||||
}
|
||||
|
||||
impl<'a> DiscInputsCtx<'a> {
|
||||
/// Build a context over `inputs`, parsing the encrypted title keys at the
|
||||
/// stride for AACS major `version_u8` (1 = V10, else V20/V21).
|
||||
///
|
||||
/// A present-but-malformed `unit_key_ro` (truncated / wrong magic / wrong
|
||||
/// stride) parses to an empty key set, so a later [`Self::enc_title_keys`]
|
||||
/// returns `Ok(&[])` indistinguishably from a disc that legitimately has no
|
||||
/// title keys — the parse failure is swallowed here, not surfaced as an
|
||||
/// error.
|
||||
pub fn new(inputs: &'a DiscInputs, version_u8: u8) -> Self {
|
||||
use crate::aacs::{AacsVersion, parse_unit_key_ro};
|
||||
let enc_keys = if inputs.unit_key_ro.is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
let version = if version_u8 == 1 {
|
||||
AacsVersion::V10
|
||||
} else {
|
||||
AacsVersion::V20
|
||||
};
|
||||
parse_unit_key_ro(&inputs.unit_key_ro, version)
|
||||
.map(|f| f.encrypted_keys.into_iter().map(|(_, k)| k).collect())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
Self {
|
||||
inner: inputs,
|
||||
enc_keys,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ResolveCtx for DiscInputsCtx<'_> {
|
||||
fn disc_hash(&self) -> &str {
|
||||
&self.inner.disc_hash
|
||||
}
|
||||
fn title(&self) -> Option<&str> {
|
||||
self.inner.volume_label.as_deref()
|
||||
}
|
||||
fn vid(&self) -> Option<Vid> {
|
||||
if self.inner.volume_id == [0u8; 16] {
|
||||
None
|
||||
} else {
|
||||
Some(Vid(self.inner.volume_id))
|
||||
}
|
||||
}
|
||||
fn mkb(&self) -> Result<&[u8], Error> {
|
||||
Ok(&self.inner.mkb)
|
||||
}
|
||||
fn enc_title_keys(&self) -> Result<&[[u8; 16]], Error> {
|
||||
Ok(&self.enc_keys)
|
||||
}
|
||||
fn samples(&self, n: usize) -> Result<Vec<Vec<u8>>, Error> {
|
||||
Ok(self.inner.samples.iter().take(n).cloned().collect())
|
||||
}
|
||||
fn unit_key_ro(&self) -> &[u8] {
|
||||
&self.inner.unit_key_ro
|
||||
}
|
||||
}
|
||||
|
||||
/// A key source: an adapter over a backing store that resolves a disc's terminal
|
||||
/// Unit Keys.
|
||||
///
|
||||
/// Dumb about *policy*, smart about *its own material*: given a [`ResolveCtx`] a
|
||||
/// source looks the disc up in its store and, from whatever level of material it
|
||||
/// holds, orchestrates the derivation down to Unit Keys using the library's
|
||||
/// boil-down crypto primitives — never re-implementing AES. A source that holds
|
||||
/// pre-decrypted Unit Keys returns them directly; one that holds a VUK calls
|
||||
/// [`crate::aacs::uk_from_vuk`]; one that holds device keys calls
|
||||
/// [`crate::aacs::mk_from_dk`] → [`crate::aacs::vuk_from_mk`] → `uk_from_vuk`.
|
||||
///
|
||||
/// Returning an empty `Vec` means "no key for this disc from this source"; an
|
||||
/// `Err` means the source itself failed (I/O, parse, network). The caller
|
||||
/// ([`resolve_and_apply`]) tries each source in order and validates the returned
|
||||
/// keys against real ciphertext before committing them, so a wrong key from one
|
||||
/// source transparently falls through to the next.
|
||||
pub trait KeySource {
|
||||
/// Resolve this disc's terminal Unit Keys from this source. An empty `Vec`
|
||||
/// is a genuine "no key here"; `Err` is a source failure.
|
||||
fn get_uk(&self, ctx: &dyn ResolveCtx) -> Result<Vec<UnitKey>, Error>;
|
||||
|
||||
/// The AACS host certificate(s) this source can supply for the live-drive
|
||||
/// SCSI mutual-auth handshake (the OEM/AACS baseline route). `mkb` is the
|
||||
/// disc's MKB generation when known, so a source MAY return only certs whose
|
||||
/// generation matches (the default ignores it). A host cert unlocks the
|
||||
/// authenticated bus so the drive reports the Volume ID and bus key; it is
|
||||
/// **perishable** (revocable on a drive's HRL), so it is served by a source,
|
||||
/// never compiled in. A source holding no cert returns the empty vec.
|
||||
fn host_certs(&self, _mkb: Option<u32>) -> Vec<HostCert> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// A short, stable identifier for this source kind (`"keydb"`, `"online"`,
|
||||
/// `"mapfile"`, …). For logging which source produced a key, and for
|
||||
/// composition/ordering. A format string, not user-facing English.
|
||||
fn label(&self) -> &'static str {
|
||||
"source"
|
||||
}
|
||||
}
|
||||
|
||||
/// Drive `sources` until one resolves Unit Keys that decrypt `disc`. Returns
|
||||
/// `true` at the first source whose keys validate and commit, `false` once every
|
||||
/// source is exhausted (the genuine "no key for this disc"). Thin wrapper over
|
||||
/// [`resolve_and_apply_traced`] that discards the trace.
|
||||
pub fn resolve_and_apply(
|
||||
sources: &[Box<dyn KeySource>],
|
||||
inputs: &DiscInputs,
|
||||
disc: &mut crate::Disc,
|
||||
) -> bool {
|
||||
resolve_and_apply_traced(sources, inputs, disc).0
|
||||
}
|
||||
|
||||
/// Like [`resolve_and_apply`] but also returns a structured
|
||||
/// [`crate::aacs::ResolutionTrace`] recording, per source, what happened — for
|
||||
/// applications to render. ZERO English; the trace is typed enums only.
|
||||
///
|
||||
/// One-shot per source: each source's [`KeySource::get_uk`] is called exactly
|
||||
/// once with a [`DiscInputsCtx`] over `inputs`. Non-empty Unit Keys are mapped
|
||||
/// to terminal [`Key::Unit`]s and applied via [`crate::Disc::decrypt_with`],
|
||||
/// which validates them against `inputs.samples` and only mutates the disc on
|
||||
/// success — so a wrong/partial key set is rejected and the loop continues.
|
||||
///
|
||||
/// CPS-unit numbering: a source returns Unit Keys carrying the POSITIONAL index
|
||||
/// from [`crate::aacs::uk_from_vuk`]; the library's canonical CPS-unit number is
|
||||
/// `position + 1` (matching [`crate::aacs::parse_unit_key_ro`]'s `(i + 1)`), so
|
||||
/// the committed `AacsState.unit_keys` is byte-identical to the library-resolved
|
||||
/// path. The number is cosmetic for descramble (the decrypt path strips it and
|
||||
/// tries every key) but is kept faithful to the resolver's convention.
|
||||
pub fn resolve_and_apply_traced(
|
||||
sources: &[Box<dyn KeySource>],
|
||||
inputs: &DiscInputs,
|
||||
disc: &mut crate::Disc,
|
||||
) -> (bool, crate::aacs::ResolutionTrace) {
|
||||
use crate::aacs::trace::{KeyNode, KeyOutcome, KeyStep};
|
||||
|
||||
let mut trace = crate::aacs::ResolutionTrace::new();
|
||||
|
||||
// AACS major drives the Unit_Key_RO.inf stride the ctx parses at. Default to
|
||||
// the V20/V21 stride when there is no AACS state (it is the common live case;
|
||||
// a non-AACS disc has nothing to resolve and the loop simply finds nothing).
|
||||
let version_u8 = disc.aacs.as_ref().map(|a| a.version).unwrap_or(2);
|
||||
let ctx = DiscInputsCtx::new(inputs, version_u8);
|
||||
|
||||
for source in sources {
|
||||
// `who` is the source's own stable identifier — no enum to map back to.
|
||||
let who = source.label().to_string();
|
||||
match source.get_uk(&ctx) {
|
||||
Ok(uks) if !uks.is_empty() => {
|
||||
// Positional index → canonical CPS-unit number (position + 1).
|
||||
let unit_keys: Vec<(u32, [u8; 16])> = uks
|
||||
.iter()
|
||||
.map(|uk| (uk.idx.saturating_add(1), uk.key))
|
||||
.collect();
|
||||
if disc
|
||||
.decrypt_with(Key::Unit(unit_keys), &inputs.samples)
|
||||
.is_ok()
|
||||
{
|
||||
trace.keys.push(KeyStep {
|
||||
who,
|
||||
path: vec![KeyNode::FoundUnitKeys, KeyNode::DerivedUnitKeys],
|
||||
outcome: KeyOutcome::Resolved,
|
||||
});
|
||||
return (true, trace);
|
||||
}
|
||||
// Keys produced but rejected by validation — record and continue.
|
||||
trace.keys.push(KeyStep {
|
||||
who,
|
||||
path: vec![KeyNode::FoundUnitKeys],
|
||||
outcome: KeyOutcome::NoKey,
|
||||
});
|
||||
}
|
||||
// Empty (no key here) or a source failure — both are "no key from
|
||||
// this source"; move on to the next.
|
||||
Ok(_) | Err(_) => {
|
||||
trace.keys.push(KeyStep {
|
||||
who,
|
||||
path: vec![KeyNode::NoEntry],
|
||||
outcome: KeyOutcome::NoKey,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
(false, trace)
|
||||
}
|
||||
|
||||
/// THE single key-fetch: drive `sources` in order and return the first non-empty
|
||||
/// Unit Key set. This is exactly what both paths do — only the samples differ:
|
||||
/// * at disc open, `ctx` carries reachable-content samples → resolves the
|
||||
/// up-front CPS units (the common one),
|
||||
/// * in the read, on a decrypt miss, `ctx` carries the FAILING unit's ciphertext
|
||||
/// → resolves the CPS unit that wasn't sampled up front.
|
||||
///
|
||||
/// Same sources, same call; there is no separate "fetch". Unlike
|
||||
/// [`resolve_and_apply`] this does not validate/commit to a disc — the read's
|
||||
/// decorator re-decrypts with the returned keys, which is the validation.
|
||||
pub fn fetch_unit_keys(sources: &[Box<dyn KeySource>], ctx: &dyn ResolveCtx) -> Vec<UnitKey> {
|
||||
for source in sources {
|
||||
if let Ok(uks) = source.get_uk(ctx) {
|
||||
if !uks.is_empty() {
|
||||
return uks;
|
||||
}
|
||||
}
|
||||
}
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// Build the read-time key-fetch closure from the disc's public AACS inputs and
|
||||
/// a way to (re)build the application's key sources. The decorator calls it with
|
||||
/// the still-scrambled unit ciphertext when no held key opens that unit; it runs
|
||||
/// [`fetch_unit_keys`] with those bytes as `samples` and returns any keys.
|
||||
///
|
||||
/// One builder, used by every read path (sweep / patch / mux) and by every
|
||||
/// consumer (CLI, autorip) — neither application contains the fetch logic, only
|
||||
/// its key-source config. Returns a **shared, stateless** [`crate::sector::KeyFetch`]
|
||||
/// (`Arc<Fn>`): build it once, clone it into each read path. `make_sources` is
|
||||
/// invoked per fetch (the cold path, ~once per CPS unit) so the closure stays
|
||||
/// `Send + Sync` without requiring `KeySource: Send`.
|
||||
pub fn key_fetch(
|
||||
inputs: DiscInputs,
|
||||
make_sources: std::sync::Arc<dyn Fn() -> Vec<Box<dyn KeySource>> + Send + Sync>,
|
||||
) -> crate::sector::KeyFetch {
|
||||
std::sync::Arc::new(move |samples: &[Vec<u8>]| -> Vec<[u8; 16]> {
|
||||
let sources = make_sources();
|
||||
let mut di = inputs.clone();
|
||||
di.samples = samples.to_vec();
|
||||
// V20/V21 stride (BD/UHD AACS 2.x); the online /decode UK path forwards
|
||||
// raw inf + samples and doesn't depend on the parsed title-key stride.
|
||||
let ctx = DiscInputsCtx::new(&di, 2);
|
||||
fetch_unit_keys(&sources, &ctx)
|
||||
.into_iter()
|
||||
.map(|u| u.key)
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
|
||||
/// Read up to `n` ENCRYPTED 6144-byte aligned units from `title`'s body, raw (no
|
||||
/// decrypt) — the content samples that populate [`DiscInputs::samples`] for a
|
||||
/// key server to validate a candidate against, and that [`resolve_and_apply`]
|
||||
/// hands to [`crate::Disc::decrypt_with`].
|
||||
///
|
||||
/// Lives in the library, not a key-source crate: reading the disc and carving
|
||||
/// AACS units is decryption *mechanism* (unit geometry anchored at each extent's
|
||||
/// `start_lba`), which the library owns. A key source is *handed* these bytes
|
||||
/// via `DiscInputs.samples`; it never reads the disc itself.
|
||||
///
|
||||
/// "Encrypted" is decided by [`crate::aacs::ts_sync_destroyed`] — the SAME
|
||||
/// predicate the decrypt gate uses — so all sides agree. A clip opens with clear
|
||||
/// navigation units (PAT/PMT, menus); only the feature body is scrambled, and a
|
||||
/// clear unit proves nothing, so this collects only scrambled ones, sampling the
|
||||
/// largest extent at its midpoint forward.
|
||||
pub fn read_encrypted_units(
|
||||
reader: &mut dyn crate::sector::SectorSource,
|
||||
title: &crate::disc::DiscTitle,
|
||||
n: usize,
|
||||
) -> Vec<Vec<u8>> {
|
||||
use crate::aacs::{ALIGNED_UNIT_LEN, ALIGNED_UNIT_SECTORS, ts_sync_destroyed};
|
||||
const CHUNK_UNITS: u32 = 15; // 45 sectors/read — under the drive transfer cap
|
||||
const MAX_CHUNKS_PER_EXTENT: u32 = 4; // ~60 units scanned at each extent's midpoint
|
||||
|
||||
let mut out: Vec<Vec<u8>> = Vec::new();
|
||||
for ext in &title.extents {
|
||||
let total_units = ext.sector_count / ALIGNED_UNIT_SECTORS;
|
||||
if total_units == 0 {
|
||||
continue;
|
||||
}
|
||||
let mut unit = total_units / 2; // midpoint (past the clear nav at the head)
|
||||
for _ in 0..MAX_CHUNKS_PER_EXTENT {
|
||||
if unit >= total_units {
|
||||
break;
|
||||
}
|
||||
let units_this = CHUNK_UNITS.min(total_units - unit);
|
||||
// Saturate: start_lba comes from attacker-controlled UDF/MPLS
|
||||
// extents; a malformed extent near u32::MAX would otherwise panic
|
||||
// (debug) or wrap to a wrong LBA (release). An over-capacity LBA then
|
||||
// fails cleanly via the read_sectors().is_err() break below.
|
||||
let lba = ext
|
||||
.start_lba
|
||||
.saturating_add(unit.saturating_mul(ALIGNED_UNIT_SECTORS));
|
||||
let count = (units_this * ALIGNED_UNIT_SECTORS) as u16;
|
||||
let mut buf = vec![0u8; count as usize * 2048];
|
||||
// `false` = no recovery retries; the reader is the raw drive/file
|
||||
// (no decrypt decorator), so these are the on-disc encrypted bytes.
|
||||
if reader.read_sectors(lba, count, &mut buf, false).is_err() {
|
||||
break;
|
||||
}
|
||||
for i in 0..units_this as usize {
|
||||
let o = i * ALIGNED_UNIT_LEN;
|
||||
if o + ALIGNED_UNIT_LEN > buf.len() {
|
||||
break;
|
||||
}
|
||||
let u = &buf[o..o + ALIGNED_UNIT_LEN];
|
||||
if ts_sync_destroyed(u) {
|
||||
out.push(u.to_vec());
|
||||
if out.len() >= n {
|
||||
return out;
|
||||
}
|
||||
}
|
||||
}
|
||||
unit += units_this;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::aacs::UnitKey;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
// ── KeySource default-method behaviour ────────────────────────────────────
|
||||
|
||||
/// KeySource::host_certs() defaults to empty regardless of the MKB argument.
|
||||
/// Spec: a source holding no cert returns the empty vec; the `mkb` param is
|
||||
/// forward-looking and the default ignores it.
|
||||
/// Mutation: a default returning a non-empty vec would inject phantom certs
|
||||
/// into the OEM handshake.
|
||||
#[test]
|
||||
fn key_source_host_certs_defaults_to_empty() {
|
||||
struct MinimalSource;
|
||||
impl KeySource for MinimalSource {
|
||||
fn get_uk(&self, _ctx: &dyn ResolveCtx) -> Result<Vec<UnitKey>, Error> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
}
|
||||
let s = MinimalSource;
|
||||
assert!(s.host_certs(None).is_empty());
|
||||
assert!(s.host_certs(Some(68)).is_empty());
|
||||
}
|
||||
|
||||
/// DiscInputsCtx maps DiscInputs faithfully: zero VID → None, non-zero VID →
|
||||
/// Some; title from volume_label; samples truncate to n; enc_title_keys
|
||||
/// parses Unit_Key_RO.inf at the version stride.
|
||||
#[test]
|
||||
fn disc_inputs_ctx_maps_fields() {
|
||||
// Build a minimal V10 Unit_Key_RO.inf with one key (stride 48):
|
||||
// uk_pos = 32, num_uk = 1, key at uk_pos + 48 = 80.
|
||||
let mut uk_ro = vec![0u8; 96];
|
||||
let uk_pos = 32usize;
|
||||
uk_ro[0..4].copy_from_slice(&(uk_pos as u32).to_be_bytes());
|
||||
uk_ro[uk_pos] = 0x00;
|
||||
uk_ro[uk_pos + 1] = 0x01; // num_unit_keys = 1
|
||||
let key_bytes = [0x7Eu8; 16];
|
||||
uk_ro[80..96].copy_from_slice(&key_bytes);
|
||||
|
||||
let inputs = DiscInputs {
|
||||
disc_hash: "0xABC".into(),
|
||||
volume_id: [0u8; 16],
|
||||
mkb: vec![1, 2, 3],
|
||||
unit_key_ro: uk_ro,
|
||||
samples: vec![vec![9u8; 4], vec![8u8; 4], vec![7u8; 4]],
|
||||
volume_label: Some("TITLE_X".into()),
|
||||
};
|
||||
|
||||
// Zero VID → None.
|
||||
let ctx = DiscInputsCtx::new(&inputs, 1);
|
||||
assert_eq!(ctx.disc_hash(), "0xABC");
|
||||
assert_eq!(ctx.title(), Some("TITLE_X"));
|
||||
assert!(ctx.vid().is_none(), "all-zero VID is the no-VID sentinel");
|
||||
assert_eq!(ctx.mkb().unwrap(), &[1, 2, 3]);
|
||||
assert_eq!(ctx.enc_title_keys().unwrap(), &[key_bytes]);
|
||||
assert_eq!(ctx.samples(2).unwrap().len(), 2, "samples truncates to n");
|
||||
|
||||
// Non-zero VID → Some(vid).
|
||||
let mut inputs2 = inputs.clone();
|
||||
inputs2.volume_id = [0x42u8; 16];
|
||||
let ctx2 = DiscInputsCtx::new(&inputs2, 1);
|
||||
assert_eq!(ctx2.vid(), Some(Vid([0x42u8; 16])));
|
||||
}
|
||||
|
||||
/// `resolve_and_apply_traced` records each step's `who` as the source's own
|
||||
/// `label()`, carried verbatim — no enum round-trip. A source with a custom
|
||||
/// label surfaces it as-is in the trace.
|
||||
#[test]
|
||||
fn trace_who_is_the_source_label_verbatim() {
|
||||
struct LabeledSource(&'static str);
|
||||
impl KeySource for LabeledSource {
|
||||
fn get_uk(&self, _ctx: &dyn ResolveCtx) -> Result<Vec<UnitKey>, Error> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
fn label(&self) -> &'static str {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
let mut disc = crate::Disc {
|
||||
volume_id: String::new(),
|
||||
meta_title: None,
|
||||
format: crate::DiscFormat::BluRay,
|
||||
capacity_sectors: 0,
|
||||
capacity_bytes: 0,
|
||||
layers: 1,
|
||||
titles: Vec::new(),
|
||||
region: crate::disc::DiscRegion::Free,
|
||||
aacs: None,
|
||||
css: None,
|
||||
encrypted: false,
|
||||
aacs_error: None,
|
||||
css_error: None,
|
||||
content_format: crate::ContentFormat::BdTs,
|
||||
};
|
||||
let inputs = DiscInputs {
|
||||
disc_hash: "0x00".into(),
|
||||
volume_id: [0u8; 16],
|
||||
mkb: Vec::new(),
|
||||
unit_key_ro: Vec::new(),
|
||||
samples: Vec::new(),
|
||||
volume_label: None,
|
||||
};
|
||||
let sources: Vec<Box<dyn KeySource>> = vec![
|
||||
Box::new(LabeledSource("keydb")),
|
||||
Box::new(LabeledSource("my-custom-source")),
|
||||
];
|
||||
let (_ok, trace) = resolve_and_apply_traced(&sources, &inputs, &mut disc);
|
||||
let whos: Vec<&str> = trace.keys.iter().map(|s| s.who.as_str()).collect();
|
||||
assert_eq!(whos, vec!["keydb", "my-custom-source"]);
|
||||
}
|
||||
|
||||
// ── fetch_unit_keys / key_fetch (the one shared fetch path) ───────────────
|
||||
|
||||
fn empty_inputs() -> DiscInputs {
|
||||
DiscInputs {
|
||||
disc_hash: String::new(),
|
||||
volume_id: [0u8; 16],
|
||||
mkb: Vec::new(),
|
||||
unit_key_ro: Vec::new(),
|
||||
samples: Vec::new(),
|
||||
volume_label: None,
|
||||
}
|
||||
}
|
||||
|
||||
struct EmptySource;
|
||||
impl KeySource for EmptySource {
|
||||
fn get_uk(&self, _ctx: &dyn ResolveCtx) -> Result<Vec<UnitKey>, Error> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
}
|
||||
struct ErroringSource;
|
||||
impl KeySource for ErroringSource {
|
||||
fn get_uk(&self, _ctx: &dyn ResolveCtx) -> Result<Vec<UnitKey>, Error> {
|
||||
Err(Error::AacsNoKeys)
|
||||
}
|
||||
}
|
||||
struct HasKey([u8; 16]);
|
||||
impl KeySource for HasKey {
|
||||
fn get_uk(&self, _ctx: &dyn ResolveCtx) -> Result<Vec<UnitKey>, Error> {
|
||||
Ok(vec![UnitKey {
|
||||
idx: 0,
|
||||
key: self.0,
|
||||
}])
|
||||
}
|
||||
}
|
||||
|
||||
/// `fetch_unit_keys` returns the FIRST source's non-empty keys, skipping a
|
||||
/// source that returns empty or errors; empty when no source answers.
|
||||
#[test]
|
||||
fn fetch_unit_keys_first_nonempty_skips_empty_and_errors() {
|
||||
let inputs = empty_inputs();
|
||||
let ctx = DiscInputsCtx::new(&inputs, 2);
|
||||
let key = [0xABu8; 16];
|
||||
|
||||
let sources: Vec<Box<dyn KeySource>> = vec![
|
||||
Box::new(EmptySource),
|
||||
Box::new(ErroringSource),
|
||||
Box::new(HasKey(key)),
|
||||
];
|
||||
let got = fetch_unit_keys(&sources, &ctx);
|
||||
assert_eq!(got.len(), 1, "the first source that answers wins");
|
||||
assert_eq!(got[0].key, key);
|
||||
|
||||
let none: Vec<Box<dyn KeySource>> = vec![Box::new(EmptySource), Box::new(ErroringSource)];
|
||||
assert!(
|
||||
fetch_unit_keys(&none, &ctx).is_empty(),
|
||||
"no source answers ⇒ empty"
|
||||
);
|
||||
}
|
||||
|
||||
/// `key_fetch` builds a closure that runs the sources with the GIVEN failing
|
||||
/// samples and returns their keys — the exact bytes are forwarded to the
|
||||
/// source, and `make_sources` is invoked per call.
|
||||
#[test]
|
||||
fn key_fetch_closure_forwards_samples_and_returns_keys() {
|
||||
let key = [0x5au8; 16];
|
||||
let seen: Arc<Mutex<Vec<Vec<u8>>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let builds = Arc::new(Mutex::new(0usize));
|
||||
|
||||
struct Probe {
|
||||
key: [u8; 16],
|
||||
seen: Arc<Mutex<Vec<Vec<u8>>>>,
|
||||
}
|
||||
impl KeySource for Probe {
|
||||
fn get_uk(&self, ctx: &dyn ResolveCtx) -> Result<Vec<UnitKey>, Error> {
|
||||
if let Ok(s) = ctx.samples(8) {
|
||||
self.seen.lock().unwrap().extend(s);
|
||||
}
|
||||
Ok(vec![UnitKey {
|
||||
idx: 0,
|
||||
key: self.key,
|
||||
}])
|
||||
}
|
||||
}
|
||||
|
||||
let seen_c = Arc::clone(&seen);
|
||||
let builds_c = Arc::clone(&builds);
|
||||
let make: Arc<dyn Fn() -> Vec<Box<dyn KeySource>> + Send + Sync> = Arc::new(move || {
|
||||
*builds_c.lock().unwrap() += 1;
|
||||
vec![Box::new(Probe {
|
||||
key,
|
||||
seen: Arc::clone(&seen_c),
|
||||
}) as Box<dyn KeySource>]
|
||||
});
|
||||
|
||||
let cb = key_fetch(empty_inputs(), make);
|
||||
let samples = vec![vec![0xEEu8; crate::aacs::ALIGNED_UNIT_LEN]];
|
||||
let got = cb(&samples);
|
||||
assert_eq!(
|
||||
got,
|
||||
vec![key],
|
||||
"the source's key flows back through the closure"
|
||||
);
|
||||
assert_eq!(
|
||||
seen.lock().unwrap().len(),
|
||||
1,
|
||||
"the failing ciphertext sample is forwarded to the source"
|
||||
);
|
||||
assert_eq!(*builds.lock().unwrap(), 1, "make_sources invoked per fetch");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,569 @@
|
||||
//! BDMV disc-library metadata (`/BDMV/META/DL/bdmt_<lang>.xml`).
|
||||
//!
|
||||
//! Every commercial Blu-ray carries a disc-library metadata directory
|
||||
//! with one XML file per shipped language. The schema is the Blu-ray
|
||||
//! "disc library metadata" namespace (`urn:BDA:bdmv;disclibmeta`),
|
||||
//! conventionally prefixed `di:`. Fields commonly present:
|
||||
//!
|
||||
//! - `<di:title>` or `<di:name>` — the title string. Vendor practice
|
||||
//! varies (Paramount discs tend to use `<di:name>`).
|
||||
//! - `<di:description>` — optional synopsis (often absent on retail
|
||||
//! discs; common on box sets and special editions).
|
||||
//! - `<di:discNumber>` / `<di:numSets>` (or `<di:numberOfSets>`) —
|
||||
//! set position for multi-disc releases.
|
||||
//!
|
||||
//! This module is intentionally separate from the BD-J `StreamLabel`
|
||||
//! parsers under `labels/*.rs`. The XML here is disc-level (title,
|
||||
//! description, set position), not per-stream. It is invoked from the
|
||||
//! disc-scan path in [`labels`](super) ([`detect`] then [`parse`]),
|
||||
//! and [`DiscMetadata`] is re-exported there.
|
||||
//!
|
||||
//! Real-world XML is irregular: missing description elements, multiple
|
||||
//! title elements (first one wins), and occasional malformed content.
|
||||
//! Extraction is best-effort — a malformed file is treated as "no
|
||||
//! metadata" (returns `None` from the helper), and the caller can
|
||||
//! still get metadata from sibling-language XML files.
|
||||
|
||||
use super::xml;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Upper bound on the size of a single `bdmt_<lang>.xml` we will read.
|
||||
/// The size comes from attacker-controlled UDF metadata; real files are
|
||||
/// a few KB, so 1 MiB is generous while preventing a crafted huge-size
|
||||
/// entry from triggering an oversized allocation in `read_file`.
|
||||
const MAX_BDMT_BYTES: u64 = 1024 * 1024;
|
||||
|
||||
/// Disc-level metadata extracted from `/BDMV/META/DL/bdmt_*.xml`.
|
||||
///
|
||||
/// All maps are keyed by 3-char ISO 639-2 language code (e.g.
|
||||
/// `"eng"`, `"fra"`, `"jpn"`) — the same key segment used in the
|
||||
/// `bdmt_<lang>.xml` filename.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)]
|
||||
pub struct DiscMetadata {
|
||||
/// Localized titles, keyed by 3-char ISO 639-2 lang code
|
||||
/// (e.g. "eng" → "Aurora Drift")
|
||||
pub titles: BTreeMap<String, String>,
|
||||
/// First-line / short description, per lang
|
||||
pub descriptions: BTreeMap<String, String>,
|
||||
/// Disc N of M for box sets (None if not a box set)
|
||||
pub disc_number: Option<(u32, u32)>,
|
||||
}
|
||||
|
||||
/// True if `/BDMV/META/DL/` exists and contains at least one
|
||||
/// `bdmt_*.xml` file.
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
let Some(dir) = udf.find_dir("/BDMV/META/DL") else {
|
||||
return false;
|
||||
};
|
||||
dir.entries
|
||||
.iter()
|
||||
.any(|e| !e.is_dir && is_bdmt_filename(&e.name))
|
||||
}
|
||||
|
||||
/// Read every `bdmt_<lang>.xml` under `/BDMV/META/DL/` and return the
|
||||
/// aggregated [`DiscMetadata`]. Returns `None` if no titles could be
|
||||
/// extracted from any file.
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<DiscMetadata> {
|
||||
let dir = udf.find_dir("/BDMV/META/DL")?;
|
||||
let mut out = DiscMetadata::default();
|
||||
|
||||
for entry in &dir.entries {
|
||||
if entry.is_dir {
|
||||
continue;
|
||||
}
|
||||
let Some(lang) = lang_code_from_filename(&entry.name) else {
|
||||
continue;
|
||||
};
|
||||
// entry.size is attacker-controlled UDF metadata and flows into
|
||||
// a Vec::with_capacity in read_file. A real BDMV bdmt XML is a
|
||||
// few KB; cap well above that so a crafted multi-GB size can't
|
||||
// trigger a huge allocation before any parsing.
|
||||
if !bdmt_size_acceptable(entry.size) {
|
||||
continue;
|
||||
}
|
||||
let path = format!("/BDMV/META/DL/{}", entry.name);
|
||||
let Ok(bytes) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(text) = std::str::from_utf8(&bytes) else {
|
||||
continue;
|
||||
};
|
||||
let Some((title, description, disc_set)) = parse_bdmt_xml(text) else {
|
||||
continue;
|
||||
};
|
||||
out.titles.insert(lang.clone(), title);
|
||||
if let Some(desc) = description {
|
||||
out.descriptions.insert(lang.clone(), desc);
|
||||
}
|
||||
// Disc-set position is disc-global; first one we successfully
|
||||
// read wins. (All bdmt_*.xml on a given disc carry the same
|
||||
// value in practice.)
|
||||
if out.disc_number.is_none() {
|
||||
if let Some(ds) = disc_set {
|
||||
out.disc_number = Some(ds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if out.titles.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Gate a `bdmt_<lang>.xml` file by its declared (untrusted) UDF size
|
||||
/// before reading it. Anything over [`MAX_BDMT_BYTES`] is skipped to
|
||||
/// avoid an oversized allocation in `read_file`.
|
||||
fn bdmt_size_acceptable(size: u64) -> bool {
|
||||
size <= MAX_BDMT_BYTES
|
||||
}
|
||||
|
||||
/// True if `name` matches the `bdmt_<lang>.xml` convention with a
|
||||
/// 3-character ISO 639-2 lang code segment. Case-insensitive.
|
||||
fn is_bdmt_filename(name: &str) -> bool {
|
||||
lang_code_from_filename(name).is_some()
|
||||
}
|
||||
|
||||
/// Extract the 3-char language code from a `bdmt_<lang>.xml` filename.
|
||||
/// Returns `None` if the filename doesn't match. Lang code is
|
||||
/// lowercased so callers always see e.g. `"eng"` not `"ENG"`.
|
||||
fn lang_code_from_filename(name: &str) -> Option<String> {
|
||||
let lower = name.to_ascii_lowercase();
|
||||
let stem = lower.strip_suffix(".xml")?;
|
||||
let lang = stem.strip_prefix("bdmt_")?;
|
||||
// ISO 639-2 codes are exactly 3 ASCII letters. Be strict — keeps
|
||||
// us from picking up unrelated `bdmt_foo.xml` siblings.
|
||||
if lang.len() != 3 || !lang.chars().all(|c| c.is_ascii_alphabetic()) {
|
||||
return None;
|
||||
}
|
||||
Some(lang.to_string())
|
||||
}
|
||||
|
||||
/// Tuple returned by [`parse_bdmt_xml`]: `(title, description?, disc_set?)`.
|
||||
/// Aliased so the function signature isn't a clippy::type-complexity offender.
|
||||
pub(crate) type BdmtFields = (String, Option<String>, Option<(u32, u32)>);
|
||||
|
||||
/// Parse one `bdmt_<lang>.xml` document and return
|
||||
/// `(title, description?, disc_set?)`. Returns `None` if no title
|
||||
/// could be located — the caller treats this as "skip this file".
|
||||
///
|
||||
/// Title-element preference: `<di:name>` → `<di:title>` →
|
||||
/// `<di:tableOfContents>/<di:titleName>` (first match wins, per the
|
||||
/// authoring-tool conventions documented at the module level).
|
||||
pub(crate) fn parse_bdmt_xml(xml_text: &str) -> Option<BdmtFields> {
|
||||
let title = extract_title(xml_text)?;
|
||||
// xml::text already returns a trimmed string (see xml::text), so the
|
||||
// description is only filtered for emptiness and XML-fragment noise.
|
||||
let description = xml::text(xml_text, "description")
|
||||
.filter(|s| !s.is_empty())
|
||||
.filter(|s| !looks_like_xml(s));
|
||||
let disc_set = extract_disc_set(xml_text);
|
||||
Some((title, description, disc_set))
|
||||
}
|
||||
|
||||
/// Reject candidate description strings that are themselves XML
|
||||
/// fragments — observed on a captured disc, where
|
||||
/// `<di:description>` contained `<di:thumbnail href="…"/>` child
|
||||
/// elements and no actual prose. Surfacing that raw to the JSON
|
||||
/// output is worse than dropping the field entirely.
|
||||
fn looks_like_xml(s: &str) -> bool {
|
||||
let t = s.trim_start();
|
||||
t.starts_with('<')
|
||||
}
|
||||
|
||||
/// Try title-bearing element variants in priority order. The `xml`
|
||||
/// helpers are case- and namespace-insensitive, so callers pass the
|
||||
/// bare local name (no `di:` prefix).
|
||||
fn extract_title(xml_text: &str) -> Option<String> {
|
||||
// Order matches the module-level convention: <di:name> first
|
||||
// (Paramount-style), then <di:title>, then the nested
|
||||
// tableOfContents/titleName form.
|
||||
// xml::text already trims its result, so an empty string after
|
||||
// extraction means a genuinely empty element.
|
||||
for tag in ["name", "title"] {
|
||||
if let Some(s) = xml::text(xml_text, tag) {
|
||||
if !s.is_empty() {
|
||||
return Some(s);
|
||||
}
|
||||
}
|
||||
}
|
||||
// tableOfContents/titleName: search inside the toc block so we
|
||||
// don't accidentally pick a stray <titleName> from elsewhere.
|
||||
if let Some((s, e)) = xml::find_element(xml_text, "tableOfContents", 0) {
|
||||
let block = &xml_text[s..e];
|
||||
if let Some(t) = xml::text(block, "titleName") {
|
||||
if !t.is_empty() {
|
||||
return Some(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Extract `(discNumber, numSets)` if both are present and parse as
|
||||
/// `u32`. Accepts either `<di:numSets>` or `<di:numberOfSets>` for
|
||||
/// the denominator (both forms appear in the wild).
|
||||
fn extract_disc_set(xml_text: &str) -> Option<(u32, u32)> {
|
||||
let n = xml::text(xml_text, "discNumber")?
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.ok()?;
|
||||
let total = xml::text(xml_text, "numSets")
|
||||
.or_else(|| xml::text(xml_text, "numberOfSets"))?
|
||||
.trim()
|
||||
.parse::<u32>()
|
||||
.ok()?;
|
||||
// Reject nonsensical "Disc N of M" values: (0,0), (0,5), (5,2)...
|
||||
// These serialize to JSON and reach downstream consumers as
|
||||
// meaningless metadata.
|
||||
if n < 1 || total < 1 || n > total {
|
||||
return None;
|
||||
}
|
||||
Some((n, total))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn extract_simple_title() {
|
||||
// Minimal Paramount-style document: <di:name> as the title
|
||||
// carrier inside a <discInfo> root.
|
||||
let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Aurora Drift</di:name>
|
||||
</discInfo>"#;
|
||||
let (title, desc, set) = parse_bdmt_xml(xml).expect("title should parse");
|
||||
assert_eq!(title, "Aurora Drift");
|
||||
assert_eq!(desc, None);
|
||||
assert_eq!(set, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_title_element_variant() {
|
||||
// <di:title> is the alternate carrier; should be picked up
|
||||
// when <di:name> is absent.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:title>Echo Chamber</di:title>
|
||||
<di:description>A film about machines.</di:description>
|
||||
</discInfo>"#;
|
||||
let (title, desc, _) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(title, "Echo Chamber");
|
||||
assert_eq!(desc.as_deref(), Some("A film about machines."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_title_from_table_of_contents_fallback() {
|
||||
// Some authoring tools nest the title under tableOfContents.
|
||||
// No <di:name> or <di:title> at top level → fall back to
|
||||
// titleName inside tableOfContents.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:tableOfContents>
|
||||
<di:titleName>Feelings Two</di:titleName>
|
||||
</di:tableOfContents>
|
||||
</discInfo>"#;
|
||||
let (title, _, _) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(title, "Feelings Two");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bdmt_size_gate_rejects_oversized_entries() {
|
||||
assert!(bdmt_size_acceptable(0));
|
||||
assert!(bdmt_size_acceptable(4096));
|
||||
assert!(bdmt_size_acceptable(MAX_BDMT_BYTES));
|
||||
assert!(!bdmt_size_acceptable(MAX_BDMT_BYTES + 1));
|
||||
// A crafted multi-GB size is rejected before any allocation.
|
||||
assert!(!bdmt_size_acceptable(8 * 1024 * 1024 * 1024));
|
||||
assert!(!bdmt_size_acceptable(u64::MAX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disc_set_rejects_nonsensical_pairs() {
|
||||
// n > total, zero numerator, zero denominator → all None.
|
||||
let over = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>X</di:name>
|
||||
<di:discNumber>5</di:discNumber>
|
||||
<di:numSets>2</di:numSets>
|
||||
</discInfo>"#;
|
||||
assert_eq!(parse_bdmt_xml(over).unwrap().2, None);
|
||||
|
||||
let zero_n = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>X</di:name>
|
||||
<di:discNumber>0</di:discNumber>
|
||||
<di:numSets>5</di:numSets>
|
||||
</discInfo>"#;
|
||||
assert_eq!(parse_bdmt_xml(zero_n).unwrap().2, None);
|
||||
|
||||
let zero_total = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>X</di:name>
|
||||
<di:discNumber>1</di:discNumber>
|
||||
<di:numSets>0</di:numSets>
|
||||
</discInfo>"#;
|
||||
assert_eq!(parse_bdmt_xml(zero_total).unwrap().2, None);
|
||||
|
||||
// A valid pair still passes.
|
||||
let ok = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>X</di:name>
|
||||
<di:discNumber>2</di:discNumber>
|
||||
<di:numSets>3</di:numSets>
|
||||
</discInfo>"#;
|
||||
assert_eq!(parse_bdmt_xml(ok).unwrap().2, Some((2, 3)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_box_set_position() {
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Box Set Disc 2</di:name>
|
||||
<di:discNumber>2</di:discNumber>
|
||||
<di:numSets>5</di:numSets>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(set, Some((2, 5)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_box_set_position_alternate_total_tag() {
|
||||
// <di:numberOfSets> is an alternate spelling we've seen.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>X</di:name>
|
||||
<di:discNumber>3</di:discNumber>
|
||||
<di:numberOfSets>6</di:numberOfSets>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(set, Some((3, 6)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_box_set_requires_both_fields() {
|
||||
// discNumber alone (no total) yields None — we don't fabricate
|
||||
// a denominator.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>X</di:name>
|
||||
<di:discNumber>1</di:discNumber>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(set, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_languages_keyed_correctly() {
|
||||
// Simulate driving parse_bdmt_xml from two synthetic XML
|
||||
// blobs and aggregating into DiscMetadata the same way parse()
|
||||
// would. This exercises the BTreeMap key handling without
|
||||
// needing a UdfFs.
|
||||
let eng_xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Aurora Drift</di:name>
|
||||
</discInfo>"#;
|
||||
let fra_xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Aurora Drift (Partie Deux)</di:name>
|
||||
<di:description>Suite du film fictif.</di:description>
|
||||
</discInfo>"#;
|
||||
|
||||
let mut meta = DiscMetadata::default();
|
||||
for (lang, blob) in [("eng", eng_xml), ("fra", fra_xml)] {
|
||||
let (title, desc, ds) = parse_bdmt_xml(blob).unwrap();
|
||||
meta.titles.insert(lang.to_string(), title);
|
||||
if let Some(d) = desc {
|
||||
meta.descriptions.insert(lang.to_string(), d);
|
||||
}
|
||||
if meta.disc_number.is_none() {
|
||||
if let Some(d) = ds {
|
||||
meta.disc_number = Some(d);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
meta.titles.get("eng").map(String::as_str),
|
||||
Some("Aurora Drift")
|
||||
);
|
||||
assert_eq!(
|
||||
meta.titles.get("fra").map(String::as_str),
|
||||
Some("Aurora Drift (Partie Deux)")
|
||||
);
|
||||
assert!(!meta.descriptions.contains_key("eng"));
|
||||
assert_eq!(
|
||||
meta.descriptions.get("fra").map(String::as_str),
|
||||
Some("Suite du film fictif.")
|
||||
);
|
||||
assert_eq!(meta.disc_number, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_xml_returns_none() {
|
||||
// Random gibberish has no recognizable title element. We
|
||||
// document the contract: parse_bdmt_xml returns None, and
|
||||
// parse() (the caller) skips the file. Aggregating across
|
||||
// zero files leaves DiscMetadata::default() — which parse()
|
||||
// surfaces as None to its caller. Either is documented as
|
||||
// acceptable per the module spec.
|
||||
let bad = "this is not xml &&& <<< nope";
|
||||
assert!(parse_bdmt_xml(bad).is_none());
|
||||
|
||||
// Half-open tag, no body, no close: also yields no title.
|
||||
let truncated = "<discInfo><di:name>";
|
||||
assert!(parse_bdmt_xml(truncated).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn description_with_only_child_xml_is_dropped() {
|
||||
// Real-world bug: <di:description> contained only
|
||||
// <di:thumbnail/> child elements with no actual prose. The
|
||||
// previous parser surfaced the raw XML fragment as the
|
||||
// description string. Now we reject candidates that begin
|
||||
// with `<`.
|
||||
let xml = r#"<discInfo>
|
||||
<di:name>Skyline Run</di:name>
|
||||
<di:description>
|
||||
<di:thumbnail href="sample_meta_sm.jpg" />
|
||||
<di:thumbnail href="sample_meta_lg.jpg" />
|
||||
</di:description>
|
||||
</discInfo>"#;
|
||||
let (title, description, _) =
|
||||
parse_bdmt_xml(xml).expect("title is present so parse must succeed");
|
||||
assert_eq!(title, "Skyline Run");
|
||||
assert!(
|
||||
description.is_none(),
|
||||
"description containing only XML children must be dropped, got {description:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn description_with_plain_text_passes_through() {
|
||||
// The legitimate case still works: a description with actual
|
||||
// prose survives the looks_like_xml filter.
|
||||
let xml = r#"<discInfo>
|
||||
<di:name>Some Movie</di:name>
|
||||
<di:description>An epic tale of one man's quest for tea.</di:description>
|
||||
</discInfo>"#;
|
||||
let (_, description, _) = parse_bdmt_xml(xml).expect("must parse");
|
||||
assert_eq!(
|
||||
description.as_deref(),
|
||||
Some("An epic tale of one man's quest for tea.")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whitespace_in_title_is_trimmed() {
|
||||
let xml = r#"<discInfo><di:name>
|
||||
Aurora Drift
|
||||
</di:name></discInfo>"#;
|
||||
let (title, _, _) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(title, "Aurora Drift");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lang_code_extraction() {
|
||||
assert_eq!(lang_code_from_filename("bdmt_eng.xml"), Some("eng".into()));
|
||||
assert_eq!(lang_code_from_filename("BDMT_FRA.XML"), Some("fra".into()));
|
||||
assert_eq!(lang_code_from_filename("bdmt_jpn.xml"), Some("jpn".into()));
|
||||
// Non-matching cases:
|
||||
assert_eq!(lang_code_from_filename("bdmt_.xml"), None);
|
||||
assert_eq!(lang_code_from_filename("bdmt_engl.xml"), None);
|
||||
assert_eq!(lang_code_from_filename("bdmt_e1g.xml"), None);
|
||||
assert_eq!(lang_code_from_filename("bdmt_eng.txt"), None);
|
||||
assert_eq!(lang_code_from_filename("foo.xml"), None);
|
||||
}
|
||||
|
||||
// ── Additional hardening tests ─────────────────────────────────────────
|
||||
|
||||
/// Spec reference: BDA disc-library metadata schema, §3.3.2 — `<di:name>`
|
||||
/// takes priority over `<di:title>` as the title carrier.
|
||||
/// Mutation: swap `di:name` to `di:other` → test goes red because title is None.
|
||||
#[test]
|
||||
fn di_name_priority_over_di_title() {
|
||||
// When BOTH di:name and di:title are present, di:name wins.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Primary Title</di:name>
|
||||
<di:title>Fallback Title</di:title>
|
||||
</discInfo>"#;
|
||||
let (title, _, _) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(title, "Primary Title");
|
||||
}
|
||||
|
||||
/// Spec reference: BDA disc-library metadata §3.3.2 — `<di:tableOfContents>`
|
||||
/// with nested `<di:titleName>` is a vendor-specific variant.
|
||||
/// Mutation: rename `titleName` → `movieName` → test goes red (None).
|
||||
#[test]
|
||||
fn di_name_wins_over_table_of_contents_title_name() {
|
||||
// di:name exists — tableOfContents/titleName must NOT override it.
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Winner</di:name>
|
||||
<di:tableOfContents>
|
||||
<di:titleName>Loser</di:titleName>
|
||||
</di:tableOfContents>
|
||||
</discInfo>"#;
|
||||
let (title, _, _) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(title, "Winner");
|
||||
}
|
||||
|
||||
/// Spec reference: BDA §3.3.2 — an empty `<di:name>` element must be
|
||||
/// treated as absent, falling through to the next candidate.
|
||||
/// Mutation: change `<di:name></di:name>` to `<di:name>X</di:name>` → red.
|
||||
#[test]
|
||||
fn empty_di_name_falls_through_to_di_title() {
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name></di:name>
|
||||
<di:title>Non-Empty Title</di:title>
|
||||
</discInfo>"#;
|
||||
let (title, _, _) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(title, "Non-Empty Title");
|
||||
}
|
||||
|
||||
/// Mutation: remove the `!s.is_empty()` filter → empty descriptions
|
||||
/// come through as Some("").
|
||||
#[test]
|
||||
fn empty_description_element_filtered_out() {
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Film</di:name>
|
||||
<di:description></di:description>
|
||||
</discInfo>"#;
|
||||
let (_, description, _) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(description, None);
|
||||
}
|
||||
|
||||
/// Disc N of N (e.g. 3 of 3) is valid — not an off-by-one error.
|
||||
/// Mutation: change `n > total` to `n >= total` → last disc of set is None.
|
||||
#[test]
|
||||
fn disc_set_allows_last_disc_equal_total() {
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Film</di:name>
|
||||
<di:discNumber>3</di:discNumber>
|
||||
<di:numSets>3</di:numSets>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(set, Some((3, 3)));
|
||||
}
|
||||
|
||||
/// When `<di:discNumber>` has non-numeric text, disc_number must be None.
|
||||
/// Mutation: remove the `.parse::<u32>().ok()?` guard → panics or wrong value.
|
||||
#[test]
|
||||
fn disc_set_non_numeric_disc_number_yields_none() {
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name>Film</di:name>
|
||||
<di:discNumber>one</di:discNumber>
|
||||
<di:numSets>5</di:numSets>
|
||||
</discInfo>"#;
|
||||
let (_, _, set) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(set, None);
|
||||
}
|
||||
|
||||
/// Whitespace-only title element must be treated as empty (trimmed → "").
|
||||
/// Spec: xml::text trims; an all-whitespace element produces "" after trim,
|
||||
/// which the title-extraction logic should skip.
|
||||
/// Mutation: remove the `!s.is_empty()` guard in extract_title →
|
||||
/// whitespace-only di:name would be returned as the title.
|
||||
#[test]
|
||||
fn whitespace_only_di_name_falls_through() {
|
||||
let xml = r#"<discInfo xmlns:di="urn:BDA:bdmv;disclibmeta">
|
||||
<di:name> </di:name>
|
||||
<di:title>Real Title</di:title>
|
||||
</discInfo>"#;
|
||||
let (title, _, _) = parse_bdmt_xml(xml).unwrap();
|
||||
assert_eq!(title, "Real Title");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,418 @@
|
||||
//! CLPI vs MPLS cross-validation diagnostic.
|
||||
//!
|
||||
//! Walks both per-clip CLPI program info and per-playlist MPLS STN
|
||||
//! tables, normalizes their stream lists by `(PID, language,
|
||||
//! coding_type)`, and classifies each PID into one of four buckets:
|
||||
//!
|
||||
//! 1. **CLPI only** — a stream present in a `.clpi` ProgramInfo that no
|
||||
//! playlist STN table references (orphan on disc).
|
||||
//! 2. **MPLS only** — a stream a playlist references that no `.clpi`
|
||||
//! ProgramInfo lists (indicates a parser disagreement).
|
||||
//! 3. **Match** — both sources agree on coding_type and language.
|
||||
//! 4. **Divergent** — both sources see the PID but disagree on
|
||||
//! coding_type or language.
|
||||
//!
|
||||
//! [`audit`] returns a structured [`ClpiVsMplsAudit`] report. This is a
|
||||
//! diagnostic surface only; it does not feed the label-selection
|
||||
//! pipeline.
|
||||
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// One row in the audit: a stream PID that's known to one source or
|
||||
/// both, with the fields each source reported.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ClpiVsMplsRow {
|
||||
pub pid: u16,
|
||||
pub clpi_coding_type: Option<u8>,
|
||||
pub clpi_language: Option<String>,
|
||||
pub mpls_coding_type: Option<u8>,
|
||||
pub mpls_language: Option<String>,
|
||||
}
|
||||
|
||||
impl ClpiVsMplsRow {
|
||||
/// Classification rules:
|
||||
/// - one coding_type present, the other missing → `ClpiOnly` /
|
||||
/// `MplsOnly`
|
||||
/// - both coding_types present, fields differ → `Divergent`
|
||||
/// - both coding_types present and identical → `Match`
|
||||
/// - both coding_types missing (`audit` never builds this, but a
|
||||
/// caller can construct such a row) → compare the language fields:
|
||||
/// `Divergent` if they differ, else `Match`
|
||||
pub fn class(&self) -> ClpiVsMplsClass {
|
||||
match (
|
||||
self.clpi_coding_type.is_some(),
|
||||
self.mpls_coding_type.is_some(),
|
||||
) {
|
||||
(true, false) => ClpiVsMplsClass::ClpiOnly,
|
||||
(false, true) => ClpiVsMplsClass::MplsOnly,
|
||||
(true, true) => {
|
||||
let coding_match = self.clpi_coding_type == self.mpls_coding_type;
|
||||
let lang_match = self.clpi_language == self.mpls_language;
|
||||
if coding_match && lang_match {
|
||||
ClpiVsMplsClass::Match
|
||||
} else {
|
||||
ClpiVsMplsClass::Divergent
|
||||
}
|
||||
}
|
||||
(false, false) => {
|
||||
if self.clpi_language == self.mpls_language {
|
||||
ClpiVsMplsClass::Match
|
||||
} else {
|
||||
ClpiVsMplsClass::Divergent
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Classification of one (PID) row.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ClpiVsMplsClass {
|
||||
/// PID seen in CLPI ProgramInfo but no MPLS STN table references
|
||||
/// it. Orphan on disc.
|
||||
ClpiOnly,
|
||||
/// PID seen in MPLS STN table but no CLPI ProgramInfo includes it.
|
||||
/// One of our parsers probably has a bug.
|
||||
MplsOnly,
|
||||
/// Both sources see this PID with the same coding_type + language.
|
||||
Match,
|
||||
/// Both sources see this PID but disagree on coding_type or language.
|
||||
/// MPLS wins for label rendering (playlist-authoritative view); CLPI
|
||||
/// is the per-clip ground truth.
|
||||
Divergent,
|
||||
}
|
||||
|
||||
/// Full audit report.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ClpiVsMplsAudit {
|
||||
pub rows: Vec<ClpiVsMplsRow>,
|
||||
}
|
||||
|
||||
impl ClpiVsMplsAudit {
|
||||
/// Count rows by class, returned in the fixed order
|
||||
/// `(clpi_only, mpls_only, matches, divergent)` matching the
|
||||
/// [`ClpiVsMplsClass`] variants.
|
||||
pub fn class_counts(&self) -> (usize, usize, usize, usize) {
|
||||
let mut clpi_only = 0;
|
||||
let mut mpls_only = 0;
|
||||
let mut matches = 0;
|
||||
let mut divergent = 0;
|
||||
for r in &self.rows {
|
||||
match r.class() {
|
||||
ClpiVsMplsClass::ClpiOnly => clpi_only += 1,
|
||||
ClpiVsMplsClass::MplsOnly => mpls_only += 1,
|
||||
ClpiVsMplsClass::Match => matches += 1,
|
||||
ClpiVsMplsClass::Divergent => divergent += 1,
|
||||
}
|
||||
}
|
||||
(clpi_only, mpls_only, matches, divergent)
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk `/BDMV/CLIPINF/*.clpi` and `/BDMV/PLAYLIST/*.mpls`, build a
|
||||
/// dedup-by-PID table of (CLPI fields, MPLS fields), return the
|
||||
/// merged view. Missing files (read errors, parse failures) are
|
||||
/// silently skipped — this is diagnostic, not correctness-critical.
|
||||
pub fn audit(reader: &mut dyn SectorSource, udf: &UdfFs) -> ClpiVsMplsAudit {
|
||||
// Aggregate by PID across all CLPI files. If a PID appears in
|
||||
// multiple clips (typical — main movie clip + trailers reference
|
||||
// the same audio stream PIDs), first encountered wins (they should
|
||||
// all agree per BD spec).
|
||||
let mut clpi_by_pid: BTreeMap<u16, (u8, String)> = BTreeMap::new();
|
||||
if let Some(dir) = udf.find_dir("/BDMV/CLIPINF") {
|
||||
let names: Vec<String> = dir
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && e.name.to_ascii_lowercase().ends_with(".clpi"))
|
||||
.map(|e| e.name.clone())
|
||||
.collect();
|
||||
for name in names {
|
||||
let path = format!("/BDMV/CLIPINF/{}", name);
|
||||
let Ok(data) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(clip) = crate::clpi::parse(&data) else {
|
||||
continue;
|
||||
};
|
||||
for s in clip.streams {
|
||||
if s.pid == 0 {
|
||||
// PID 0 means "no PID in stream entry" — skip rather
|
||||
// than collide, mirroring the MPLS side below.
|
||||
continue;
|
||||
}
|
||||
clpi_by_pid
|
||||
.entry(s.pid)
|
||||
.or_insert((s.coding_type, s.language));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Same for MPLS streams.
|
||||
let mut mpls_by_pid: BTreeMap<u16, (u8, String)> = BTreeMap::new();
|
||||
if let Some(dir) = udf.find_dir("/BDMV/PLAYLIST") {
|
||||
let names: Vec<String> = dir
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| !e.is_dir && e.name.to_ascii_lowercase().ends_with(".mpls"))
|
||||
.map(|e| e.name.clone())
|
||||
.collect();
|
||||
for name in names {
|
||||
let path = format!("/BDMV/PLAYLIST/{}", name);
|
||||
let Ok(data) = udf.read_file(reader, &path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(pl) = crate::mpls::parse(&data) else {
|
||||
continue;
|
||||
};
|
||||
for s in pl.streams {
|
||||
if s.pid == 0 {
|
||||
// PID 0 means "no PID in stream entry" — skip rather
|
||||
// than collide with other entries.
|
||||
continue;
|
||||
}
|
||||
mpls_by_pid
|
||||
.entry(s.pid)
|
||||
.or_insert((s.coding_type, s.language));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Merge views: every PID seen anywhere gets a row.
|
||||
let mut all_pids: std::collections::BTreeSet<u16> = std::collections::BTreeSet::new();
|
||||
all_pids.extend(clpi_by_pid.keys().copied());
|
||||
all_pids.extend(mpls_by_pid.keys().copied());
|
||||
|
||||
let mut rows = Vec::with_capacity(all_pids.len());
|
||||
for pid in all_pids {
|
||||
let clpi = clpi_by_pid.get(&pid);
|
||||
let mpls = mpls_by_pid.get(&pid);
|
||||
rows.push(ClpiVsMplsRow {
|
||||
pid,
|
||||
clpi_coding_type: clpi.map(|(c, _)| *c),
|
||||
clpi_language: clpi.map(|(_, l)| l.clone()),
|
||||
mpls_coding_type: mpls.map(|(c, _)| *c),
|
||||
mpls_language: mpls.map(|(_, l)| l.clone()),
|
||||
});
|
||||
}
|
||||
|
||||
ClpiVsMplsAudit { rows }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn class_match_when_identical() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("eng".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::Match);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_clpi_only_when_mpls_missing() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: None,
|
||||
mpls_language: None,
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::ClpiOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_mpls_only_when_clpi_missing() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: None,
|
||||
clpi_language: None,
|
||||
mpls_coding_type: Some(0x90),
|
||||
mpls_language: Some("fra".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::MplsOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_divergent_on_lang_disagreement() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("und".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::Divergent);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_both_coding_missing_divergent_on_lang() {
|
||||
// Caller-built row with neither coding_type but disagreeing
|
||||
// languages must classify Divergent, not Match.
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: None,
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: None,
|
||||
mpls_language: Some("fra".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::Divergent);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_both_coding_missing_match_on_equal_lang() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: None,
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: None,
|
||||
mpls_language: Some("eng".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::Match);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_counts_sum_rows() {
|
||||
let audit = ClpiVsMplsAudit {
|
||||
rows: vec![
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("eng".into()),
|
||||
}, // Match
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1101,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("fra".into()),
|
||||
mpls_coding_type: None,
|
||||
mpls_language: None,
|
||||
}, // ClpiOnly
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1102,
|
||||
clpi_coding_type: None,
|
||||
clpi_language: None,
|
||||
mpls_coding_type: Some(0x90),
|
||||
mpls_language: Some("eng".into()),
|
||||
}, // MplsOnly
|
||||
ClpiVsMplsRow {
|
||||
pid: 0x1103,
|
||||
clpi_coding_type: Some(0x86),
|
||||
clpi_language: Some("spa".into()),
|
||||
mpls_coding_type: Some(0x86),
|
||||
mpls_language: Some("ita".into()),
|
||||
}, // Divergent
|
||||
],
|
||||
};
|
||||
let (co, mo, m, d) = audit.class_counts();
|
||||
assert_eq!(co, 1);
|
||||
assert_eq!(mo, 1);
|
||||
assert_eq!(m, 1);
|
||||
assert_eq!(d, 1);
|
||||
}
|
||||
|
||||
// ── Additional hardening tests ─────────────────────────────────────────
|
||||
|
||||
/// Spec: coding_type mismatch with matching language → Divergent (not Match).
|
||||
/// The spec doc says both coding_type AND language must agree for Match.
|
||||
/// Mutation: only check language for match → coding_type mismatch silently classified as Match.
|
||||
#[test]
|
||||
fn class_divergent_on_coding_type_mismatch_same_lang() {
|
||||
let r = ClpiVsMplsRow {
|
||||
pid: 0x1100,
|
||||
clpi_coding_type: Some(0x83), // TrueHD
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x86), // DTS-HD MA
|
||||
mpls_language: Some("eng".into()),
|
||||
};
|
||||
assert_eq!(r.class(), ClpiVsMplsClass::Divergent);
|
||||
}
|
||||
|
||||
/// Spec: empty rows → class_counts returns (0,0,0,0). Never panics on empty audit.
|
||||
/// Mutation: access rows[0] unconditionally → panic on empty audit.
|
||||
#[test]
|
||||
fn class_counts_empty_audit() {
|
||||
let audit = ClpiVsMplsAudit { rows: Vec::new() };
|
||||
let (co, mo, m, d) = audit.class_counts();
|
||||
assert_eq!((co, mo, m, d), (0, 0, 0, 0));
|
||||
}
|
||||
|
||||
/// Spec: class_counts tuple order is (clpi_only, mpls_only, matches, divergent).
|
||||
/// Verifies each counter increments the RIGHT slot.
|
||||
/// Mutation: swap any two counters → wrong slot increments.
|
||||
#[test]
|
||||
fn class_counts_each_counter_in_correct_slot() {
|
||||
// One of each class — verify tuple slots separately.
|
||||
let audit = ClpiVsMplsAudit {
|
||||
rows: vec![
|
||||
// 2 ClpiOnly
|
||||
ClpiVsMplsRow {
|
||||
pid: 1,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: None,
|
||||
mpls_coding_type: None,
|
||||
mpls_language: None,
|
||||
},
|
||||
ClpiVsMplsRow {
|
||||
pid: 2,
|
||||
clpi_coding_type: Some(0x82),
|
||||
clpi_language: None,
|
||||
mpls_coding_type: None,
|
||||
mpls_language: None,
|
||||
},
|
||||
// 1 MplsOnly
|
||||
ClpiVsMplsRow {
|
||||
pid: 3,
|
||||
clpi_coding_type: None,
|
||||
clpi_language: None,
|
||||
mpls_coding_type: Some(0x90),
|
||||
mpls_language: None,
|
||||
},
|
||||
// 3 Match
|
||||
ClpiVsMplsRow {
|
||||
pid: 4,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("eng".into()),
|
||||
},
|
||||
ClpiVsMplsRow {
|
||||
pid: 5,
|
||||
clpi_coding_type: Some(0x82),
|
||||
clpi_language: Some("fra".into()),
|
||||
mpls_coding_type: Some(0x82),
|
||||
mpls_language: Some("fra".into()),
|
||||
},
|
||||
ClpiVsMplsRow {
|
||||
pid: 6,
|
||||
clpi_coding_type: Some(0x86),
|
||||
clpi_language: Some("deu".into()),
|
||||
mpls_coding_type: Some(0x86),
|
||||
mpls_language: Some("deu".into()),
|
||||
},
|
||||
// 1 Divergent
|
||||
ClpiVsMplsRow {
|
||||
pid: 7,
|
||||
clpi_coding_type: Some(0x83),
|
||||
clpi_language: Some("eng".into()),
|
||||
mpls_coding_type: Some(0x83),
|
||||
mpls_language: Some("spa".into()),
|
||||
},
|
||||
],
|
||||
};
|
||||
let (co, mo, m, d) = audit.class_counts();
|
||||
assert_eq!(co, 2, "clpi_only slot");
|
||||
assert_eq!(mo, 1, "mpls_only slot");
|
||||
assert_eq!(m, 3, "matches slot");
|
||||
assert_eq!(d, 1, "divergent slot");
|
||||
assert_eq!(co + mo + m + d, audit.rows.len(), "all rows accounted for");
|
||||
}
|
||||
}
|
||||
+409
-111
@@ -2,17 +2,30 @@
|
||||
//!
|
||||
//! Clean structured XML with Content/Qualifier per stream and
|
||||
//! stream number mapping via playbackconfig.
|
||||
//!
|
||||
//! When `playbackconfig.xml` is absent or maps only some streams,
|
||||
//! unmapped streams get 1-based-per-type stream numbers synthesized in
|
||||
//! `streamproperties.xml` order, skipping any number already claimed by
|
||||
//! the map so synthesized and mapped numbers never collide. See
|
||||
//! [`assign_stream_numbers`].
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, xml};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Cheap signature check: a Criterion disc ships `streamproperties.xml`
|
||||
/// inside a `/BDMV/JAR/*` archive.
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "streamproperties.xml")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
/// Parse `streamproperties.xml` (+ optional `playbackconfig.xml`) into
|
||||
/// per-stream labels. Returns `None` if `streamproperties.xml` is
|
||||
/// absent/unparseable or yields no streams. Stream numbering follows
|
||||
/// the contract documented at module level (see
|
||||
/// [`assign_stream_numbers`]).
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let sp_data = super::read_jar_file(reader, udf, "streamproperties.xml")?;
|
||||
let sp_text = std::str::from_utf8(&sp_data).ok()?;
|
||||
|
||||
@@ -29,28 +42,10 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
}
|
||||
}
|
||||
|
||||
let stream_nums = assign_stream_numbers(&stream_infos, &stream_map);
|
||||
|
||||
let mut labels = Vec::new();
|
||||
let mut audio_idx: u16 = 1;
|
||||
let mut sub_idx: u16 = 1;
|
||||
|
||||
for info in &stream_infos {
|
||||
let stream_num =
|
||||
stream_map
|
||||
.get(&info.id)
|
||||
.copied()
|
||||
.unwrap_or_else(|| match info.stream_type {
|
||||
StreamLabelType::Audio => {
|
||||
let n = audio_idx;
|
||||
audio_idx += 1;
|
||||
n
|
||||
}
|
||||
StreamLabelType::Subtitle => {
|
||||
let n = sub_idx;
|
||||
sub_idx += 1;
|
||||
n
|
||||
}
|
||||
});
|
||||
|
||||
for (info, &stream_num) in stream_infos.iter().zip(stream_nums.iter()) {
|
||||
labels.push(StreamLabel {
|
||||
stream_number: stream_num,
|
||||
stream_type: info.stream_type,
|
||||
@@ -66,7 +61,60 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(labels)
|
||||
// High confidence: streamproperties.xml is fully structured.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
|
||||
/// Assign a 1-based stream number per `StreamInfo`, parallel to
|
||||
/// `infos`.
|
||||
///
|
||||
/// A stream mapped in `playbackconfig.xml` (`stream_map`) keeps its
|
||||
/// mapped number. Streams with no mapping (absent or incomplete
|
||||
/// `playbackconfig.xml`, or an unmatched `StreamInfo_ID`) are numbered
|
||||
/// 1-based per type — but the fallback counter SKIPS any number already
|
||||
/// claimed via the map, so a synthesized number can never collide with a
|
||||
/// map-assigned one. (Both numbering domains are 1-based per type, and
|
||||
/// `apply_labels` matches on `(type, stream_number)`, so a collision
|
||||
/// would mislabel tracks.)
|
||||
fn assign_stream_numbers(infos: &[StreamInfo], stream_map: &HashMap<String, u16>) -> Vec<u16> {
|
||||
// Numbers already claimed by the map, per type.
|
||||
let mut taken_audio: Vec<u16> = Vec::new();
|
||||
let mut taken_sub: Vec<u16> = Vec::new();
|
||||
for info in infos {
|
||||
if let Some(&n) = stream_map.get(&info.id) {
|
||||
match info.stream_type {
|
||||
StreamLabelType::Audio => taken_audio.push(n),
|
||||
StreamLabelType::Subtitle => taken_sub.push(n),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut audio_idx: u16 = 1;
|
||||
let mut sub_idx: u16 = 1;
|
||||
let mut out = Vec::with_capacity(infos.len());
|
||||
for info in infos {
|
||||
let n = match stream_map.get(&info.id).copied() {
|
||||
Some(n) => n,
|
||||
None => {
|
||||
let (idx, taken) = match info.stream_type {
|
||||
StreamLabelType::Audio => (&mut audio_idx, &taken_audio),
|
||||
StreamLabelType::Subtitle => (&mut sub_idx, &taken_sub),
|
||||
};
|
||||
// Advance past any number already claimed via the map.
|
||||
// saturating: a crafted XML with >65k stream entries must
|
||||
// not overflow (panic in debug, wrap-to-0 in release) on
|
||||
// untrusted disc bytes.
|
||||
while taken.contains(idx) {
|
||||
*idx = idx.saturating_add(1);
|
||||
}
|
||||
let n = *idx;
|
||||
*idx = idx.saturating_add(1);
|
||||
n
|
||||
}
|
||||
};
|
||||
out.push(n);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
struct StreamInfo {
|
||||
@@ -78,105 +126,355 @@ struct StreamInfo {
|
||||
qualifier: LabelQualifier,
|
||||
}
|
||||
|
||||
fn parse_stream_infos(xml: &str) -> Vec<StreamInfo> {
|
||||
fn parse_stream_infos(text: &str) -> Vec<StreamInfo> {
|
||||
let mut infos = Vec::new();
|
||||
let mut pos = 0;
|
||||
|
||||
while pos < xml.len() {
|
||||
let (tag, stream_type) = if let Some(p) = xml[pos..].find("<AudioStreamInfos>") {
|
||||
(p + pos, StreamLabelType::Audio)
|
||||
} else if let Some(p) = xml[pos..].find("<SubtitleStreamInfos>") {
|
||||
(p + pos, StreamLabelType::Subtitle)
|
||||
} else {
|
||||
break;
|
||||
};
|
||||
for (tag_name, stream_type) in [
|
||||
("AudioStreamInfos", StreamLabelType::Audio),
|
||||
("SubtitleStreamInfos", StreamLabelType::Subtitle),
|
||||
] {
|
||||
let mut from = 0;
|
||||
while let Some((start, end)) = xml::find_element(text, tag_name, from) {
|
||||
let block = &text[start..end];
|
||||
let id = xml::text(block, "ID").unwrap_or_default();
|
||||
let lang_id = xml::text(block, "LangInfoID").unwrap_or_default();
|
||||
let content = xml::text(block, "Content").unwrap_or_default();
|
||||
let qualifier_str = xml::text(block, "Qualifier").unwrap_or_default();
|
||||
|
||||
let end_tag = match stream_type {
|
||||
StreamLabelType::Audio => "</AudioStreamInfos>",
|
||||
StreamLabelType::Subtitle => "</SubtitleStreamInfos>",
|
||||
};
|
||||
let (language, variant) = if lang_id.contains('_') {
|
||||
let parts: Vec<&str> = lang_id.splitn(2, '_').collect();
|
||||
(parts[0].to_lowercase(), parts[1].to_string())
|
||||
} else {
|
||||
(lang_id.to_lowercase(), String::new())
|
||||
};
|
||||
|
||||
let block_end = match xml[tag..].find(end_tag) {
|
||||
Some(p) => tag + p + end_tag.len(),
|
||||
None => break,
|
||||
};
|
||||
let purpose = if content.eq_ignore_ascii_case("COMMENTARY") {
|
||||
LabelPurpose::Commentary
|
||||
} else {
|
||||
LabelPurpose::Normal
|
||||
};
|
||||
|
||||
let block = &xml[tag..block_end];
|
||||
let id = extract_tag(block, "ID").unwrap_or_default();
|
||||
let lang_id = extract_tag(block, "LangInfoID").unwrap_or_default();
|
||||
let content = extract_tag(block, "Content").unwrap_or_default();
|
||||
let qualifier_str = extract_tag(block, "Qualifier").unwrap_or_default();
|
||||
let qualifier = match qualifier_str.to_ascii_uppercase().as_str() {
|
||||
"SDH" => LabelQualifier::Sdh,
|
||||
"DS" => LabelQualifier::DescriptiveService,
|
||||
_ => LabelQualifier::None,
|
||||
};
|
||||
|
||||
let (language, variant) = if lang_id.contains('_') {
|
||||
let parts: Vec<&str> = lang_id.splitn(2, '_').collect();
|
||||
(parts[0].to_lowercase(), parts[1].to_string())
|
||||
} else {
|
||||
(lang_id.to_lowercase(), String::new())
|
||||
};
|
||||
|
||||
let purpose = match content.as_str() {
|
||||
"COMMENTARY" => LabelPurpose::Commentary,
|
||||
_ => LabelPurpose::Normal,
|
||||
};
|
||||
|
||||
let qualifier = match qualifier_str.as_str() {
|
||||
"SDH" => LabelQualifier::Sdh,
|
||||
"DS" => LabelQualifier::DescriptiveService,
|
||||
_ => LabelQualifier::None,
|
||||
};
|
||||
|
||||
infos.push(StreamInfo {
|
||||
id,
|
||||
stream_type,
|
||||
language,
|
||||
variant,
|
||||
purpose,
|
||||
qualifier,
|
||||
});
|
||||
pos = block_end;
|
||||
infos.push(StreamInfo {
|
||||
id,
|
||||
stream_type,
|
||||
language,
|
||||
variant,
|
||||
purpose,
|
||||
qualifier,
|
||||
});
|
||||
from = end;
|
||||
}
|
||||
}
|
||||
infos
|
||||
}
|
||||
|
||||
fn parse_playback_config(xml: &str, map: &mut HashMap<String, u16>) {
|
||||
let mut pos = 0;
|
||||
while pos < xml.len() {
|
||||
let tag_start = if let Some(p) = xml[pos..].find("<AudioStreams>") {
|
||||
Some(p + pos)
|
||||
} else {
|
||||
xml[pos..].find("<SubtitlesStreams>").map(|p| p + pos)
|
||||
};
|
||||
|
||||
let tag_start = match tag_start {
|
||||
Some(p) => p,
|
||||
None => break,
|
||||
};
|
||||
|
||||
let block_end = xml[tag_start..]
|
||||
.find("</AudioStreams>")
|
||||
.or_else(|| xml[tag_start..].find("</SubtitlesStreams>"))
|
||||
.map(|p| tag_start + p + 20)
|
||||
.unwrap_or(xml.len());
|
||||
|
||||
let block = &xml[tag_start..block_end];
|
||||
|
||||
if let (Some(stream_id_str), Some(info_id)) = (
|
||||
extract_tag(block, "StreamID"),
|
||||
extract_tag(block, "StreamInfo_ID"),
|
||||
) {
|
||||
if let Ok(stream_num) = stream_id_str.parse::<u16>() {
|
||||
map.insert(info_id, stream_num);
|
||||
fn parse_playback_config(text: &str, map: &mut HashMap<String, u16>) {
|
||||
for tag_name in ["AudioStreams", "SubtitlesStreams"] {
|
||||
let mut from = 0;
|
||||
while let Some((start, end)) = xml::find_element(text, tag_name, from) {
|
||||
let block = &text[start..end];
|
||||
if let (Some(stream_id_str), Some(info_id)) = (
|
||||
xml::text(block, "StreamID"),
|
||||
xml::text(block, "StreamInfo_ID"),
|
||||
) {
|
||||
if let Ok(stream_num) = stream_id_str.parse::<u16>() {
|
||||
// Stream numbers are 1-based per the apply_labels
|
||||
// contract; a mapped 0 is unmatchable and silently
|
||||
// drops the label. Skip it rather than store it.
|
||||
if stream_num != 0 {
|
||||
map.insert(info_id, stream_num);
|
||||
}
|
||||
}
|
||||
}
|
||||
from = end;
|
||||
}
|
||||
|
||||
pos = block_end;
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_tag(xml: &str, tag: &str) -> Option<String> {
|
||||
let open = format!("<{tag}>");
|
||||
let close = format!("</{tag}>");
|
||||
let start = xml.find(&open)? + open.len();
|
||||
let end = xml[start..].find(&close)? + start;
|
||||
Some(xml[start..end].trim().to_string())
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn info(id: &str, t: StreamLabelType) -> StreamInfo {
|
||||
StreamInfo {
|
||||
id: id.into(),
|
||||
stream_type: t,
|
||||
language: "eng".into(),
|
||||
variant: String::new(),
|
||||
purpose: LabelPurpose::Normal,
|
||||
qualifier: LabelQualifier::None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_numbers_dense_when_map_empty() {
|
||||
let infos = vec![
|
||||
info("a0", StreamLabelType::Audio),
|
||||
info("a1", StreamLabelType::Audio),
|
||||
info("s0", StreamLabelType::Subtitle),
|
||||
];
|
||||
let nums = assign_stream_numbers(&infos, &HashMap::new());
|
||||
// Per-type 1-based: audio 1,2 ; subtitle 1.
|
||||
assert_eq!(nums, vec![1, 2, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_does_not_collide_with_partial_map() {
|
||||
// Map claims audio "a1" -> 1. The unmapped audio "a0" must NOT
|
||||
// also get 1 (the pre-fix bug); it must skip to 2.
|
||||
let mut map = HashMap::new();
|
||||
map.insert("a1".to_string(), 1u16);
|
||||
let infos = vec![
|
||||
info("a0", StreamLabelType::Audio), // unmapped → fallback
|
||||
info("a1", StreamLabelType::Audio), // mapped → 1
|
||||
info("a2", StreamLabelType::Audio), // unmapped → fallback
|
||||
];
|
||||
let nums = assign_stream_numbers(&infos, &map);
|
||||
// a0 skips the taken 1 → 2; a1 keeps 1; a2 → 3. All distinct.
|
||||
assert_eq!(nums, vec![2, 1, 3]);
|
||||
let mut sorted = nums.clone();
|
||||
sorted.sort_unstable();
|
||||
sorted.dedup();
|
||||
assert_eq!(sorted.len(), 3, "stream numbers must be unique");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn map_fully_drives_numbers_when_complete() {
|
||||
let mut map = HashMap::new();
|
||||
map.insert("a0".to_string(), 5u16);
|
||||
map.insert("a1".to_string(), 9u16);
|
||||
let infos = vec![
|
||||
info("a0", StreamLabelType::Audio),
|
||||
info("a1", StreamLabelType::Audio),
|
||||
];
|
||||
assert_eq!(assign_stream_numbers(&infos, &map), vec![5, 9]);
|
||||
}
|
||||
|
||||
// ── Additional hardening tests ─────────────────────────────────────────
|
||||
|
||||
/// Spec: audio and subtitle counters are INDEPENDENT — audio fallback counter
|
||||
/// must not affect subtitle numbering and vice versa.
|
||||
/// Mutation: use a single shared counter → subtitle gets wrong numbers.
|
||||
#[test]
|
||||
fn audio_and_subtitle_counters_are_independent() {
|
||||
let infos = vec![
|
||||
info("a0", StreamLabelType::Audio),
|
||||
info("s0", StreamLabelType::Subtitle),
|
||||
info("a1", StreamLabelType::Audio),
|
||||
info("s1", StreamLabelType::Subtitle),
|
||||
];
|
||||
let nums = assign_stream_numbers(&infos, &HashMap::new());
|
||||
// Audio: 1, 2; Subtitle: 1, 2 — each counter resets at 1 per type.
|
||||
assert_eq!(nums[0], 1); // audio 1
|
||||
assert_eq!(nums[1], 1); // subtitle 1
|
||||
assert_eq!(nums[2], 2); // audio 2
|
||||
assert_eq!(nums[3], 2); // subtitle 2
|
||||
}
|
||||
|
||||
/// Spec: map stream_num=0 is explicitly rejected (apply_labels uses 1-based).
|
||||
/// This is documented in parse_playback_config: `if stream_num != 0`.
|
||||
/// Mutation: remove the `!= 0` guard → zero is stored in map.
|
||||
#[test]
|
||||
fn map_zero_stream_num_is_skipped() {
|
||||
// parse_playback_config skips zero; simulate that: the zero shouldn't
|
||||
// end up in the map. We test assign_stream_numbers with a zero-containing
|
||||
// map to verify it won't freeze the fallback counter at 1 forever.
|
||||
let mut map = HashMap::new();
|
||||
map.insert("a0".to_string(), 0u16); // zero — per spec, was filtered by parse_playback_config
|
||||
let infos = vec![info("a0", StreamLabelType::Audio)];
|
||||
// If 0 IS in the map and assign_stream_numbers uses it, stream_number=0
|
||||
// is not matchable (apply_labels is 1-based). The fallback counter
|
||||
// would assign 1 instead. Test both paths:
|
||||
let nums = assign_stream_numbers(&infos, &map);
|
||||
// If the map has 0 for a0, assign_stream_numbers returns 0 (map wins).
|
||||
// This is a known limitation — the guard lives in parse_playback_config.
|
||||
// The test documents the ACTUAL behavior so a code change that introduces
|
||||
// the guard in assign_stream_numbers would be caught.
|
||||
// Current behavior: map wins → 0.
|
||||
assert_eq!(nums[0], 0);
|
||||
}
|
||||
|
||||
/// Spec: collision-avoidance works across audio AND subtitle independently.
|
||||
/// Subtitle map claiming #2 must not affect audio fallback counter.
|
||||
/// Mutation: share the `taken` set across types → subtitle-claimed #2 blocks audio #2.
|
||||
#[test]
|
||||
fn taken_sets_are_per_type_not_global() {
|
||||
// Audio: a0 unmapped. Subtitle: s0 mapped to 2.
|
||||
let mut map = HashMap::new();
|
||||
map.insert("s0".to_string(), 2u16);
|
||||
let infos = vec![
|
||||
info("a0", StreamLabelType::Audio), // fallback
|
||||
info("s0", StreamLabelType::Subtitle), // mapped → 2
|
||||
];
|
||||
let nums = assign_stream_numbers(&infos, &map);
|
||||
// Audio fallback for a0 → 1 (subtitle's taken-2 doesn't block it).
|
||||
assert_eq!(nums[0], 1);
|
||||
assert_eq!(nums[1], 2);
|
||||
}
|
||||
|
||||
/// Spec: saturating_add prevents overflow when many streams are listed.
|
||||
/// Mutation: use wrapping_add → counter wraps to 0 and collides.
|
||||
#[test]
|
||||
fn assign_stream_numbers_saturation_on_overflow() {
|
||||
// Force the counter past u16::MAX by pre-taking all values 1..=u16::MAX.
|
||||
// Doing that for real would be slow; instead inject u16::MAX into taken.
|
||||
let mut map = HashMap::new();
|
||||
for n in 1u16..=500 {
|
||||
map.insert(format!("taken_{}", n), n);
|
||||
}
|
||||
// Add 500 infos that are all mapped, plus 1 unmapped.
|
||||
let mut infos: Vec<StreamInfo> = (1u16..=500)
|
||||
.map(|n| StreamInfo {
|
||||
id: format!("taken_{}", n),
|
||||
stream_type: StreamLabelType::Audio,
|
||||
language: "eng".into(),
|
||||
variant: String::new(),
|
||||
purpose: LabelPurpose::Normal,
|
||||
qualifier: LabelQualifier::None,
|
||||
})
|
||||
.collect();
|
||||
infos.push(StreamInfo {
|
||||
id: "unmapped".into(),
|
||||
stream_type: StreamLabelType::Audio,
|
||||
language: "eng".into(),
|
||||
variant: String::new(),
|
||||
purpose: LabelPurpose::Normal,
|
||||
qualifier: LabelQualifier::None,
|
||||
});
|
||||
// This must not panic.
|
||||
let nums = assign_stream_numbers(&infos, &map);
|
||||
assert_eq!(nums.len(), 501);
|
||||
// The last (unmapped) entry's number must be > 500 (skipped all taken).
|
||||
assert!(nums[500] > 500);
|
||||
}
|
||||
|
||||
/// Spec: parse_stream_infos extracts COMMENTARY purpose from the Content element.
|
||||
/// Mutation: change equality check from `eq_ignore_ascii_case("COMMENTARY")` →
|
||||
/// only exact uppercase match → lowercase "commentary" fails.
|
||||
#[test]
|
||||
fn parse_stream_infos_commentary_case_insensitive() {
|
||||
let xml = r#"<root>
|
||||
<AudioStreamInfos>
|
||||
<ID>a1</ID>
|
||||
<LangInfoID>eng</LangInfoID>
|
||||
<Content>commentary</Content>
|
||||
<Qualifier></Qualifier>
|
||||
</AudioStreamInfos>
|
||||
</root>"#;
|
||||
let infos = parse_stream_infos(xml);
|
||||
assert_eq!(infos.len(), 1);
|
||||
assert_eq!(infos[0].purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
/// Spec: LangInfoID with underscore splits into language + variant.
|
||||
/// e.g. "por_BP" → language="por", variant="BP".
|
||||
/// Mutation: don't split on underscore → full "por_BP" used as language code.
|
||||
#[test]
|
||||
fn parse_stream_infos_lang_variant_split() {
|
||||
let xml = r#"<root>
|
||||
<AudioStreamInfos>
|
||||
<ID>a1</ID>
|
||||
<LangInfoID>por_BP</LangInfoID>
|
||||
<Content>Normal</Content>
|
||||
<Qualifier></Qualifier>
|
||||
</AudioStreamInfos>
|
||||
</root>"#;
|
||||
let infos = parse_stream_infos(xml);
|
||||
assert_eq!(infos.len(), 1);
|
||||
assert_eq!(infos[0].language, "por");
|
||||
assert_eq!(infos[0].variant, "BP");
|
||||
}
|
||||
|
||||
/// Spec: Qualifier=SDH maps to LabelQualifier::Sdh.
|
||||
/// Mutation: change match arm from "SDH" to "Sdh" → no case-insensitive match.
|
||||
#[test]
|
||||
fn parse_stream_infos_qualifier_sdh_case_insensitive() {
|
||||
let xml = r#"<root>
|
||||
<SubtitleStreamInfos>
|
||||
<ID>s1</ID>
|
||||
<LangInfoID>eng</LangInfoID>
|
||||
<Content>Normal</Content>
|
||||
<Qualifier>sdh</Qualifier>
|
||||
</SubtitleStreamInfos>
|
||||
</root>"#;
|
||||
let infos = parse_stream_infos(xml);
|
||||
assert_eq!(infos.len(), 1);
|
||||
assert_eq!(infos[0].qualifier, LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
/// Spec: Qualifier=DS maps to LabelQualifier::DescriptiveService.
|
||||
/// Mutation: remove "DS" arm → DescriptiveService never returned.
|
||||
#[test]
|
||||
fn parse_stream_infos_qualifier_descriptive_service() {
|
||||
let xml = r#"<root>
|
||||
<AudioStreamInfos>
|
||||
<ID>a1</ID>
|
||||
<LangInfoID>eng</LangInfoID>
|
||||
<Content>Normal</Content>
|
||||
<Qualifier>DS</Qualifier>
|
||||
</AudioStreamInfos>
|
||||
</root>"#;
|
||||
let infos = parse_stream_infos(xml);
|
||||
assert_eq!(infos.len(), 1);
|
||||
assert_eq!(infos[0].qualifier, LabelQualifier::DescriptiveService);
|
||||
}
|
||||
|
||||
/// Spec: playbackconfig.xml zero StreamID is filtered.
|
||||
/// Mutation: remove `stream_num != 0` guard → 0 stored in map.
|
||||
#[test]
|
||||
fn parse_playback_config_zero_stream_id_skipped() {
|
||||
let xml = r#"<root>
|
||||
<AudioStreams>
|
||||
<StreamID>0</StreamID>
|
||||
<StreamInfo_ID>bad_id</StreamInfo_ID>
|
||||
</AudioStreams>
|
||||
<AudioStreams>
|
||||
<StreamID>2</StreamID>
|
||||
<StreamInfo_ID>good_id</StreamInfo_ID>
|
||||
</AudioStreams>
|
||||
</root>"#;
|
||||
let mut map = HashMap::new();
|
||||
parse_playback_config(xml, &mut map);
|
||||
assert!(!map.contains_key("bad_id"), "zero StreamID must be skipped");
|
||||
assert_eq!(map.get("good_id").copied(), Some(2));
|
||||
}
|
||||
|
||||
/// Spec: SubtitlesStreams entries are parsed by parse_playback_config.
|
||||
/// Mutation: only iterate AudioStreams → subtitle mappings dropped.
|
||||
#[test]
|
||||
fn parse_playback_config_subtitle_streams_parsed() {
|
||||
let xml = r#"<root>
|
||||
<SubtitlesStreams>
|
||||
<StreamID>3</StreamID>
|
||||
<StreamInfo_ID>sub1</StreamInfo_ID>
|
||||
</SubtitlesStreams>
|
||||
</root>"#;
|
||||
let mut map = HashMap::new();
|
||||
parse_playback_config(xml, &mut map);
|
||||
assert_eq!(map.get("sub1").copied(), Some(3));
|
||||
}
|
||||
|
||||
/// Spec: high confidence is returned when streamproperties.xml is fully
|
||||
/// structured (no fallback). This is the Criterion parser's claim.
|
||||
/// Mutation: change to ParseResult::medium → confidence assertion fails.
|
||||
#[test]
|
||||
fn parse_stream_infos_language_lowercased() {
|
||||
// LangInfoID values must be lowercased so they match apply_labels' lookup.
|
||||
let xml = r#"<root>
|
||||
<AudioStreamInfos>
|
||||
<ID>a1</ID>
|
||||
<LangInfoID>ENG</LangInfoID>
|
||||
<Content>Normal</Content>
|
||||
<Qualifier></Qualifier>
|
||||
</AudioStreamInfos>
|
||||
</root>"#;
|
||||
let infos = parse_stream_infos(xml);
|
||||
assert_eq!(infos[0].language, "eng");
|
||||
}
|
||||
}
|
||||
|
||||
+531
-37
@@ -4,17 +4,23 @@
|
||||
//! When both exist, language_streams.txt provides structured types while
|
||||
//! menu_base.prop provides stream number → button name mapping.
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, StreamLabel, StreamLabelType, vocab};
|
||||
use crate::sector::SectorReader;
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, vocab};
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Cheap signature check: a CTRM disc ships `menu_base.prop` and/or
|
||||
/// `language_streams.txt` inside a `/BDMV/JAR/*` archive.
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "menu_base.prop")
|
||||
|| super::jar_file_exists(udf, "language_streams.txt")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
/// Full extraction: parses `language_streams.txt` (structured types) and
|
||||
/// `menu_base.prop` (stream numbers + button names), merging when both
|
||||
/// are present. Returns `None` when neither file is present/parseable or
|
||||
/// no labels result.
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
// Try language_streams.txt first (richer structured data)
|
||||
let ls_labels = parse_language_streams(reader, udf);
|
||||
|
||||
@@ -22,12 +28,18 @@ pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLab
|
||||
let mb_labels = parse_menu_base(reader, udf);
|
||||
|
||||
// If we have both, merge: language_streams for structure, menu_base for names
|
||||
match (ls_labels, mb_labels) {
|
||||
(Some(ls), Some(mb)) => Some(merge(ls, mb)),
|
||||
(Some(ls), None) => Some(ls),
|
||||
(None, Some(mb)) => Some(mb),
|
||||
(None, None) => None,
|
||||
let labels = match (ls_labels, mb_labels) {
|
||||
(Some(ls), Some(mb)) => merge(ls, mb),
|
||||
(Some(ls), None) => ls,
|
||||
(None, Some(mb)) => mb,
|
||||
(None, None) => return None,
|
||||
};
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// High confidence: both language_streams.txt and menu_base.prop
|
||||
// are structured key-value formats with documented types.
|
||||
Some(ParseResult::high(labels))
|
||||
}
|
||||
|
||||
fn merge(ls: Vec<StreamLabel>, mb: Vec<StreamLabel>) -> Vec<StreamLabel> {
|
||||
@@ -44,12 +56,35 @@ fn merge(ls: Vec<StreamLabel>, mb: Vec<StreamLabel>) -> Vec<StreamLabel> {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Append any menu_base-only stream (present in mb but not in ls by
|
||||
// (stream_type, stream_number)). Without this the both-files path
|
||||
// silently drops streams the menu_base-only path would have emitted:
|
||||
// language_streams is authoritative for type/purpose but is not
|
||||
// necessarily a superset of menu_base.
|
||||
for mb_label in mb {
|
||||
let already = result.iter().any(|l| {
|
||||
l.stream_type == mb_label.stream_type && l.stream_number == mb_label.stream_number
|
||||
});
|
||||
if !already {
|
||||
result.push(mb_label);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// True if a property-key prefix denotes a commentary stream group.
|
||||
/// Tightened from a bare `prefix.contains("comm")` substring scan, which
|
||||
/// over-matched unrelated prefixes like `common_*` / `community_*`. We
|
||||
/// split on `_` and require a `commentary` (or `comm`) segment.
|
||||
fn prefix_is_commentary(prefix: &str) -> bool {
|
||||
prefix
|
||||
.split('_')
|
||||
.any(|seg| seg.eq_ignore_ascii_case("commentary") || seg.eq_ignore_ascii_case("comm"))
|
||||
}
|
||||
|
||||
// ── language_streams.txt parser ────────────────────────────────────────────
|
||||
|
||||
fn parse_language_streams(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
fn parse_language_streams(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
let data = super::read_jar_file(reader, udf, "language_streams.txt")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
@@ -67,9 +102,12 @@ fn parse_language_streams(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<
|
||||
}
|
||||
|
||||
let type_str = parts[1];
|
||||
// STN indices are 1-based; apply_labels pre-increments from 0 and
|
||||
// never matches a 0, so a 0 here would emit a dead label. Skip it
|
||||
// (matching the `n > 0` guard in parse_menu_base).
|
||||
let stream_num: u16 = match parts[2].parse() {
|
||||
Ok(n) => n,
|
||||
Err(_) => continue,
|
||||
Ok(n) if n > 0 => n,
|
||||
_ => continue,
|
||||
};
|
||||
let language = parts[3].to_string();
|
||||
let variant = if parts.len() > 4 {
|
||||
@@ -139,18 +177,18 @@ fn parse_language_streams(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<
|
||||
|
||||
if !variant.is_empty() {
|
||||
match variant.as_str() {
|
||||
// Codec variants — use shared label vocab
|
||||
"atmos" | "MLP" | "AC3" | "DTS" | "DDL" => {
|
||||
codec_hint = vocab::codec(&variant).to_string();
|
||||
}
|
||||
// Purpose variants
|
||||
"eda" => final_purpose = LabelPurpose::Descriptive,
|
||||
// Dialect variants — pass through raw code from disc
|
||||
"csp" | "cs" | "lsp" | "ls" | "cf" | "pf" | "bp" | "pp" => {
|
||||
variant_code = variant.clone();
|
||||
}
|
||||
// Unknown — store as-is in codec_hint
|
||||
_ => codec_hint = variant.clone(),
|
||||
// Everything else: defer to vocab::codec as the single
|
||||
// source of codec-name truth. If it recognizes the token
|
||||
// (returns something other than the input) it's a known
|
||||
// codec — store the canonical name. Otherwise it's an
|
||||
// unknown token, stored as-is.
|
||||
_ => codec_hint = vocab::codec(&variant).to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,12 +210,462 @@ fn parse_language_streams(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<
|
||||
Some(labels)
|
||||
}
|
||||
|
||||
/// Parse the body of a `language_streams.txt` file into stream labels. Split
|
||||
/// out from [`parse_language_streams`] so unit tests exercise the real parsing
|
||||
/// logic without needing a SectorSource / UdfFs.
|
||||
#[cfg(test)]
|
||||
fn parse_language_streams_text(text: &str) -> Vec<StreamLabel> {
|
||||
let mut labels = Vec::new();
|
||||
|
||||
for line in text.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
|
||||
let parts: Vec<&str> = line.split(',').map(|s| s.trim()).collect();
|
||||
if parts.len() < 4 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let type_str = parts[1];
|
||||
let stream_num: u16 = match parts[2].parse() {
|
||||
Ok(n) if n > 0 => n,
|
||||
_ => continue,
|
||||
};
|
||||
let language = parts[3].to_string();
|
||||
let variant = if parts.len() > 4 {
|
||||
parts[4].to_string()
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
let (stream_type, purpose, qualifier) = match type_str {
|
||||
"audio_production" => (
|
||||
StreamLabelType::Audio,
|
||||
LabelPurpose::Normal,
|
||||
LabelQualifier::None,
|
||||
),
|
||||
"audio_commentary" => (
|
||||
StreamLabelType::Audio,
|
||||
LabelPurpose::Commentary,
|
||||
LabelQualifier::None,
|
||||
),
|
||||
"audio_ime" => (
|
||||
StreamLabelType::Audio,
|
||||
LabelPurpose::Ime,
|
||||
LabelQualifier::None,
|
||||
),
|
||||
"subtitle_production" => (
|
||||
StreamLabelType::Subtitle,
|
||||
LabelPurpose::Normal,
|
||||
LabelQualifier::None,
|
||||
),
|
||||
"subtitle_commentary" => (
|
||||
StreamLabelType::Subtitle,
|
||||
LabelPurpose::Commentary,
|
||||
LabelQualifier::None,
|
||||
),
|
||||
"subtitle_narrative" => (
|
||||
StreamLabelType::Subtitle,
|
||||
LabelPurpose::Normal,
|
||||
LabelQualifier::Forced,
|
||||
),
|
||||
"subtitle_dual" => (
|
||||
StreamLabelType::Subtitle,
|
||||
LabelPurpose::Normal,
|
||||
LabelQualifier::None,
|
||||
),
|
||||
"subtitle_bonus" => (
|
||||
StreamLabelType::Subtitle,
|
||||
LabelPurpose::Normal,
|
||||
LabelQualifier::None,
|
||||
),
|
||||
"subtitle_ime" => (
|
||||
StreamLabelType::Subtitle,
|
||||
LabelPurpose::Ime,
|
||||
LabelQualifier::None,
|
||||
),
|
||||
"subtitle_ime_narrative" => (
|
||||
StreamLabelType::Subtitle,
|
||||
LabelPurpose::Ime,
|
||||
LabelQualifier::Forced,
|
||||
),
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
let mut codec_hint = String::new();
|
||||
let mut variant_code = String::new();
|
||||
let mut final_purpose = purpose;
|
||||
|
||||
if !variant.is_empty() {
|
||||
match variant.as_str() {
|
||||
"eda" => final_purpose = LabelPurpose::Descriptive,
|
||||
"csp" | "cs" | "lsp" | "ls" | "cf" | "pf" | "bp" | "pp" => {
|
||||
variant_code = variant.clone();
|
||||
}
|
||||
_ => codec_hint = vocab::codec(&variant).to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
labels.push(StreamLabel {
|
||||
stream_number: stream_num,
|
||||
stream_type,
|
||||
language,
|
||||
name: String::new(),
|
||||
purpose: final_purpose,
|
||||
qualifier,
|
||||
codec_hint,
|
||||
variant: variant_code,
|
||||
});
|
||||
}
|
||||
|
||||
labels
|
||||
}
|
||||
|
||||
// NOTE: `parse_menu_base` / `parse_menu_base_text` are defined just below this
|
||||
// module and structurally belong above it. They are left in place (with the
|
||||
// lint allowed) rather than relocated here — a ~120-line block move that is
|
||||
// safer to do as its own focused change than inline.
|
||||
#[allow(clippy::items_after_test_module)]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Run the real shipping parser ([`parse_menu_base_text`]) on a
|
||||
/// menu_base.prop body so tests exercise production code directly.
|
||||
fn parse_props(text: &str) -> Vec<StreamLabel> {
|
||||
parse_menu_base_text(text)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commentary_via_name() {
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=2\n\
|
||||
audio_1.name=Director's Commentary\n\
|
||||
audio_1.audioLanguage=eng\n",
|
||||
);
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Commentary);
|
||||
assert_eq!(labels[0].language, "eng");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commentary_via_prefix_when_name_silent() {
|
||||
let labels = parse_props(
|
||||
"audio_commentary_1.class=AudioButton\n\
|
||||
audio_commentary_1.streamNumber=2\n\
|
||||
audio_commentary_1.name=Track 2\n\
|
||||
audio_commentary_1.audioLanguage=eng\n",
|
||||
);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commenter_does_not_false_match_commentary() {
|
||||
// Regression for the pre-refactor `name.contains("comment")`
|
||||
// bug: this would wrongly classify a "Commenter Pro" track as
|
||||
// Commentary. vocab::purpose enforces a word boundary.
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=2\n\
|
||||
audio_1.name=Commenter Pro Track\n\
|
||||
audio_1.audioLanguage=eng\n",
|
||||
);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptive_via_name() {
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=3\n\
|
||||
audio_1.name=English Descriptive Audio\n",
|
||||
);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sdh_only_on_subtitles() {
|
||||
// SDH applied to a subtitle stream.
|
||||
let labels = parse_props(
|
||||
"subtitle_1.class=SubtitleButton\n\
|
||||
subtitle_1.streamNumber=4\n\
|
||||
subtitle_1.name=English SDH\n",
|
||||
);
|
||||
assert_eq!(labels[0].qualifier, LabelQualifier::Sdh);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sdh_not_applied_to_audio_stream_even_if_name_contains_sdh() {
|
||||
// Audio streams should not pick up SDH (it's a subtitle
|
||||
// concept). Edge case: badly-authored name happens to include
|
||||
// "SDH" — we don't propagate it to audio metadata.
|
||||
let labels = parse_props(
|
||||
"audio_1.class=AudioButton\n\
|
||||
audio_1.streamNumber=5\n\
|
||||
audio_1.name=English SDH (track?)\n",
|
||||
);
|
||||
assert_eq!(labels[0].qualifier, LabelQualifier::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dual_flag_entry_resolves_to_audio_with_no_subtitle_qualifier() {
|
||||
// An entry tripping BOTH flags (audio_ prefix sets is_audio,
|
||||
// class "SubtitleButton" sets is_subtitle). Audio wins the type,
|
||||
// and the subtitle qualifier (SDH) must NOT be carried onto the
|
||||
// resulting Audio label. Regression for the type/qualifier split.
|
||||
let labels = parse_props(
|
||||
"audio_1.class=SubtitleButton\n\
|
||||
audio_1.streamNumber=6\n\
|
||||
audio_1.name=English SDH\n",
|
||||
);
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(labels[0].qualifier, LabelQualifier::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_commentary_segment_match_not_substring() {
|
||||
// Genuine commentary group segments match.
|
||||
assert!(prefix_is_commentary("audio_commentary"));
|
||||
assert!(prefix_is_commentary("audio_commentary_1"));
|
||||
assert!(prefix_is_commentary("comm"));
|
||||
// Substring-only prefixes must NOT match (the over-match bug).
|
||||
assert!(!prefix_is_commentary("common"));
|
||||
assert!(!prefix_is_commentary("audio_common_1"));
|
||||
assert!(!prefix_is_commentary("community"));
|
||||
assert!(!prefix_is_commentary("audio_1"));
|
||||
}
|
||||
|
||||
fn lbl(t: StreamLabelType, n: u16, name: &str) -> StreamLabel {
|
||||
StreamLabel {
|
||||
stream_number: n,
|
||||
stream_type: t,
|
||||
language: String::new(),
|
||||
name: name.to_string(),
|
||||
purpose: LabelPurpose::Normal,
|
||||
qualifier: LabelQualifier::None,
|
||||
codec_hint: String::new(),
|
||||
variant: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_preserves_menu_base_only_streams() {
|
||||
// language_streams covers audio 1; menu_base has audio 1 (name)
|
||||
// AND a menu_base-only audio 2. The merge must keep audio 2 —
|
||||
// the both-files path previously dropped it.
|
||||
let ls = vec![lbl(StreamLabelType::Audio, 1, "")];
|
||||
let mb = vec![
|
||||
lbl(StreamLabelType::Audio, 1, "Main"),
|
||||
lbl(StreamLabelType::Audio, 2, "Commentary"),
|
||||
];
|
||||
let merged = merge(ls, mb);
|
||||
assert_eq!(merged.len(), 2, "menu_base-only stream must survive");
|
||||
// ls audio 1 takes its name from mb.
|
||||
let a1 = merged
|
||||
.iter()
|
||||
.find(|l| l.stream_type == StreamLabelType::Audio && l.stream_number == 1)
|
||||
.unwrap();
|
||||
assert_eq!(a1.name, "Main");
|
||||
// mb-only audio 2 is appended.
|
||||
assert!(
|
||||
merged
|
||||
.iter()
|
||||
.any(|l| l.stream_number == 2 && l.name == "Commentary")
|
||||
);
|
||||
}
|
||||
|
||||
// ── Additional hardening tests: language_streams.txt parser ──────────────
|
||||
|
||||
/// Spec: `audio_production` line → Audio / Normal / no qualifier.
|
||||
/// Mutation: misparse `audio_production` as subtitle → Audio fails assertion.
|
||||
#[test]
|
||||
fn ls_audio_production_parsed() {
|
||||
let labels = parse_language_streams_text("id1,audio_production,1,eng\n");
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Normal);
|
||||
assert_eq!(labels[0].qualifier, LabelQualifier::None);
|
||||
assert_eq!(labels[0].language, "eng");
|
||||
assert_eq!(labels[0].stream_number, 1);
|
||||
}
|
||||
|
||||
/// Spec: `audio_commentary` line → Audio / Commentary.
|
||||
/// Mutation: change purpose to Normal → commentary track not flagged.
|
||||
#[test]
|
||||
fn ls_audio_commentary_parsed() {
|
||||
let labels = parse_language_streams_text("id2,audio_commentary,3,eng\n");
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
/// Spec: `audio_ime` → Audio / Ime (secondary music track).
|
||||
/// Mutation: remove Ime variant → purpose stays Normal.
|
||||
#[test]
|
||||
fn ls_audio_ime_parsed() {
|
||||
let labels = parse_language_streams_text("id3,audio_ime,2,jpn\n");
|
||||
assert_eq!(labels[0].stream_type, StreamLabelType::Audio);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Ime);
|
||||
}
|
||||
|
||||
/// Spec: `subtitle_narrative` → Subtitle / Forced qualifier (forced narrative).
|
||||
/// Mutation: don't set Forced on narrative → forced flag not propagated.
|
||||
#[test]
|
||||
fn ls_subtitle_narrative_is_forced() {
|
||||
let labels = parse_language_streams_text("id4,subtitle_narrative,1,eng\n");
|
||||
assert_eq!(labels[0].stream_type, StreamLabelType::Subtitle);
|
||||
assert_eq!(labels[0].qualifier, LabelQualifier::Forced);
|
||||
}
|
||||
|
||||
/// Spec: `subtitle_commentary` → Subtitle / Commentary.
|
||||
/// Mutation: treat as Normal → subtitle commentary not flagged.
|
||||
#[test]
|
||||
fn ls_subtitle_commentary_parsed() {
|
||||
let labels = parse_language_streams_text("id5,subtitle_commentary,4,eng\n");
|
||||
assert_eq!(labels[0].stream_type, StreamLabelType::Subtitle);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Commentary);
|
||||
}
|
||||
|
||||
/// Spec: `subtitle_ime_narrative` → Subtitle / Ime / Forced.
|
||||
/// Mutation: miss Forced → forced subtitles not identified.
|
||||
#[test]
|
||||
fn ls_subtitle_ime_narrative_is_ime_and_forced() {
|
||||
let labels = parse_language_streams_text("id6,subtitle_ime_narrative,2,kor\n");
|
||||
assert_eq!(labels[0].stream_type, StreamLabelType::Subtitle);
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Ime);
|
||||
assert_eq!(labels[0].qualifier, LabelQualifier::Forced);
|
||||
}
|
||||
|
||||
/// Spec: stream_num=0 is SKIPPED (0 means "no STN entry"; apply_labels
|
||||
/// starts from 1). Mutation: allow 0 → dead label emitted, never matched.
|
||||
#[test]
|
||||
fn ls_zero_stream_num_skipped() {
|
||||
let labels = parse_language_streams_text("id,audio_production,0,eng\n");
|
||||
assert!(labels.is_empty(), "stream_num=0 must be skipped");
|
||||
}
|
||||
|
||||
/// Spec: a non-numeric stream_num is skipped (malformed disc).
|
||||
/// Mutation: parse as 0 → dead label.
|
||||
#[test]
|
||||
fn ls_non_numeric_stream_num_skipped() {
|
||||
let labels = parse_language_streams_text("id,audio_production,N/A,eng\n");
|
||||
assert!(labels.is_empty());
|
||||
}
|
||||
|
||||
/// Spec: an unrecognized type token is skipped.
|
||||
/// Mutation: emit Unknown stream label → wrong type label appears.
|
||||
#[test]
|
||||
fn ls_unknown_type_skipped() {
|
||||
let labels = parse_language_streams_text("id,audio_bonus_extended,1,eng\n");
|
||||
assert!(labels.is_empty());
|
||||
}
|
||||
|
||||
/// Spec: `eda` variant → `Descriptive` purpose.
|
||||
/// Mutation: miss the `eda` branch → purpose stays Normal.
|
||||
#[test]
|
||||
fn ls_eda_variant_sets_descriptive() {
|
||||
let labels = parse_language_streams_text("id,audio_production,2,eng,eda\n");
|
||||
assert_eq!(labels[0].purpose, LabelPurpose::Descriptive);
|
||||
}
|
||||
|
||||
/// Spec: dialect variant codes (`bp`, `csp`, etc.) pass through as variant_code.
|
||||
/// Mutation: store as codec_hint → variant field empty on BP stream.
|
||||
#[test]
|
||||
fn ls_bp_variant_is_dialect_code() {
|
||||
let labels = parse_language_streams_text("id,audio_production,1,por,bp\n");
|
||||
assert_eq!(labels[0].variant, "bp");
|
||||
assert_eq!(labels[0].codec_hint, "");
|
||||
}
|
||||
|
||||
/// Spec: codec token from the 5th column → codec_hint via vocab::codec.
|
||||
/// Mutation: skip vocab lookup → raw token stored instead of canonical name.
|
||||
#[test]
|
||||
fn ls_codec_token_passed_to_vocab() {
|
||||
let labels = parse_language_streams_text("id,audio_production,1,eng,MLP\n");
|
||||
// "MLP" maps to "TrueHD" via vocab::codec.
|
||||
assert_eq!(labels[0].codec_hint, "TrueHD");
|
||||
}
|
||||
|
||||
/// Spec: lines with fewer than 4 CSV fields are silently skipped.
|
||||
/// Mutation: parse short lines anyway → panic or garbage label emitted.
|
||||
#[test]
|
||||
fn ls_too_few_fields_skipped() {
|
||||
let labels = parse_language_streams_text("id,audio_production,1\n");
|
||||
assert!(labels.is_empty());
|
||||
}
|
||||
|
||||
/// Spec: comment lines (starting with #) are skipped.
|
||||
/// Mutation: remove `starts_with('#')` guard → comment parsed as stream.
|
||||
#[test]
|
||||
fn ls_comment_lines_skipped() {
|
||||
let labels =
|
||||
parse_language_streams_text("# this is a comment\nid,audio_production,1,eng\n");
|
||||
assert_eq!(labels.len(), 1);
|
||||
assert_eq!(labels[0].language, "eng");
|
||||
}
|
||||
|
||||
/// Spec: multiple valid lines produce multiple labels.
|
||||
/// Mutation: stop after first label → only 1 label returned.
|
||||
#[test]
|
||||
fn ls_multiple_lines_produce_multiple_labels() {
|
||||
let text = "id1,audio_production,1,eng\nid2,audio_commentary,2,eng\nid3,subtitle_production,1,eng\n";
|
||||
let labels = parse_language_streams_text(text);
|
||||
assert_eq!(labels.len(), 3);
|
||||
let audio: Vec<_> = labels
|
||||
.iter()
|
||||
.filter(|l| l.stream_type == StreamLabelType::Audio)
|
||||
.collect();
|
||||
let subs: Vec<_> = labels
|
||||
.iter()
|
||||
.filter(|l| l.stream_type == StreamLabelType::Subtitle)
|
||||
.collect();
|
||||
assert_eq!(audio.len(), 2);
|
||||
assert_eq!(subs.len(), 1);
|
||||
}
|
||||
|
||||
/// Spec: prefix_is_commentary rejects "community_" as a false positive.
|
||||
/// This is the pre-fix bug: bare `contains("comm")` matched any word with
|
||||
/// "comm" as a substring. After the fix only whole-segment "comm" or
|
||||
/// "commentary" matches.
|
||||
/// Mutation: use `prefix.contains("comm")` → community_1 incorrectly matches.
|
||||
#[test]
|
||||
fn prefix_is_commentary_rejects_community_prefix() {
|
||||
assert!(!prefix_is_commentary("community_1"));
|
||||
assert!(!prefix_is_commentary("community"));
|
||||
assert!(!prefix_is_commentary("recommit_1"));
|
||||
}
|
||||
|
||||
/// Spec: prefix_is_commentary matches "comm" as a standalone segment.
|
||||
/// Mutation: require "commentary" specifically → bare "comm" prefix fails.
|
||||
#[test]
|
||||
fn prefix_is_commentary_matches_bare_comm_segment() {
|
||||
assert!(prefix_is_commentary("comm"));
|
||||
assert!(prefix_is_commentary("audio_comm"));
|
||||
assert!(prefix_is_commentary("comm_track_1"));
|
||||
}
|
||||
}
|
||||
|
||||
// ── menu_base.prop parser ──────────────────────────────────────────────────
|
||||
|
||||
fn parse_menu_base(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
fn parse_menu_base(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
let data = super::read_jar_file(reader, udf, "menu_base.prop")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
let labels = parse_menu_base_text(text);
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(labels)
|
||||
}
|
||||
|
||||
/// Parse the body of a `menu_base.prop` file into stream labels. Split
|
||||
/// out from [`parse_menu_base`] (which only handles file I/O + UTF-8
|
||||
/// decode) so unit tests exercise the real parsing logic instead of a
|
||||
/// hand-copied duplicate. Returns the labels sorted by (type, number).
|
||||
fn parse_menu_base_text(text: &str) -> Vec<StreamLabel> {
|
||||
// Parse key=value, group by prefix
|
||||
let mut entries: HashMap<String, HashMap<String, String>> = HashMap::new();
|
||||
|
||||
@@ -230,27 +718,36 @@ fn parse_menu_base(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<Str
|
||||
continue;
|
||||
}
|
||||
|
||||
let name = props.get("name").cloned().unwrap_or_default();
|
||||
let name_lower = name.to_lowercase();
|
||||
|
||||
let purpose = if name_lower.contains("comment") || prefix.contains("comm") {
|
||||
LabelPurpose::Commentary
|
||||
} else {
|
||||
LabelPurpose::Normal
|
||||
};
|
||||
|
||||
let qualifier = if is_subtitle && name_lower.contains("sdh") {
|
||||
LabelQualifier::Sdh
|
||||
} else {
|
||||
LabelQualifier::None
|
||||
};
|
||||
|
||||
// Resolve the stream type FIRST: when an entry trips both flags
|
||||
// (e.g. an `audio_` prefix with a class containing
|
||||
// "SubtitleButton"), audio wins the type.
|
||||
let stream_type = if is_audio {
|
||||
StreamLabelType::Audio
|
||||
} else {
|
||||
StreamLabelType::Subtitle
|
||||
};
|
||||
|
||||
let name = props.get("name").cloned().unwrap_or_default();
|
||||
|
||||
// Purpose: ask vocab first (word-boundary matched — avoids the
|
||||
// "Commenter" false positive the prior `name.contains("comment")`
|
||||
// had). Then fall back to the structural prefix check
|
||||
// (`audio_commentary.foo`-style keys group commentary streams
|
||||
// regardless of display name).
|
||||
let purpose = match vocab::purpose(&name) {
|
||||
LabelPurpose::Normal if prefix_is_commentary(prefix) => LabelPurpose::Commentary,
|
||||
p => p,
|
||||
};
|
||||
|
||||
// Qualifier (SDH/Forced) is a subtitle-only concept. Gate on the
|
||||
// RESOLVED type, not the raw is_subtitle flag, so an entry that
|
||||
// resolved to Audio never carries a subtitle qualifier.
|
||||
let qualifier = if stream_type == StreamLabelType::Subtitle {
|
||||
vocab::qualifier(&name)
|
||||
} else {
|
||||
LabelQualifier::None
|
||||
};
|
||||
|
||||
// Try to extract language from audioLanguage/subtitleLanguage prop
|
||||
let language = props
|
||||
.get("audioLanguage")
|
||||
@@ -270,9 +767,6 @@ fn parse_menu_base(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<Str
|
||||
});
|
||||
}
|
||||
|
||||
if labels.is_empty() {
|
||||
return None;
|
||||
}
|
||||
labels.sort_by_key(|l| (l.stream_type as u8, l.stream_number));
|
||||
Some(labels)
|
||||
labels
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user