From 55b22bfb9d6fec5e88ca5b96334b9d152014d3ec Mon Sep 17 00:00:00 2001 From: MattJackson <1085847+MattJackson@users.noreply.github.com> Date: Fri, 10 Apr 2026 08:19:53 -0700 Subject: [PATCH] Add design docs for API and MKV muxer architecture --- docs/api-design.md | 203 ++++++++++++++++++++++++++++++++++ docs/mkv-muxer-plan.md | 240 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 443 insertions(+) create mode 100644 docs/api-design.md create mode 100644 docs/mkv-muxer-plan.md diff --git a/docs/api-design.md b/docs/api-design.md new file mode 100644 index 0000000..6836ecb --- /dev/null +++ b/docs/api-design.md @@ -0,0 +1,203 @@ +# libfreemkv API Design + +## Principles + +1. Lib provides building blocks. App composes them. +2. No English text in lib. Error codes only. App handles i18n. +3. No display logic in lib. App decides what to show. +4. Streams are the pipeline. Each stage wraps the next. +5. Lib fires events. App listens. + +## Core API + +```rust +// Open drive — explicit steps, app prints between them +let mut session = DriveSession::open(path)?; +session.wait_ready()?; +session.init()?; +session.probe_disc()?; + +// Scan disc +let disc = Disc::scan(&mut session, &ScanOptions::default())?; + +// Browse +disc.titles // Vec +disc.format // BD / UHD / DVD +disc.capacity_gb() + +// Rip with events +disc.rip(&mut session, 0, output, |event| { + match event.kind { + EventKind::BytesRead { bytes, total } => ..., + EventKind::ReadError { sector, error } => ..., + EventKind::Retry { attempt } => ..., + EventKind::SpeedChange { speed_kbs } => ..., + EventKind::Complete { bytes, errors } => ..., + } +})?; + +// Rip without events +disc.rip(&mut session, 0, output, event::ignore)?; +``` + +## Stream Chains + +Each stream wraps the next. Builder pattern, no `.build()`. + +### Raw m2ts +```rust +disc.rip(&mut session, 0, File::create("movie.m2ts")?, event::ignore)?; +``` + +### MKV +```rust +let output = MkvStream::new(File::create("movie.mkv")?) + .title(&disc.titles[0]) + .max_buffer(10 * 1024 * 1024); + +disc.rip(&mut session, 0, output, |e| { ... })?; +``` + +### MKV with progress (CLI) +```rust +let output = ProgressStream::new( + MkvStream::new(File::create("movie.mkv")?) + .title(&disc.titles[0]) + .max_buffer(10 * 1024 * 1024), + total_bytes, + |pct, speed| eprint!("\r {}% {:.1} MB/s", pct, speed), +); + +disc.rip(&mut session, 0, output, |e| { ... })?; +``` + +### Future: transcode +```rust +let output = ProgressStream::new( + TranscodeStream::new( + MkvStream::new(File::create("movie.mkv")?) + .title(&disc.titles[0]) + .max_buffer(50 * 1024 * 1024), + ) + .codec(H265) + .quality(22), + total_bytes, + |pct, speed| eprint!("\r {}% {:.1} MB/s", pct, speed), +); + +disc.rip(&mut session, 0, output, |e| { ... })?; +``` + +## Events + +Lib fires events during operations. App provides a callback. No display, no text. + +```rust +pub struct Event { + pub kind: EventKind, +} + +pub enum EventKind { + BytesRead { bytes: u64, total: u64 }, + ReadError { sector: u64, error: Error }, + Retry { attempt: u32 }, + SpeedChange { speed_kbs: u16 }, + ExtentStart { index: usize, start_sector: u64, sector_count: u64 }, + Complete { bytes: u64, errors: u32 }, +} +``` + +Events report what happened. App decides what to do. GUI shows a dialog. CLI prints a line. Server logs to file. + +## Error Codes + +Lib errors are codes, not messages. Like HTTP status codes. + +```rust +pub enum Error { + // Drive + DriveNotFound, + DriveOpenFailed, + DriveNotReady, + + // Unlock + UnlockFailed, + NoProfile, + + // AACS + AacsNoKeys, + AacsCertVerifyFailed, + AacsAgidAllocFailed, + AacsHandshakeFailed, + AacsVidMacFailed, + + // Disc + DiscReadError { sector: u64 }, + MplsParseError, + ClpiParseError, + UdfFileNotFound { path: String }, + + // Mux + LookaheadOverflow, + MuxWriteError, + + // SCSI + ScsiError { sense: u8 }, +} +``` + +App maps codes to localized strings. Lib never contains display text. + +## Streams the Lib Provides + +| Stream | Purpose | +|--------|---------| +| MkvStream | BD-TS → MKV (demux + mux) | + +CLI provides: +| Stream | Purpose | +|--------|---------| +| ProgressStream | Byte counting + progress callback | +| Future: TranscodeStream | Re-encode video | + +Any `impl Write` works as a stream. Third-party apps create their own. + +## MkvStream Internals + +LookaheadBuffer (default 5MB, configurable): +1. Phase 1: buffer incoming data, scan for codec setup (SPS/PPS) +2. Found it? Write MKV header, flush buffer, switch to streaming +3. Buffer full? Error — app handles it +4. Phase 2: parse TS → frames → MKV clusters, direct to output + +## File Layout + +``` +libfreemkv/src/ +├── lib.rs Public exports +├── error.rs Error codes (no English) +├── event.rs Event types for callbacks +├── drive.rs DriveSession (open, init, read) +├── disc.rs Disc (scan, rip, titles) +├── scsi/ SCSI transport (Linux, macOS) +├── platform/ Drive unlock (MT1959 A/B) +├── aacs/ AACS decryption +├── udf.rs UDF filesystem parser +├── mpls.rs Playlist parser +├── clpi.rs Clip info parser +├── mux/ +│ ├── stream.rs MkvStream (builder pattern, impl Write) +│ ├── lookahead.rs LookaheadBuffer (generic, reusable) +│ ├── ts.rs BD-TS demuxer +│ ├── ebml.rs EBML primitives +│ ├── mkv.rs MKV muxer +│ └── codec/ Frame parsers (H.264, HEVC, VC-1, AC3, DTS, TrueHD, PGS) +└── ... + +freemkv/src/ +├── main.rs CLI entry, command routing +├── rip.rs Rip command (streams + progress) +├── remux.rs Remux command (m2ts → MKV, no drive) +├── info.rs Drive info display +└── disc_info.rs Disc info display +``` diff --git a/docs/mkv-muxer-plan.md b/docs/mkv-muxer-plan.md new file mode 100644 index 0000000..32f555c --- /dev/null +++ b/docs/mkv-muxer-plan.md @@ -0,0 +1,240 @@ +# MKV Native Muxer — Architecture + +## Goal + +Replace raw m2ts output with in-pipeline MKV muxing. +Disc → TS demux → MKV mux → .mkv file. One pass, no temp files. + +## Data Flow + +``` +ContentReader::read_batch() returns &[u8] of raw BD transport stream + ↓ +TsDemuxer::feed(batch) parses 192-byte BD-TS packets, extracts PES + ↓ +PES reassembly per PID builds complete PES packets with PTS/DTS + ↓ +ElementaryStreamParser per track finds frame boundaries, extracts codec headers + ↓ +MkvMuxer::write_frame(track, pts, data) writes EBML clusters + blocks + ↓ +.mkv file on disk +``` + +## Current Integration Point + +```rust +// rip.rs line ~287 +match reader.read_batch() { + Ok(Some(batch)) => { + writer.write_all(batch)?; // ← replace with muxer.feed(batch) + } +} +``` + +Becomes: +```rust +match reader.read_batch() { + Ok(Some(batch)) => { + muxer.feed(batch)?; + } +} +``` + +## Components + +### 1. BD Transport Stream Demuxer (`ts.rs`) + +BD uses 192-byte packets (not standard 188): +``` +[0-3] TP_extra_header: 2-bit copy_permission + 30-bit arrival_time_stamp +[4] Sync byte: 0x47 +[5] TEI + PUSI + priority + PID[12:8] +[6] PID[7:0] +[7] Scrambling + adaptation + continuity_counter +[8..] Adaptation field (if present) + payload +``` + +API: +```rust +pub struct TsDemuxer { + pes_assemblers: HashMap<u16, PesAssembler>, // PID → assembler +} + +impl TsDemuxer { + pub fn new(pids: &[u16]) -> Self; + pub fn feed(&mut self, data: &[u8]) -> Vec<PesPacket>; +} + +pub struct PesPacket { + pub pid: u16, + pub pts: Option<i64>, // 90kHz ticks + pub dts: Option<i64>, // 90kHz ticks + pub data: Vec<u8>, // elementary stream data +} +``` + +### 2. Elementary Stream Parsers (`codec/`) + +Each codec parser finds frame boundaries and extracts initialization data. + +**H.264 (`codec/h264.rs`):** +- Parse NAL units (start code 00 00 01 or 00 00 00 01) +- Extract SPS + PPS for codecPrivate +- Frame boundary = Access Unit Delimiter (NAL type 9) or SPS + +**HEVC (`codec/hevc.rs`):** +- Parse NAL units +- Extract VPS + SPS + PPS for codecPrivate +- Frame boundary = VCL NAL with first_slice_segment_in_pic_flag + +**AC3/EAC3 (`codec/ac3.rs`):** +- Syncword 0x0B77 +- Parse frame size from header +- No codecPrivate needed (or minimal) + +**DTS (`codec/dts.rs`):** +- Syncword 0x7FFE8001 +- Parse frame size +- No codecPrivate needed + +**TrueHD (`codec/truehd.rs`):** +- Major sync: 0xF8726FBA +- Access unit = major sync + minor syncs +- AC3 core embedded in first substream + +**LPCM (`codec/lpcm.rs`):** +- Fixed frame sizes based on sample rate + channels +- Header describes format + +**PGS (`codec/pgs.rs`):** +- Segment types: PCS, WDS, PDS, ODS, END +- Each segment is a complete unit +- No codecPrivate needed + +### 3. MKV/EBML Muxer (`mkv.rs`) + +Matroska uses EBML (Extensible Binary Meta Language). + +**EBML primitives:** +- Variable-length element ID (1-4 bytes) +- Variable-length size (1-8 bytes) +- Data: uint, int, float, string, UTF-8, binary, date + +**MKV structure:** +``` +EBML Header +Segment +├── SeekHead (index of top-level elements) +├── Info (title, duration, muxing app) +├── Tracks (one entry per stream) +│ ├── TrackEntry (video) +│ │ ├── CodecID: "V_MPEG4/ISO/AVC" or "V_MPEGH/ISO/HEVC" +│ │ ├── CodecPrivate: SPS+PPS (H.264) or VPS+SPS+PPS (HEVC) +│ │ └── Video: PixelWidth, PixelHeight, DisplayWidth, DisplayHeight +│ ├── TrackEntry (audio) +│ │ ├── CodecID: "A_AC3" or "A_TRUEHD" or "A_DTS" +│ │ └── Audio: SamplingFrequency, Channels, BitDepth +│ └── TrackEntry (subtitle) +│ └── CodecID: "S_HDMV/PGS" +├── Chapters (optional, from MPLS chapter marks) +├── Cluster (every ~5 seconds) +│ ├── Timestamp (cluster base time) +│ ├── SimpleBlock (track, relative_ts, data) +│ ├── SimpleBlock ... +│ └── ... +├── Cluster ... +├── Cues (seek index, written at end) +└── Tags (metadata) +``` + +**API:** +```rust +pub struct MkvMuxer<W: Write + Seek> { + writer: W, + tracks: Vec<MkvTrack>, + cluster_start: Option<i64>, + cue_points: Vec<CuePoint>, +} + +impl<W: Write + Seek> MkvMuxer<W> { + pub fn new(writer: W, tracks: &[MkvTrack]) -> Result<Self>; + pub fn write_frame(&mut self, track_idx: usize, pts_ns: i64, keyframe: bool, data: &[u8]) -> Result<()>; + pub fn finish(self) -> Result<()>; // writes Cues + fixes SeekHead +} +``` + +### 4. Pipeline Glue (`mux.rs`) + +Ties everything together: + +```rust +pub struct MuxPipeline<W: Write + Seek> { + demuxer: TsDemuxer, + parsers: HashMap<u16, Box<dyn CodecParser>>, + muxer: MkvMuxer<W>, + pid_to_track: HashMap<u16, usize>, +} + +impl<W: Write + Seek> MuxPipeline<W> { + pub fn new(writer: W, streams: &[Stream]) -> Result<Self>; + pub fn feed(&mut self, ts_data: &[u8]) -> Result<()>; + pub fn finish(self) -> Result<()>; +} +``` + +## File Layout + +``` +libfreemkv/src/ +├── mux/ +│ ├── mod.rs MuxPipeline (glue) +│ ├── ts.rs BD-TS demuxer (192-byte packets) +│ ├── ebml.rs EBML primitives (write variable-length ints) +│ ├── mkv.rs MKV muxer (Segment, Tracks, Clusters) +│ └── codec/ +│ ├── mod.rs CodecParser trait +│ ├── h264.rs H.264 NAL parser +│ ├── hevc.rs HEVC NAL parser +│ ├── ac3.rs AC3/EAC3 frame parser +│ ├── dts.rs DTS frame parser +│ ├── truehd.rs TrueHD/Atmos parser +│ ├── lpcm.rs LPCM frame parser +│ └── pgs.rs PGS subtitle parser +``` + +## MKV Codec IDs + +| Our Codec | MKV CodecID | codecPrivate | +|-----------|------------|--------------| +| H264 | V_MPEG4/ISO/AVC | AVCDecoderConfigurationRecord (SPS+PPS) | +| Hevc | V_MPEGH/ISO/HEVC | HEVCDecoderConfigurationRecord (VPS+SPS+PPS) | +| Vc1 | V_MS/VFW/FOURCC | BITMAPINFOHEADER | +| Mpeg2 | V_MPEG2 | sequence_header | +| Ac3 | A_AC3 | none | +| Ac3Plus | A_EAC3 | none | +| TrueHd | A_TRUEHD | none | +| DtsHdMa | A_DTS | none (core + extension) | +| Dts | A_DTS | none | +| Lpcm | A_PCM/INT/BIG | none | +| Pgs | S_HDMV/PGS | none | + +## Timestamps + +BD uses 90kHz PTS/DTS. MKV uses nanoseconds. +Conversion: `ns = pts * 1_000_000_000 / 90_000` = `pts * 100_000 / 9` + +MKV TimestampScale default = 1,000,000 (1ms precision). +For BD content, 1ms is sufficient. + +## Build Order + +1. `ebml.rs` — EBML write primitives (smallest, no dependencies) +2. `ts.rs` — BD-TS demuxer (parse 192-byte packets, PES assembly) +3. `codec/ac3.rs` — simplest codec parser (fixed syncword) +4. `mkv.rs` — MKV muxer (header, tracks, clusters, blocks) +5. `mux.rs` — pipeline glue +6. Test with AC3-only stream (simplest case) +7. `codec/h264.rs` — video parser (NAL units, SPS/PPS) +8. Full BD rip test (video + audio + subs) +9. Remaining codecs (HEVC, DTS, TrueHD, PGS, LPCM, VC-1)