From 789b699f95929af4a958983e999f1740f7d52c8f Mon Sep 17 00:00:00 2001 From: Matthew Jackson <1085847+MattJackson@users.noreply.github.com> Date: Mon, 29 Jun 2026 09:39:03 -0700 Subject: [PATCH] mux: make B1 concealment decode-clean on every gap shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the three residual holes where a concealed/lost gap could still let a dangling-reference frame reach the muxer (degraded/undecryptable-disc path only; clean rips are byte-identical and untouched). Root cause: the discontinuity signal was reconstructed from the 4-bit continuity counter and applied per-PES, both of which are lossy. Three coordinated changes: 1. CC-INDEPENDENT marker. fill_null_ts_unit now tags its NULL packets with an adaptation-field discontinuity_indicator; the demuxer recognises a 0x1FFF packet carrying it as a concealed gap and forces a discontinuity on every tracked PID (the lost unit's PID is unknowable). This survives a loss that is an exact multiple of 16 packets (CC aliases to in-sequence — hole 3) and a loss at a PID's very start (no prior CC — hole 4); it also drops any open, potentially-truncated partial PES. 2. PUSI ATTRIBUTION. A gap landing on a PES boundary now flags the PES STARTING after it, not the one flushed at the boundary (hole 1) — stamping the pre-gap frame could arm-then-disarm the gate on a keyframe and admit the real post-gap inter frame. 3. PER-FRAME signal. codec::Frame gains `discontinuity`; each parser propagates it onto the first post-gap frame. MPEG-2 buffers whole GOPs asynchronously, so it associates the gap by ES OFFSET (like PTS/source), landing it on the exact post-gap picture mid-GOP (hole 2) — a per-PES flag stamped the previous picture. consume_ts (and the EOF flush drain) gate on frame.discontinuity. Tests: CC-independent marker with in-sequence CC + leading-loss; PUSI attribution flags the post-gap PES; MPEG-2 offset-mark stamps the post-gap picture through GOP reorder, not the previous one. Existing B1 gate + EOF tests still green (2270 lib tests). --- src/aacs/decrypt.rs | 34 +++-- src/mux/codec/ac3.rs | 2 + src/mux/codec/dts.rs | 2 + src/mux/codec/dvdsub.rs | 3 + src/mux/codec/h264.rs | 3 + src/mux/codec/hevc.rs | 3 + src/mux/codec/lpcm.rs | 1 + src/mux/codec/mod.rs | 14 ++ src/mux/codec/mpeg2.rs | 78 +++++++++++ src/mux/codec/pgs.rs | 6 + src/mux/codec/truehd.rs | 1 + src/mux/codec/vc1.rs | 3 + src/mux/pipelined_stream.rs | 56 ++++++-- src/mux/ts.rs | 250 ++++++++++++++++++++++++++++-------- src/sector/decrypting.rs | 22 +++- 15 files changed, 396 insertions(+), 82 deletions(-) diff --git a/src/aacs/decrypt.rs b/src/aacs/decrypt.rs index 25262d7..1116270 100644 --- a/src/aacs/decrypt.rs +++ b/src/aacs/decrypt.rs @@ -217,18 +217,23 @@ pub fn aacs_unit_still_ciphertext(unit: &[u8]) -> bool { /// Zero-filling such a unit is wrong at the TS layer: a run of `0x00` bytes /// carries no `0x47` sync, so the demuxer loses packet framing and can mis-parse /// the *next* unit if a stray `0x47` appears mid-zero. Instead we lay down 32 -/// well-formed BD source packets, each a TS null packet (PID `0x1FFF`): +/// well-formed BD source packets, each a TS null packet (PID `0x1FFF`) carrying +/// an adaptation-field **discontinuity_indicator**: /// /// ```text -/// [4-byte TP_extra_header = 0][47 1F FF 10][184 bytes 0xFF stuffing] +/// [4-byte TP_extra_header = 0][47 1F FF 20 B7 80 + 182 bytes 0xFF stuffing] +/// ^sync ^PID ^AF-only ^af_len=183 ^disc_indicator /// ``` /// /// The demuxer stays byte-synced on the 192-byte stride, and because PID -/// `0x1FFF` matches no elementary stream every null packet is silently dropped — -/// so the *video/audio* PID simply loses these packets. That shows up downstream -/// as a continuity-counter gap on the real PID, which the TS assembler already -/// turns into a dropped partial PES (see `mux::ts`), the foundation B1 builds on. -/// This NEVER emits ciphertext and is lossless framing, not fabricated content. +/// `0x1FFF` matches no elementary stream every null packet is silently dropped. +/// The discontinuity_indicator is the B1 loss SIGNAL: `mux::ts` recognises a +/// `0x1FFF` packet with that bit set as a concealed gap and forces a discontinuity +/// on every tracked PID's next PES (the codec consumer then drops forward to the +/// next keyframe). This is CC-INDEPENDENT — unlike the real PID's continuity +/// counter it survives a loss that is an exact multiple of 16 packets, or a loss +/// at a PID's very start. NEVER emits ciphertext; lossless framing, not fabricated +/// content. pub fn fill_null_ts_unit(unit: &mut [u8]) { const PKT: usize = BD_SOURCE_PACKET_BYTES; // 192 let mut off = 0; @@ -236,14 +241,19 @@ pub fn fill_null_ts_unit(unit: &mut [u8]) { // TP_extra_header (arrival timestamp / copy-control) — zero is fine; the // demuxer never reads it for a PID it does not track. unit[off..off + 4].fill(0); - // 188-byte TS null packet: sync, PID 0x1FFF (no PUSI/TEI), payload-only - // with continuity counter 0. + // 188-byte TS null packet: sync, PID 0x1FFF (no PUSI/TEI). unit[off + 4] = TS_SYNC; // 0x47 unit[off + 5] = 0x1F; // PID high (top 5 bits of 0x1FFF, flags clear) unit[off + 6] = 0xFF; // PID low - unit[off + 7] = 0x10; // adaptation=01 (payload only), CC=0 - // Stuffing: 0xFF is the conventional null-packet payload fill. - unit[off + 8..off + PKT].fill(0xFF); + // adaptation_field_control = 0b10 (AF only, no payload), CC = 0. + unit[off + 7] = 0x20; + // adaptation_field_length = 183: the AF (its flags byte + 182 stuffing) + // fills the rest of the 188-byte packet. + unit[off + 8] = 0xB7; + // AF flags: discontinuity_indicator (0x80) — the concealed-gap signal. + unit[off + 9] = 0x80; + // Stuffing: 0xFF is the conventional adaptation-field fill. + unit[off + 10..off + PKT].fill(0xFF); off += PKT; } } diff --git a/src/mux/codec/ac3.rs b/src/mux/codec/ac3.rs index e2fabc3..5471ba5 100644 --- a/src/mux/codec/ac3.rs +++ b/src/mux/codec/ac3.rs @@ -115,6 +115,7 @@ impl CodecParser for Ac3Parser { let duration_ns = frame_duration_ns(remaining, bsid); frames.push(Frame { + discontinuity: false, coding: None, source: None, pts_ns: frame_pts_ns, @@ -201,6 +202,7 @@ impl CodecParser for Ac3Parser { } let duration_ns = frame_duration_ns(frame, bsid); vec![Frame { + discontinuity: false, coding: None, source: None, pts_ns: self.flush_pts_ns, diff --git a/src/mux/codec/dts.rs b/src/mux/codec/dts.rs index b5fc8e4..0510f02 100644 --- a/src/mux/codec/dts.rs +++ b/src/mux/codec/dts.rs @@ -243,6 +243,7 @@ impl CodecParser for DtsParser { // extensions or the next core. let au_pts = self.front_pts(); frames.push(Frame { + discontinuity: false, coding: None, source: None, pts_ns: au_pts, @@ -299,6 +300,7 @@ impl CodecParser for DtsParser { let au = std::mem::take(&mut self.buf); self.pts_marks.clear(); vec![Frame { + discontinuity: false, coding: None, source: None, pts_ns, diff --git a/src/mux/codec/dvdsub.rs b/src/mux/codec/dvdsub.rs index 47b0320..c96c838 100644 --- a/src/mux/codec/dvdsub.rs +++ b/src/mux/codec/dvdsub.rs @@ -43,6 +43,7 @@ impl DvdSubParser { if force || buf.len() >= *size { let (pts_ns, _, data) = self.pending.take().unwrap(); return Some(Frame { + discontinuity: false, coding: None, source: None, pts_ns, @@ -103,6 +104,7 @@ impl CodecParser for DvdSubParser { let d = ((pes.data[0] as usize) << 8) | pes.data[1] as usize; if d < 2 { out.push(Frame { + discontinuity: false, coding: None, source: None, pts_ns, @@ -116,6 +118,7 @@ impl CodecParser for DvdSubParser { } else { // Too short to carry SPU_size — pass through as a lone frame. out.push(Frame { + discontinuity: false, coding: None, source: None, pts_ns, diff --git a/src/mux/codec/h264.rs b/src/mux/codec/h264.rs index 4757128..32976ea 100644 --- a/src/mux/codec/h264.rs +++ b/src/mux/codec/h264.rs @@ -243,6 +243,9 @@ impl CodecParser for H264Parser { source: pes.source, pts_ns, keyframe, + // One access unit per PES (BD-TS aligns AUs to PES), so the gap + // signal maps straight onto this frame. + discontinuity: pes.discontinuity, data: frame_data, duration_ns: None, }] diff --git a/src/mux/codec/hevc.rs b/src/mux/codec/hevc.rs index dd83527..da59e15 100644 --- a/src/mux/codec/hevc.rs +++ b/src/mux/codec/hevc.rs @@ -661,6 +661,9 @@ impl CodecParser for HevcParser { source: pes.source, pts_ns, keyframe, + // One access unit per PES (BD-TS aligns AUs to PES), so the gap + // signal maps straight onto this frame. + discontinuity: pes.discontinuity, data: frame_data, duration_ns: None, }] diff --git a/src/mux/codec/lpcm.rs b/src/mux/codec/lpcm.rs index 3bdd896..fa53638 100644 --- a/src/mux/codec/lpcm.rs +++ b/src/mux/codec/lpcm.rs @@ -72,6 +72,7 @@ impl CodecParser for LpcmParser { } let pts_ns = pes.pts.map(pts_to_ns).unwrap_or(0); vec![Frame { + discontinuity: false, coding: None, source: None, pts_ns, diff --git a/src/mux/codec/mod.rs b/src/mux/codec/mod.rs index bd37662..54895ea 100644 --- a/src/mux/codec/mod.rs +++ b/src/mux/codec/mod.rs @@ -44,6 +44,16 @@ pub struct Frame { pub pts_ns: i64, /// Whether this is a keyframe (used for cue points). pub keyframe: bool, + /// This frame is the FIRST coded picture after a concealed/lost gap (P3/B1): + /// its data begins after packets the demuxer never received (an undecryptable + /// unit concealed as NULL-TS upstream, or a continuity break in a damaged + /// source). Inter-coded video frames carrying this flag reference data that is + /// gone, so the consumer's `ResyncGate` arms here and drops forward to the next + /// keyframe. Carried per-FRAME (not per-PES) because buffering parsers — MPEG-2 + /// emits whole GOPs, H.264/HEVC lag one access unit — decouple the frame from + /// the PES that carried the gap signal. Default `false`; only ever set on the + /// degraded/conceal path, so a clean rip leaves every frame `false`. + pub discontinuity: bool, /// Frame data (elementary stream bytes). pub data: Vec, /// Optional duration in nanoseconds — only set by parsers that @@ -123,11 +133,15 @@ impl PassthroughParser { impl CodecParser for PassthroughParser { fn parse(&mut self, pes: &PesPacket) -> Vec { let pts_ns = pes.pts.or(pes.dts).map(pts_to_ns).unwrap_or(0); + // Passthrough emits exactly one frame per PES with no cross-PES buffering, + // so the PES's discontinuity maps directly onto this frame. (Buffering + // parsers must instead defer the flag to the next emitted frame.) vec![Frame { coding: None, source: None, pts_ns, keyframe: self.keyframe, + discontinuity: pes.discontinuity, data: pes.data.clone(), duration_ns: None, }] diff --git a/src/mux/codec/mpeg2.rs b/src/mux/codec/mpeg2.rs index 23e8472..ae4f9c1 100644 --- a/src/mux/codec/mpeg2.rs +++ b/src/mux/codec/mpeg2.rs @@ -129,6 +129,15 @@ pub struct Mpeg2Parser { /// each GOP's first PES PTS so video stays in sync with the PES-timestamped /// audio. None until the first PES timestamp is seen. origin_pts_ns: Option, + /// B1: absolute ES offsets at which a concealed/lost-gap PES began, parallel + /// to `pts_marks`/`source_marks` and drained by the SAME mark-drain invariant. + /// MPEG-2 emits whole GOPs asynchronously, so a per-PES flag can't ride + /// through to the right frame (the PES that carries the gap completes the + /// PREVIOUS picture); associating by OFFSET instead stamps `discontinuity` on + /// the access unit whose own bytes begin after the gap — the first post-gap + /// picture — surviving GOP buffering + temporal reorder. The consumer's + /// ResyncGate then arms at that exact picture, mid-GOP if need be. + disc_marks: VecDeque, } /// One coded picture buffered awaiting its GOP's completion (see `gop_buf`). @@ -165,6 +174,7 @@ impl Mpeg2Parser { gop_buf: Vec::new(), emitted_fields: 0, origin_pts_ns: None, + disc_marks: VecDeque::new(), } } @@ -223,6 +233,13 @@ impl Mpeg2Parser { break; } } + while let Some(&off) = self.disc_marks.front() { + if off < cutoff { + self.disc_marks.pop_front(); + } else { + break; + } + } } break; }; @@ -316,6 +333,11 @@ impl Mpeg2Parser { if gop_boundary && !self.gop_buf.is_empty() { self.flush_gop(&mut out); } + // A concealed-gap mark inside this AU's range [start, end_abs) means + // this picture's own bytes begin after the gap — the first post-gap + // AU. Same front-mark invariant as PTS/source. Carries through GOP + // buffering/reorder to the ResyncGate (which arms at this picture). + let discontinuity = self.disc_marks.front().is_some_and(|&off| off < end_abs); self.gop_buf.push(BufferedPicture { tr, info, @@ -323,6 +345,7 @@ impl Mpeg2Parser { frame: Frame { pts_ns: 0, keyframe, + discontinuity, data, duration_ns: None, coding: Some(info), @@ -352,6 +375,13 @@ impl Mpeg2Parser { break; } } + while let Some(&off) = self.disc_marks.front() { + if off < end_abs { + self.disc_marks.pop_front(); + } else { + break; + } + } } // EOF: emit the final (possibly incomplete) GOP so nothing is dropped. if force { @@ -429,6 +459,12 @@ impl CodecParser for Mpeg2Parser { if let Some(src) = pes.source { self.source_marks.push_back((off, src)); } + // A concealed/lost gap on this PES marks the access unit its bytes begin — + // associated by offset (like PTS/source) so it lands on the first post-gap + // picture, not the previous one that completes when this PES arrives. + if pes.discontinuity { + self.disc_marks.push_back(off); + } self.buf.extend_from_slice(&pes.data); self.drain_complete_aus(false) } @@ -997,6 +1033,48 @@ mod tests { assert!(!frames[1].keyframe); } + /// B1 hole-2 regression: MPEG-2 buffers a GOP and emits asynchronously, so a + /// concealed gap must be associated by OFFSET (like PTS), landing on the + /// picture whose own bytes begin after the gap — NOT the previous picture + /// that completes when the discontinuity PES arrives. pic1 (I) is pre-gap; + /// pic2 (P), carried by a `discontinuity` PES, is the first post-gap AU. + #[test] + fn discontinuity_offset_mark_stamps_post_gap_picture_not_previous() { + let mut parser = Mpeg2Parser::new(); + let mut pic1 = make_picture_header(PICTURE_TYPE_I); + pic1.extend_from_slice(&[0x11; 100]); + let mut pic2 = make_picture_header(2); // P + pic2.extend_from_slice(&[0x22; 100]); + + // pic1 on a clean PES; nothing emits (same GOP, buffered). + assert!(parser.parse(&make_pes(pic1.clone(), Some(0))).is_empty()); + // pic2 on a PES flagged discontinuity (a concealed gap preceded it). + // parse() of this PES completes pic1's AU (the PREVIOUS picture) — which + // must stay clean — while pic2 keeps buffering. + let pes2 = PesPacket { + source: None, + pid: 0x1011, + pts: Some(90000), + dts: None, + data: pic2.clone(), + discontinuity: true, + }; + assert!(parser.parse(&pes2).is_empty(), "same GOP — still buffered"); + + let frames = parser.flush(); + assert_eq!(frames.len(), 2); + assert_eq!(frames[0].data, pic1); + assert!( + !frames[0].discontinuity, + "the previous (pre-gap) I picture must NOT be flagged" + ); + assert_eq!(frames[1].data, pic2); + assert!( + frames[1].discontinuity, + "the post-gap P picture (the discontinuity PES's own AU) IS flagged" + ); + } + #[test] fn picture_coding_extension_stays_with_its_picture() { // Regression for `ignoring pic cod ext after 0`: the picture coding diff --git a/src/mux/codec/pgs.rs b/src/mux/codec/pgs.rs index 7779592..f15e156 100644 --- a/src/mux/codec/pgs.rs +++ b/src/mux/codec/pgs.rs @@ -57,6 +57,7 @@ impl PgsParser { let (start_pts, data) = self.pending.take()?; let duration = end_pts_ns.saturating_sub(start_pts).max(0) as u64; Some(Frame { + discontinuity: false, coding: None, source: None, pts_ns: start_pts, @@ -92,6 +93,7 @@ impl CodecParser for PgsParser { .take() .map(|(start_pts, data)| { vec![Frame { + discontinuity: false, coding: None, source: None, pts_ns: start_pts, @@ -119,6 +121,7 @@ impl CodecParser for PgsParser { let frame = match pts { Some(end) => self.emit_pending(end), None => self.pending.take().map(|(start_pts, data)| Frame { + discontinuity: false, coding: None, source: None, pts_ns: start_pts, @@ -142,6 +145,7 @@ impl CodecParser for PgsParser { // Flush any prior pending undurated and skip storing this one. None => { out.extend(self.pending.take().map(|(start_pts, data)| Frame { + discontinuity: false, coding: None, source: None, pts_ns: start_pts, @@ -167,6 +171,7 @@ impl CodecParser for PgsParser { // (A missing PTS falls through to the drop path below: a // bitmap with no timing reference would land at 00:00:00.) out.push(Frame { + discontinuity: false, coding: None, source: None, pts_ns: pts.unwrap_or(0), @@ -195,6 +200,7 @@ impl CodecParser for PgsParser { // the final on-screen subtitle (see the module doc). match self.pending.take() { Some((start_pts, data)) => vec![Frame { + discontinuity: false, coding: None, source: None, pts_ns: start_pts, diff --git a/src/mux/codec/truehd.rs b/src/mux/codec/truehd.rs index d6b8302..53043ed 100644 --- a/src/mux/codec/truehd.rs +++ b/src/mux/codec/truehd.rs @@ -255,6 +255,7 @@ impl CodecParser for TrueHdParser { } frames.push(Frame { + discontinuity: false, coding: None, source: None, pts_ns: self.next_pts_ns, diff --git a/src/mux/codec/vc1.rs b/src/mux/codec/vc1.rs index 334c153..23ac4b4 100644 --- a/src/mux/codec/vc1.rs +++ b/src/mux/codec/vc1.rs @@ -328,6 +328,9 @@ impl CodecParser for Vc1Parser { source: pes.source, pts_ns: ts_ns, keyframe, + // One frame per PES (BD-TS aligns frames to PES), so the gap signal + // maps straight onto this frame. + discontinuity: pes.discontinuity, data: frame_data, duration_ns: None, }] diff --git a/src/mux/pipelined_stream.rs b/src/mux/pipelined_stream.rs index 558e3da..715a7f9 100644 --- a/src/mux/pipelined_stream.rs +++ b/src/mux/pipelined_stream.rs @@ -182,17 +182,21 @@ impl PipelinedPesStream { self.parsers.iter_mut().find(|(pid, _)| *pid == pes.pid) { let is_video = self.is_video.get(track).copied().unwrap_or(false); - let discontinuity = pes.discontinuity; for frame in parser.parse(&pes) { - // B1: after a TS gap on a video track, drop forward to the - // next keyframe so no frame with a dangling reference is - // emitted. Audio/subtitle always admit (independent frames). - // A track with no gate (out-of-range index) emits as-is. + // B1: after a concealed/lost gap, drop forward to the next + // keyframe on a video track so no frame with a dangling + // reference is emitted. The signal is read PER-FRAME + // (`frame.discontinuity`), not per-PES: buffering parsers + // (MPEG-2 GOPs, H.264/HEVC AU lag) stamp the exact post-gap + // picture, so only it arms the gate — not a whole PES of + // frames. Audio/subtitle always admit (independent frames); + // a track with no gate (out-of-range index) emits as-is. let emit = match self.resync.get_mut(track) { Some(gate) => { let was_armed = gate.is_armed(); let dropped = gate.dropped_in_run(); - let admit = gate.admit(is_video, discontinuity, frame.keyframe); + let admit = + gate.admit(is_video, frame.discontinuity, frame.keyframe); if admit && was_armed { tracing::warn!( target: "mux", @@ -299,15 +303,47 @@ impl Stream for PipelinedPesStream { ); } // Drain any access unit a parser buffered past the last - // PES (e.g. DTS-HD's final core+extension unit). + // PES (e.g. DTS-HD's final core+extension unit, or MPEG-2's + // final GOP). These flush frames carry their own per-frame + // `discontinuity` (a post-gap picture buffered at EOF was + // stamped by the parser), so route them through the SAME B1 + // gate the in-stream path uses — otherwise a trailing + // dangling-reference frame (MPEG-2 final-GOP corner) would + // bypass the resync. Disjoint field borrows so the gate + + // is_video reads coexist with the mutable parser drain. let pid_to_track = &self.pid_to_track; let pending = &mut self.pending_frames; + let resync = &mut self.resync; + let is_video = &self.is_video; for (pid, parser) in self.parsers.iter_mut() { let Some(&(_, track)) = pid_to_track.iter().find(|(p, _)| p == pid) else { continue; }; for frame in parser.flush() { - pending.push_back(PesFrame::from_codec_frame(track, frame)); + let emit = match resync.get_mut(track) { + Some(gate) => gate.admit( + is_video.get(track).copied().unwrap_or(false), + frame.discontinuity, + frame.keyframe, + ), + None => true, + }; + if emit { + pending.push_back(PesFrame::from_codec_frame(track, frame)); + } + } + } + // A gate still armed at EOF dropped post-gap frames that never + // reached a keyframe (e.g. a concealed gap in the final GOP). + // Surface it once so the loss is visible, not silent. + for (track, gate) in self.resync.iter().enumerate() { + if gate.is_armed() { + tracing::warn!( + target: "mux", + track, + dropped = gate.dropped_in_run(), + "B1: stream ended while dropping to a keyframe after a concealed gap (no trailing keyframe)" + ); } } return Ok(self.pending_frames.pop_front()); @@ -427,6 +463,7 @@ mod tests { source: None, pts_ns: pes.pts.unwrap_or(0) + i as i64, keyframe: i == 0, + discontinuity: false, data: pes.data.clone(), duration_ns: None, }) @@ -439,6 +476,7 @@ mod tests { source: None, pts_ns: 0, keyframe: false, + discontinuity: false, data: vec![0xEE], duration_ns: None, }) @@ -555,6 +593,8 @@ mod tests { source: None, pts_ns: pes.pts.unwrap_or(0), keyframe: pes.data.first() == Some(&b'K'), + // Propagate so the B1 gate can be driven end-to-end in tests. + discontinuity: pes.discontinuity, data: pes.data.clone(), duration_ns: None, }] diff --git a/src/mux/ts.rs b/src/mux/ts.rs index bd66375..7c3b64c 100644 --- a/src/mux/ts.rs +++ b/src/mux/ts.rs @@ -12,6 +12,10 @@ use crate::consts::TS_PACKET_BYTES; /// TS sync byte. const SYNC_BYTE: u8 = 0x47; +/// MPEG-TS null-packet PID (0x1FFF). Carries no elementary stream; the P3 +/// concealment fill emits null packets on this PID, tagged with an +/// adaptation-field discontinuity_indicator to signal a concealed gap. +const NULL_PID: u16 = 0x1FFF; /// A reassembled PES packet with timestamp info. #[derive(Debug)] @@ -28,14 +32,17 @@ pub struct PesPacket { /// from the producer's known stream offset. `None` when the demuxer was fed /// without a base offset (callers that don't need provenance). pub source: Option, - /// True when a TS continuity gap (a CC discontinuity, or an adaptation-field - /// discontinuity_indicator) was seen on this PID since the previous PES - /// completed — i.e. one or more packets for this stream were lost (e.g. the - /// mux replaced an undecryptable unit with NULL TS packets, P3/A2). This is - /// the FIRST surviving PES after the gap, so for inter-coded video it (and - /// every later frame up to the next IRAP/IDR) may reference data that is now - /// gone. The codec-parse consumer uses it to drop forward to the next - /// keyframe (B1) instead of emitting frames with dangling references. + /// True when one or more packets for this stream were lost before this PES — + /// a continuity break (CC gap or adaptation-field discontinuity_indicator) on + /// a tracked PID, or the CC-independent concealment marker the mux emits when + /// it replaces an undecryptable unit with NULL-TS packets (P3/A2). This PES is + /// the FIRST whose data is entirely after the gap: a mid-frame loss drops the + /// truncated partial and flags the next complete PES; a loss landing on a PES + /// boundary flags the PES STARTING after it (never the one just flushed). So + /// for inter-coded video this PES — and every later frame up to the next + /// IRAP/IDR — may reference data that is now gone. The codec-parse consumer + /// (via the per-frame `Frame::discontinuity` its parser propagates) drops + /// forward to the next keyframe (B1) instead of emitting dangling references. pub discontinuity: bool, } @@ -67,10 +74,11 @@ struct PesAssembler { /// Stamped at PES start, emitted on the completed packet — provenance is /// carried, never reconstructed downstream. pes_source: Option, - /// Sticky "a continuity gap occurred on this PID" flag. Set whenever a CC - /// gap or an explicit discontinuity_indicator is seen; carried onto the NEXT - /// completed PES (which is the first surviving frame after the loss) and then - /// cleared. Drives B1 drop-to-keyframe in the codec consumer. + /// Sticky "a gap occurred on this PID" flag. Set by a CC gap, an explicit + /// discontinuity_indicator, or the concealment marker; consumed by the NEXT + /// PES this assembler completes — the first whose data is entirely post-gap — + /// then cleared. A gap detected on a PUSI sets it AFTER `start()` so it rides + /// the new PES, not the one just flushed. Drives B1 drop-to-keyframe. pending_discontinuity: bool, } @@ -356,6 +364,39 @@ impl TsDemuxer { let pusi = ts[1] & 0x40 != 0; // Payload Unit Start Indicator let adaptation = (ts[3] >> 4) & 0x03; + // P3/B1 CONCEALMENT MARKER. The decrypt layer fills an undecryptable + // aligned unit with NULL-TS packets (PID 0x1FFF) that carry an + // adaptation-field discontinuity_indicator (see `aacs::fill_null_ts_unit`). + // This is the authoritative loss signal — unlike a tracked PID's 4-bit + // continuity_counter it is CC-INDEPENDENT, so it survives a loss that is + // an exact multiple of 16 packets and a loss at the very start of a PID + // (no prior CC to diff against). The decrypt layer cannot know which + // elementary PID(s) the lost unit carried (the data was undecryptable), + // so force a pending discontinuity on EVERY tracked assembler: the next + // completed PES of each resyncs. Harmless for audio/subtitle (the codec + // gate is a no-op there) and at most one extra GOP on a video track that + // did not actually lose packets — bounded, and only on a degraded disc. + if pid == NULL_PID + && (adaptation == 0x02 || adaptation == 0x03) + && (ts[4] as usize) > 0 + && (ts[5] & 0x80) != 0 + { + for a in &mut self.assemblers { + // A concealed unit may have dropped packets belonging to a PES + // currently open on any PID — so that partial is potentially + // TRUNCATED (a hole in the middle of its access unit). Drop it + // like a mid-PES continuity break, and flag pending so the NEXT + // completed PES (the first frame whose data is entirely post-gap) + // resyncs. Mirrors the non-PUSI cc_gap path, applied to every PID + // because the lost unit's PID(s) are unknowable (undecryptable). + a.buffer.clear(); + a.active = false; + a.header_remaining = 0; + a.pending_discontinuity = true; + } + return; + } + let idx = if (pid as usize) < self.pid_index.len() { self.pid_index[pid as usize] } else { @@ -401,8 +442,9 @@ impl TsDemuxer { // — splicing the new payload would corrupt the elementary stream — so // drop the partial and resync on the next PUSI. let cc = ts[3] & 0x0f; - let discontinuity_flag = - (adaptation == 0x03 || adaptation == 0x02) && ts[4] > 0 && (ts[5] & 0x80) != 0; + // adaptation == 0x02 (AF only) already returned above, so only 0x03 + // (AF + payload) can carry an adaptation field here. + let discontinuity_flag = adaptation == 0x03 && ts[4] > 0 && (ts[5] & 0x80) != 0; // A gap is a CC that is neither the expected `(prev + 1) & 0xf` nor a // duplicate `prev` (ISO 13818-1 permits a packet to repeat its CC; a // duplicate is not a loss). Anything else means one or more packets for @@ -412,35 +454,30 @@ impl TsDemuxer { None => false, }; asm.last_cc = Some(cc); - // Any continuity gap — at a PUSI boundary or mid-PES — means packets for - // this stream were lost (an upstream NULL-TS conceal, a damaged source). - // Mark it sticky so the NEXT completed PES carries `discontinuity` and the - // codec consumer can drop forward to the next keyframe (B1). The partial - // PES is still dropped below only for a NON-PUSI continuation (a hole in - // the middle of the current frame); a gap landing exactly on a PUSI starts - // a clean new frame, but it is still the first frame after the loss. - if discontinuity_flag || cc_gap { - asm.pending_discontinuity = true; - } - if !pusi && (discontinuity_flag || cc_gap) && asm.active { - tracing::trace!( - target: "mux", - pid = asm.pid, - "TS continuity break on non-PUSI continuation; dropping partial PES", - ); - asm.buffer.clear(); - asm.active = false; - asm.header_remaining = 0; - return; - } + // A continuity gap means packets for THIS PID were lost (a damaged source, + // or — for the conceal path — a loss that the CC-independent NULL-TS marker + // above did not already flag). The flag is sticky and rides to the FIRST + // post-gap PES so the codec consumer drops forward to the next keyframe + // (B1). Attribution differs by where the gap lands (see below). + let gap = discontinuity_flag || cc_gap; if pusi { // `header_len` is the FULL (uncapped) PES-header length: // 0 = malformed (payload is not a PES start), else 6/9+N. let (pts, dts, header_len) = parse_pes_header(payload); + // Flush the previous PES FIRST — a gap detected on this PUSI packet + // belongs to the PES STARTING now (its data begins after the lost + // packets), NOT the one just completing. So set `pending_discontinuity` + // AFTER start(): it rides the new PES to its own completion. (Setting + // it before would stamp the pre-gap frame; if that frame were a + // keyframe the gate would arm-then-disarm on it and admit the real + // post-gap inter frame with a dangling reference.) if let Some(prev) = asm.start(pts, dts, source) { completed.push(prev); } + if gap { + asm.pending_discontinuity = true; + } if header_len == 0 { // PUSI packet whose payload is not a valid PES start. Do // NOT push it — those bytes are not elementary-stream data @@ -457,16 +494,37 @@ impl TsDemuxer { // the following continuation packet(s). asm.header_remaining = header_len - payload.len(); } - } else if asm.header_remaining > 0 { - // Continuation packet still inside a PES header that spanned - // the boundary — consume header bytes before any ES data. - let skip = asm.header_remaining.min(payload.len()); - asm.header_remaining -= skip; - if skip < payload.len() { - asm.push(&payload[skip..]); - } } else { - asm.push(payload); + // Non-PUSI continuation. + if gap { + // Mid-PES hole: the open partial has a gap, so splicing this + // payload would corrupt the ES. Flag pending (consumed at the + // NEXT completed PES — the first post-gap frame) and drop the + // open partial; resync on the next PUSI. + asm.pending_discontinuity = true; + if asm.active { + tracing::trace!( + target: "mux", + pid = asm.pid, + "TS continuity break on non-PUSI continuation; dropping partial PES", + ); + asm.buffer.clear(); + asm.active = false; + asm.header_remaining = 0; + return; + } + } + if asm.header_remaining > 0 { + // Continuation packet still inside a PES header that spanned + // the boundary — consume header bytes before any ES data. + let skip = asm.header_remaining.min(payload.len()); + asm.header_remaining -= skip; + if skip < payload.len() { + asm.push(&payload[skip..]); + } + } else { + asm.push(payload); + } } } @@ -947,12 +1005,13 @@ mod tests { ); } - /// B1 plumbing: a continuity gap must STAMP `discontinuity = true` on the - /// next completed PES so the codec consumer can drop forward to the next - /// keyframe. A clean in-sequence PES carries `discontinuity = false`. We - /// open A (cc=0), flush it cleanly via B's PUSI (cc=1), then jump the CC - /// (cc 1 -> 5) on C's PUSI: the gap is sticky and lands on the PES flushed - /// at that boundary (B — the frame whose tail packets were the lost ones). + /// B1 plumbing: a continuity gap detected on a PUSI must stamp + /// `discontinuity = true` on the PES STARTING after the gap, NOT the one + /// flushed at the boundary — the post-gap PES is the one whose data begins + /// after the lost packets and references them. (Attribution fix: stamping the + /// pre-gap PES would, if it were a keyframe, arm-then-disarm the gate and let + /// the real post-gap inter frame through with a dangling reference.) A clean + /// in-sequence PES carries `discontinuity = false`. #[test] fn continuity_gap_stamps_discontinuity_on_next_pes() { let pid = 0x1011; @@ -971,23 +1030,100 @@ mod tests { "in-sequence PES is not a discontinuity" ); - // C's PUSI jumps cc 1 -> 5: packets were lost. The gap is sticky and is - // attributed to the PES flushed here (B), which lost its tail packets. + // C's PUSI jumps cc 1 -> 5: packets were lost between B and C. B (flushed + // here) is PRE-gap and stays clean — the gap belongs to C, which starts + // after the lost packets. let out = demux.feed(&ts_payload_packet(pid, true, 5, &pes_start(b"CCCC"))); assert_eq!(out.len(), 1, "B completes"); assert_eq!(&out[0].data[..4], b"BBBB"); assert!( - out[0].discontinuity, - "the PES at the continuity gap must be flagged so B1 can resync" + !out[0].discontinuity, + "the pre-gap PES flushed at the boundary must NOT be flagged" ); - // C itself was opened clean (after the gap) and carries no new gap. + // C carries the discontinuity — it is the first post-gap PES. let out = demux.flush(); assert_eq!(out.len(), 1); assert_eq!(&out[0].data[..4], b"CCCC"); assert!( - !out[0].discontinuity, - "post-gap PES with no further gap is clean" + out[0].discontinuity, + "the post-gap PES must be flagged so B1 resyncs at/after it" + ); + } + + /// One 192-byte BD source packet that is a B1 concealment marker: a PID-0x1FFF + /// null packet carrying the adaptation-field discontinuity_indicator (the byte + /// shape `fill_null_ts_unit` writes for every packet of a concealed unit). + fn null_marker_packet() -> Vec { + let mut pkt = vec![0u8; BD_SOURCE_PACKET_BYTES]; + pkt[4] = SYNC_BYTE; // 0x47 + pkt[5] = 0x1F; // PID 0x1FFF + pkt[6] = 0xFF; + pkt[7] = 0x20; // adaptation-field only + pkt[8] = 0xB7; // af_len 183 + pkt[9] = 0x80; // discontinuity_indicator + for b in &mut pkt[10..] { + *b = 0xFF; + } + pkt + } + + /// HOLE 3 (16-multiple CC blind spot) + the truncated-partial drop. A concealed + /// unit can drop an exact multiple of 16 packets on a PID, leaving its 4-bit + /// continuity_counter looking IN-SEQUENCE — so CC-based detection is blind. The + /// CC-INDEPENDENT marker flags the loss anyway, drops the (potentially + /// truncated) open PES, and stamps the first post-gap PES. + #[test] + fn conceal_marker_forces_discontinuity_with_in_sequence_cc() { + let pid = 0x1011; + let mut demux = TsDemuxer::new(&[pid]); + // A (cc=0) opens; B's PUSI (cc=1) flushes A clean. + demux.feed(&ts_payload_packet(pid, true, 0, &pes_start(b"AAAA"))); + let out = demux.feed(&ts_payload_packet(pid, true, 1, &pes_start(b"BBBB"))); + assert_eq!(&out[0].data[..4], b"AAAA"); + assert!(!out[0].discontinuity); + + // Concealment marker: a unit was dropped. B is open → potentially truncated + // → dropped. CC is NOT consulted. + assert!( + demux.feed(&null_marker_packet()).is_empty(), + "marker emits nothing itself" + ); + + // C (cc=2) is EXACTLY in-sequence after B's cc=1 — as if a multiple of 16 + // packets were lost, so `cc_gap` is false. The marker is the only signal. + let out = demux.feed(&ts_payload_packet(pid, true, 2, &pes_start(b"CCCC"))); + assert!( + out.is_empty(), + "the truncated open PES (B) is dropped, not emitted" + ); + let out = demux.flush(); + assert_eq!(out.len(), 1); + assert_eq!(&out[0].data[..4], b"CCCC"); + assert!( + out[0].discontinuity, + "marker flags the post-gap PES despite in-sequence CC (16-aligned blind spot)" + ); + } + + /// HOLE 4 (leading loss). If the disc's very first unit is undecryptable the + /// first surviving packet has no predecessor CC (`last_cc == None`), so CC + /// detection is blind. The marker still flags the first PES. + #[test] + fn conceal_marker_at_stream_start_flags_first_pes() { + let pid = 0x1011; + let mut demux = TsDemuxer::new(&[pid]); + // Marker FIRST — no prior CC exists for this PID. + assert!(demux.feed(&null_marker_packet()).is_empty()); + // The first real PES of the PID. + let out = demux.feed(&ts_payload_packet(pid, true, 0, &pes_start(b"AAAA"))); + assert!(out.is_empty()); + let out = demux.flush(); + assert_eq!(out.len(), 1); + assert_eq!(&out[0].data[..4], b"AAAA"); + assert!( + out[0].discontinuity, + "leading concealed loss flags the first PES (last_cc == None case)" ); } diff --git a/src/sector/decrypting.rs b/src/sector/decrypting.rs index cf01a4f..58af3cd 100644 --- a/src/sector/decrypting.rs +++ b/src/sector/decrypting.rs @@ -1542,20 +1542,32 @@ mod tests { } /// `fill_null_ts_unit` round-trip: every BD source packet in the unit becomes - /// a well-formed TS null packet, and a TS demuxer tracking a real PID sees - /// none of them (PID 0x1FFF matches nothing) — the basis for A2 concealment. + /// a well-formed TS null packet (PID 0x1FFF, invisible to any real PID) that + /// carries the B1 adaptation-field discontinuity_indicator — the marker + /// `mux::ts` reads as a concealed gap. #[test] fn null_ts_fill_is_well_formed_and_invisible_to_real_pids() { let mut unit = vec![0xAAu8; crate::aacs::ALIGNED_UNIT_LEN]; crate::aacs::fill_null_ts_unit(&mut unit); - // 32 packets, each sync 0x47, PID 0x1FFF, payload-only CC 0. + // 32 packets, each: sync 0x47, PID 0x1FFF, adaptation-only (0b10) with a + // discontinuity_indicator in the adaptation field. let mut off = 0; let mut pkts = 0; while off + 192 <= unit.len() { - assert_eq!(unit[off + 4], 0x47); + assert_eq!(unit[off + 4], 0x47, "sync"); let pid = ((unit[off + 5] as u16 & 0x1F) << 8) | unit[off + 6] as u16; assert_eq!(pid, 0x1FFF, "null PID"); - assert_eq!(unit[off + 7] & 0x30, 0x10, "payload-only"); + assert_eq!( + (unit[off + 7] >> 4) & 0x03, + 0x02, + "adaptation_field_control = AF only (no payload)" + ); + assert!(unit[off + 8] > 0, "adaptation_field_length > 0"); + assert_eq!( + unit[off + 9] & 0x80, + 0x80, + "adaptation-field discontinuity_indicator set (the B1 marker)" + ); off += 192; pkts += 1; }