From 6a17e560fc3ce63bd6ed0ec5f2ab15f1bcb48460 Mon Sep 17 00:00:00 2001 From: Matthew Jackson <1085847+MattJackson@users.noreply.github.com> Date: Fri, 2 Oct 2026 10:37:53 -0700 Subject: [PATCH] h264, hevc, vc1: drop pictures a decoder cannot reconstruct at a start, join or gap resync The MPEG-2 fix left its siblings emitting pictures whose reference is not in the output, and every codec, MPEG-2 included, resumed after a gap on a keyframe whose leading pictures predict from an anchor the resync gate dropped. H.264, HEVC and VC-1 now decide each picture in decode order against the anchors the output holds, as MPEG-2 does: none before the first random-access picture, at a join (the decode clock, else the PTS frontier, moves at the random-access picture), at a VC-1 BROKEN_LINK, and from the keyframe the gate resumes on after a discontinuity (MPEG-2 too); the gate still drops and counts the gap itself. Open-GOP leading B-pictures (H.264 by PTS, VC-1 CLOSED_ENTRY 0 by decode order, interlaced field pairs from FCM/FPTYPE) and HEVC RASL are dropped when that anchor is missing; a CRA at any join, not only a 3 s step back, is written as BLA. Dropped pictures keep their display slots, so every kept frame keeps its PTS, and an MVC dependent unit goes with its base picture. Closed-GOP and IDR-only titles are unchanged. connection_condition 1 is the non-seamless join and 5/6 the seamless ones; the PlayItem and mark_clip_boundary docs had them reversed. --- CHANGELOG.md | 1 + src/mpls.rs | 2 +- src/mux/codec/decodable.rs | 198 ++++++++++++++++++++++++++++ src/mux/codec/h264.rs | 185 ++++++++++++++++++++++---- src/mux/codec/hevc.rs | 263 ++++++++++++++++++++++--------------- src/mux/codec/mod.rs | 3 + src/mux/codec/mpeg2.rs | 10 +- src/mux/codec/vc1.rs | 161 +++++++++++++++++++++-- src/mux/mkvstream.rs | 18 +++ 9 files changed, 699 insertions(+), 142 deletions(-) create mode 100644 src/mux/codec/decodable.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 67c6d77e..b950a0b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ ### Output-format changes - **MPEG-2 video starts on a decodable picture:** pictures before the first I-picture, and the leading B-pictures of an open GOP whose forward reference is not in the output (at the start of a title, after a `broken_link` GOP, or where the timeline jumps at a join), are left out instead of showing as a green or corrupt frame. Every kept frame keeps its PTS, so audio sync is unchanged; a title that opens on a closed GOP is unchanged. +- **H.264, HEVC and VC-1 video start on a decodable picture, and every codec resumes cleanly:** pictures before the first random-access picture are left out, and so are leading pictures that predict from a picture the output does not hold: at the start of a title, at a join (the decode clock or PTS moves at the random-access picture), at a VC-1 `BROKEN_LINK`, and at the keyframe the resync gate resumes on after a gap (MPEG-2 too). These are H.264/VC-1 open-GOP leading B-pictures and HEVC RASL pictures (also after a stream-start CRA, which a conforming decoder discards anyway); an MVC dependent-view unit goes with its base picture, and a CRA at any join is written as BLA (before, only after a PTS step back of over 3 s). Every kept frame keeps its PTS, so audio sync is unchanged; these are not counted as lost data. A closed-GOP or IDR-only title is unchanged. - **MKV DefaultDuration of MPEG-2 video** follows the kept frames' measured period when that is another standard rate: a soft-telecined NTSC film title (DVD) declares 23.976 fps instead of the sequence's 29.97, so the frame count agrees with the duration. Video at its sequence rate is unchanged. - **LPCM** (BD and DVD) is decoded to interleaved big-endian PCM in WAVE_FORMAT_EXTENSIBLE channel order (BD pad channel removed, LFE/surround remapped, DVD 20/24-bit groups unpacked) at the source depth: 16-bit stays 16-bit, 20- and 24-bit sources output 24-bit; MKV `BitDepth` matches, and M2TS/MPG output keeps the depth. BD 20-bit packets are kept (24-bit container); reserved header codes are dropped. On M2TS output LPCM is repacked to the BD layout; LPCM BD cannot carry (e.g. 44.1 kHz) is dropped and reported. - **M2TS:** reordered video (H.264, HEVC, MPEG-2, VC-1) now carries a DTS (H.222.0 2.7.5); PTS round to the nearest 90 kHz tick (a PTS can be 1 tick later than 1.7.7), the first frame lands one second above zero, PCR never decreases with B-frames; output always declares `bd-ts` in its FMKV header. PGS `.sup` ticks use the same rounding. diff --git a/src/mpls.rs b/src/mpls.rs index ea7eb80a..6d395f73 100644 --- a/src/mpls.rs +++ b/src/mpls.rs @@ -55,7 +55,7 @@ pub(crate) struct PlayItem { pub in_time: u32, /// Out-time in 45kHz ticks pub out_time: u32, - /// Connection condition (1=seamless, 5/6=non-seamless). Parsed for + /// Connection condition (1=non-seamless, 5/6=seamless). Parsed for /// completeness; no production reader yet. #[allow(dead_code)] pub connection_condition: u8, diff --git a/src/mux/codec/decodable.rs b/src/mux/codec/decodable.rs new file mode 100644 index 00000000..8c11197a --- /dev/null +++ b/src/mux/codec/decodable.rs @@ -0,0 +1,198 @@ +//! Which pictures of a one-picture-per-PES video stream (H.264, HEVC, VC-1) a decoder can +//! reconstruct from the pictures this output holds. MPEG-2 decides per GOP in its own parser. +//! +//! Pictures are decided in decode order against the anchors the output holds: none at the +//! stream start, at a join (the decode clock or PTS moves at a random-access picture), at a +//! broken link, or after a gap (from the keyframe the `ResyncGate` resumes on; until then +//! the gate drops and counts). A picture that needs a missing anchor is dropped and its +//! discontinuity flag carries to the next emitted frame. Dropped pictures keep their display +//! slots, so every kept frame keeps its PTS. + +use super::Frame; + +/// What a picture needs from the pictures before it in decode order. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Need { + /// Not a picture, or one decodable from its own random-access picture: kept. + Nothing, + /// The frame's keyframe. `closed`: its leading pictures need no earlier picture; + /// `broken`: the anchor before it is not the one its leading pictures were coded against. + Rap { closed: bool, broken: bool }, + /// Intra picture that is not a random-access point: kept, and an anchor. + Intra, + /// Predicted anchor: needs the last anchor, then is one. + Anchor, + /// Leading picture: needs the anchor before its random-access picture unless that is closed. + Leading, + /// Leading when displayed before its random-access picture, else trailing. + ByPts, + /// Trailing non-anchor picture: needs one anchor. + Trailing, +} + +// Most leading pictures assumed where only PTS tells a join (no DTS): no stream bounds how +// many pictures display before a random-access picture and decode after it. +const MAX_LEADING: i64 = 16; + +// Frame period assumed before one is measured (24 fps). +const FALLBACK_PERIOD_NS: i64 = 1_000_000_000 / 24; + +// PTS steps below this are not a frame period (none is under 1/120 s). +const MIN_PERIOD_NS: i64 = 5_000_000; + +/// Per-stream decodability state; see the module docs. +pub(crate) struct Decodable { + /// Anchors the output holds, saturating at 2. + held: u8, + /// Whether the current random-access picture is closed. + rap_closed: bool, + /// PTS (unwrapped ns) of the current random-access picture. + rap_pts: Option, + /// A discontinuity was seen; the held anchors reset at the next keyframe. + gap: bool, + /// A dropped picture's discontinuity flag, carried to the next emitted frame. + carry_discontinuity: bool, + /// Highest PTS seen (unwrapped ns): the display frontier a random-access picture continues. + high: Option, + /// Last PTS seen (unwrapped ns), for the frame period. + last: Option, + /// Smallest PTS step seen between consecutive pictures (ns). + period: Option, + /// Decode time (unwrapped ns: DTS, else PTS) of the last picture. + last_decode: Option, + /// Accumulated 33-bit wrap offset (ns). + wrap: i64, + /// Whether a random-access picture always displays after every earlier picture (HEVC + /// IRAP, VC-1 entry point), so any backward step at one is a join. + strict: bool, +} + +// 2^33 90 kHz ticks in ns: the PTS wrap. +const WRAP_NS: i64 = (1i64 << 33) * 100_000 / 9; + +impl Decodable { + pub(crate) fn new(strict: bool) -> Self { + Self { + held: 0, + rap_closed: false, + rap_pts: None, + gap: false, + carry_discontinuity: false, + high: None, + last: None, + period: None, + last_decode: None, + wrap: 0, + strict, + } + } + + fn unwrap(&self, pts: i64) -> i64 { + let u = pts + self.wrap; + match self.high { + Some(h) if h - u > WRAP_NS / 2 => u + WRAP_NS, + _ => u, + } + } + + /// Whether a random-access picture at `pts` / `dts` (ns) starts a join. With a DTS (it + /// has leading pictures only then), the decode clock moved by other than ~one frame from + /// the last picture; else its PTS does not continue the display frontier within a run of + /// leading pictures, or (strict) steps back. + pub(crate) fn joined(&self, pts: Option, dts: Option) -> bool { + if let (Some(d), Some(last)) = (dts, self.last_decode) { + let p = self.period.unwrap_or(FALLBACK_PERIOD_NS); + // A 2:3 pulldown frame decodes 1.5 periods after the one before it. + let step = self.unwrap(d) - last; + return step < p / 2 || step > p * 7 / 4; + } + let (Some(pts), Some(high)) = (pts, self.high) else { + return false; + }; + // A forward move is judged only once a frame period is measured. + let window = |p: i64| (MAX_LEADING + 1) * p + p / 2; + let step = self.unwrap(pts) - high; + let forward = self.period.is_some_and(|p| step > window(p)); + let back = if self.strict { + step < 0 + } else { + step < -window(self.period.unwrap_or(FALLBACK_PERIOD_NS)) + }; + forward || back + } + + // Move the frontier, decode clock and measured period to `pts` / `dts`; a join re-bases + // the frontier. + fn observe(&mut self, pts: i64, dts: Option, joined: bool) { + let u = self.unwrap(pts); + self.wrap = u - pts; + if let Some(last) = self.last { + let d = (u - last).abs(); + if !joined && d >= MIN_PERIOD_NS && self.period.is_none_or(|p| d < p) { + self.period = Some(d); + } + } + self.last = Some(u); + self.last_decode = Some(dts.map_or(u, |d| self.unwrap(d))); + self.high = Some(match self.high { + Some(h) if !joined => h.max(u), + _ => u, + }); + } + + /// Decide `frame` (decode order); `None` drops it. `pts` / `dts` are its PES timestamps, + /// as carried. + pub(crate) fn admit( + &mut self, + mut frame: Frame, + need: Need, + pts: Option, + dts: Option, + ) -> Option { + let rap = matches!(need, Need::Rap { .. }); + let joined = rap && self.joined(pts, dts); + if let Some(p) = pts { + self.observe(p, dts, joined); + } + if frame.discontinuity { + self.gap = true; + } + if self.gap && frame.keyframe { + self.gap = false; + self.held = 0; + } + let ok = match need { + Need::Nothing => true, + Need::Rap { closed, broken } => { + if joined || broken { + self.held = 0; + } + self.rap_closed = closed; + self.rap_pts = pts.map(|p| self.unwrap(p)); + true + } + Need::Intra => true, + Need::Anchor | Need::Trailing => self.held >= 1, + Need::Leading => self.held >= self.leading_need(), + Need::ByPts => { + let leading = + matches!((pts, self.rap_pts), (Some(p), Some(r)) if self.unwrap(p) < r); + self.held >= if leading { self.leading_need() } else { 1 } + } + }; + if ok && matches!(need, Need::Rap { .. } | Need::Intra | Need::Anchor) { + self.held = (self.held + 1).min(2); + } + if ok { + frame.discontinuity |= std::mem::take(&mut self.carry_discontinuity); + Some(frame) + } else { + self.carry_discontinuity |= frame.discontinuity; + None + } + } + + fn leading_need(&self) -> u8 { + if self.rap_closed { 1 } else { 2 } + } +} diff --git a/src/mux/codec/h264.rs b/src/mux/codec/h264.rs index 59c04eb9..6c93fbc4 100644 --- a/src/mux/codec/h264.rs +++ b/src/mux/codec/h264.rs @@ -6,6 +6,7 @@ //! Each PES packet = one access unit = one frame. use super::coding::{CodingType, PictureInfo}; +use super::decodable::{Decodable, Need}; use super::startcode::{BitReader, find_start_code, skip_start_code}; use super::{CodecParser, Frame, PesPacket, pts_to_ns}; @@ -54,6 +55,11 @@ pub struct H264Parser { /// dependent access unit suitable for a Matroska `BlockAdditional`. The base /// view's avcC/param-set stripping is unchanged (separate parser instance). mvc_passthrough: bool, + /// Which pictures decode from the pictures this output holds (not the MVC dependent + /// view, whose access units follow their base picture's fate). + decodable: Decodable, + /// What each frame held by `reorder` needs, in decode order. + needs: std::collections::VecDeque, } impl Default for H264Parser { @@ -72,6 +78,8 @@ impl H264Parser { cur_pps: None, reorder: None, mvc_passthrough: false, + decodable: Decodable::new(false), + needs: std::collections::VecDeque::new(), } } @@ -80,6 +88,8 @@ impl H264Parser { pub(crate) fn with_ps_reorder(mut self, enabled: bool) -> Self { if enabled { self.reorder = Some(super::reorder::SparsePtsReorder::new()); + // Reconstructed PTS are approximate: only a step past the leading window is a join. + self.decodable = Decodable::new(false); } self } @@ -91,14 +101,40 @@ impl H264Parser { self } - /// Route a finished frame through the PTS reorderer when enabled, else emit - /// it directly (unchanged transport-stream behaviour). - fn finish(&mut self, explicit: Option, frame: Frame) -> Vec { + /// Route a finished frame through the PTS reorderer when enabled, then keep it + /// only if it decodes from what this output holds. + fn finish( + &mut self, + explicit: Option, + dts: Option, + frame: Frame, + need: Need, + ) -> Vec { match self.reorder.as_mut() { - Some(r) => r.push(explicit, frame), - None => vec![frame], + Some(r) => { + self.needs.push_back(need); + let out = r.push(explicit, frame); + self.decide(out) + } + None => self + .decodable + .admit(frame, need, explicit, dts) + .into_iter() + .collect(), } } + + // Decide frames the reorderer released (decode order, display PTS assigned). + fn decide(&mut self, frames: Vec) -> Vec { + frames + .into_iter() + .filter_map(|f| { + let need = self.needs.pop_front().unwrap_or(Need::Nothing); + let pts = Some(f.pts_ns); + self.decodable.admit(f, need, pts, None) + }) + .collect() + } } // Bytes reserved at the front of every assembled access unit so the keyframe @@ -198,6 +234,7 @@ impl CodecParser for H264Parser { // player reorders). Use PTS not DTS: DTS presents B-frames in decode // order (visible judder) and breaks PTS-based seeking; fall back only if absent. let explicit_pts = pes.pts.or(pes.dts).map(pts_to_ns); + let dts = pes.pts.and(pes.dts).map(pts_to_ns); let pts_ns = explicit_pts.unwrap_or(0); // Single pass: detect IDR keyframes, seed/strip param sets, and convert @@ -285,9 +322,20 @@ impl CodecParser for H264Parser { // Open-GOP anchor heuristic (recovery_point SEI §D.2.8 is authoritative, not read): // promote when every slice of the first picture is intra, a following P field // allowed, base view only. Else the resync gate can drop frames to EOF. + let idr = keyframe; if saw_vcl && all_vcl_intra && !mvc { keyframe = true; } + // A B displayed before its random-access picture is a leading picture; an IDR has none. + let need = match coding_type { + _ if mvc || !saw_vcl => Need::Nothing, + _ if keyframe => Need::Rap { + closed: idr, + broken: false, + }, + Some(CodingType::B) => Need::ByPts, + _ => Need::Anchor, + }; // Every keyframe is self-contained: re-assert active SPS/PPS in-band so a // decoder that dropped the set recovers, and a stale avcC re-apply can't @@ -329,12 +377,15 @@ impl CodecParser for H264Parser { data: frame_data, duration_ns: None, }; - self.finish(explicit_pts, frame) + self.finish(explicit_pts, dts, frame, need) } fn flush(&mut self) -> Vec { match self.reorder.as_mut() { - Some(r) => r.flush(), + Some(r) => { + let out = r.flush(); + self.decide(out) + } None => Vec::new(), } } @@ -908,6 +959,14 @@ mod tests { ); } + /// A parser that has emitted an IDR at PTS 0, so an inter picture after it decodes. + fn primed() -> H264Parser { + let mut p = H264Parser::new(); + let idr = p.parse(&make_pes(vec![0x00, 0x00, 0x01, 0x65, 0x88, 0x00], Some(0))); + assert!(idr[0].keyframe); + p + } + // --- Pictures a decoder cannot reconstruct from the stream's own data --- /// One non-IDR picture AU: `hdr` = NAL header byte, `body` = slice-header byte @@ -939,7 +998,6 @@ mod tests { } #[test] - #[ignore = "defect: an open GOP at the stream start emits its leading B-pictures, whose forward reference precedes the title"] fn an_open_gop_at_the_stream_start_drops_its_leading_b_pictures() { let mut p = H264Parser::new(); let f: Vec = open_gop(0).iter().flat_map(|x| p.parse(x)).collect(); @@ -951,7 +1009,6 @@ mod tests { } #[test] - #[ignore = "defect: a join onto an open GOP emits its leading B-pictures, predicted from the previous clip's anchor"] fn a_join_onto_an_open_gop_drops_its_leading_b_pictures() { // Clip 1 ends on a complete open GOP; clip 2's PTS restarts far away, so // its leading B1/B2 reference a picture of the clip the playlist skipped. @@ -968,7 +1025,6 @@ mod tests { } #[test] - #[ignore = "defect: inter pictures before the stream's first random-access picture are emitted"] fn pictures_before_the_first_random_access_picture_are_dropped() { let mut p = H264Parser::new(); let mut pes = vec![pic_au(0x41, 0x98, 0), pic_au(0x01, 0x9C, 1)]; @@ -981,7 +1037,6 @@ mod tests { } #[test] - #[ignore = "defect: after a gap the ResyncGate resumes on an open-GOP I-picture and passes its leading B-pictures, whose forward reference the gate dropped"] fn a_gap_resync_onto_an_open_gop_emits_no_leading_b_pictures() { // IDR, then a P that follows lost data (and the B predicted from it), then // an open GOP continuing the same clock: its B1/B2 reference the dropped P. @@ -1008,6 +1063,88 @@ mod tests { ); } + #[test] + fn a_gap_leaves_its_loss_to_the_gate() { + // The P after the gap and the B predicted from it decode from the stream's own + // anchors, so the parser passes them: the gate drops and counts them as loss. + let mut p = H264Parser::new(); + let mut lost = pic_au(0x41, 0x98, 9); + lost.discontinuity = true; + let pes = [ + make_pes(vec![0x00, 0x00, 0x01, 0x65, 0x88, 0x00], Some(7 * 3750)), + lost, + pic_au(0x01, 0x9C, 8), + ]; + let f: Vec = pes.iter().flat_map(|x| p.parse(x)).collect(); + assert_eq!(frame_nos(&f), [7, 9, 8]); + let mut gate = crate::mux::resync::ResyncGate::new(); + for x in &f { + gate.admit(true, x.discontinuity, x.keyframe); + } + assert_eq!(gate.dropped_total(), 2); + } + + #[test] + fn a_continuous_open_gop_keeps_its_leading_b_pictures() { + // The leading B7/B8 predict from P6, which the output holds. + let mut p = H264Parser::new(); + let mut pes = vec![make_pes(vec![0x00, 0x00, 0x01, 0x65, 0x88, 0x00], Some(0))]; + pes.extend(open_gop(0).into_iter().skip(3)); + pes.extend(open_gop(6)); + let f: Vec = pes.iter().flat_map(|x| p.parse(x)).collect(); + assert_eq!(frame_nos(&f), [0, 6, 4, 5, 9, 7, 8, 12, 10, 11]); + } + + /// `pic_au` carrying a DTS (`dts` frames) when it differs from its PTS. + fn pic_au_dts(hdr: u8, body: u8, frame: i64, dts: i64) -> PesPacket { + let mut p = pic_au(hdr, body, frame); + p.dts = (dts != frame).then_some(dts * 3750); + p + } + + // Decode order IDR P B B, then an open GOP I B B P whose I decodes `skip` frames late. + fn decode_clock_run(skip: i64) -> Vec { + let s = skip; + let pes = [ + pic_au_dts(0x65, 0x88, 1, 0), + pic_au_dts(0x41, 0x98, 4, 1), + pic_au_dts(0x01, 0x9C, 2, 2), + pic_au_dts(0x01, 0x9C, 3, 3), + pic_au_dts(0x61, 0x88, 7 + s, 4 + s), + pic_au_dts(0x01, 0x9C, 5 + s, 5 + s), + pic_au_dts(0x01, 0x9C, 6 + s, 6 + s), + pic_au_dts(0x41, 0x98, 10 + s, 7 + s), + ]; + let mut p = H264Parser::new(); + frame_nos(&pes.iter().flat_map(|x| p.parse(x)).collect::>()) + } + + #[test] + fn the_decode_clock_tells_a_join_of_a_frame_or_two() { + assert_eq!(decode_clock_run(0), [1, 4, 2, 3, 7, 5, 6, 10]); + // Within the PTS-only window, but the decode clock skipped: another clip. + assert_eq!(decode_clock_run(2), [1, 4, 2, 3, 9, 12]); + } + + #[test] + fn a_closed_gop_title_keeps_every_picture() { + let idr = + |frame: i64| make_pes(vec![0x00, 0x00, 0x01, 0x65, 0x88, 0x00], Some(frame * 3750)); + let pes = [ + idr(0), + pic_au(0x41, 0x98, 3), + pic_au(0x01, 0x9C, 1), + pic_au(0x01, 0x9C, 2), + idr(4), + pic_au(0x41, 0x98, 7), + pic_au(0x01, 0x9C, 5), + pic_au(0x01, 0x9C, 6), + ]; + let mut p = H264Parser::new(); + let f: Vec = pes.iter().flat_map(|x| p.parse(x)).collect(); + assert_eq!(frame_nos(&f), [0, 3, 1, 2, 4, 7, 5, 6]); + } + // The escaped slice header must be unescaped at the parse call site: a second slice with // first_mb_in_slice=65535 (00 00 03 escape) is intra only when read unescaped. #[test] @@ -1045,7 +1182,7 @@ mod tests { // Same NAL type 1, but slice_type = 0 (P): ue "1", so with first_mb = 0 // the byte is "1" + "1" padded = 0b1100_0000 = 0xC0. let au = vec![0x00, 0x00, 0x01, 0x61, 0xC0, 0x00]; - let mut parser = H264Parser::new(); + let mut parser = primed(); let frames = parser.parse(&make_pes(au, Some(0))); assert_eq!(frames.len(), 1); assert_eq!( @@ -1071,7 +1208,7 @@ mod tests { 0x00, 0x00, 0x01, 0x61, 0x58, // P slice, first_mb 1 (same picture) 0x00, ]; - let mut parser = H264Parser::new(); + let mut parser = primed(); let frames = parser.parse(&make_pes(au, Some(0))); assert_eq!(frames.len(), 1, "one PES access unit → one frame"); assert_eq!( @@ -1463,7 +1600,7 @@ mod tests { // 0x88='1 0001000'->I (7); 0x98='1 00110..'->P (5); 0x9C='1 00111..'->B (6). let src = crate::pes::SourcePos::at_byte(8192); let parse = |nal_type: u8, body: u8| { - let mut p = H264Parser::new(); + let mut p = primed(); let mut pe = make_pes(h264_nal(nal_type, &[body]), Some(0)); pe.source = Some(src); p.parse(&pe) @@ -1673,7 +1810,7 @@ mod tests { #[test] fn parse_non_idr() { - let mut parser = H264Parser::new(); + let mut parser = primed(); // PES with non-IDR slice (type 1 = 0x61 or 0x41) let mut data = Vec::new(); @@ -1692,7 +1829,7 @@ mod tests { #[test] fn length_prefix_conversion() { - let mut parser = H264Parser::new(); + let mut parser = primed(); // PES with a single non-IDR NAL let nal_payload = [0x41, 0xAA, 0xBB, 0xCC, 0xDD]; // type 1, 5 bytes @@ -1783,7 +1920,7 @@ mod tests { #[test] fn pts_conversion() { - let mut parser = H264Parser::new(); + let mut parser = primed(); let mut data = Vec::new(); data.extend_from_slice(&[0x00, 0x00, 0x01]); @@ -1811,7 +1948,7 @@ mod tests { #[test] fn pts_preferred_over_dts() { - let mut parser = H264Parser::new(); + let mut parser = primed(); let mut data = Vec::new(); data.extend_from_slice(&[0x00, 0x00, 0x01]); @@ -2052,7 +2189,7 @@ mod tests { // One real NAL at the end so the iterator yields something. data.extend_from_slice(&[0x41, 0xAA, 0xBB]); - let mut parser = H264Parser::new(); + let mut parser = primed(); let frames = parser.parse(&make_pes(data, Some(0))); // Exactly one populated frame; the empty NALs are skipped without // overflowing. @@ -2168,7 +2305,7 @@ mod tests { fn four_byte_start_code_parsed() { // A 4-byte start code (00 00 00 01) must be skipped correctly so the NAL // body begins at the right offset (skip_start_code returns pos+4). - let mut parser = H264Parser::new(); + let mut parser = primed(); let data = vec![0x00, 0x00, 0x00, 0x01, 0x41, 0xAA, 0xBB]; let f = parser.parse(&make_pes(data, Some(0))); assert_eq!(f.len(), 1); @@ -2183,7 +2320,7 @@ mod tests { // The byte(s) before a following 4-byte start code (00 00 00 01) are // leading zeros of that start code, not RBSP, and must be stripped from // the current NAL — NAL 1 must not absorb the extra 00. - let mut parser = H264Parser::new(); + let mut parser = primed(); let mut data = vec![0x00, 0x00, 0x01, 0x41, 0xAA]; // NAL1 = 0x41 0xAA data.extend_from_slice(&[0x00, 0x00, 0x00, 0x01, 0x41, 0xBB]); // 4-byte SC let f = parser.parse(&make_pes(data, Some(0))); @@ -2198,7 +2335,7 @@ mod tests { #[test] fn aud_dropped_but_following_slice_kept() { // AUD (type 9) is dropped from frame data; a following slice survives. - let mut parser = H264Parser::new(); + let mut parser = primed(); let mut data = vec![0x00, 0x00, 0x01, 0x09, 0xF0]; // AUD data.extend_from_slice(&[0x00, 0x00, 0x01, 0x41, 0xAA, 0xBB]); // slice let f = parser.parse(&make_pes(data, Some(0))); @@ -2226,7 +2363,7 @@ mod tests { #[test] fn dts_fallback_when_pts_absent() { // PTS absent → DTS is used (or().map). pts.or(dts) per the comment. - let mut parser = H264Parser::new(); + let mut parser = primed(); let pes = PesPacket { source: None, pid: 0x1011, @@ -2242,7 +2379,7 @@ mod tests { #[test] fn no_pts_no_dts_defaults_zero() { - let mut parser = H264Parser::new(); + let mut parser = primed(); let pes = PesPacket { source: None, pid: 0x1011, diff --git a/src/mux/codec/hevc.rs b/src/mux/codec/hevc.rs index 3bc2633a..a3017686 100644 --- a/src/mux/codec/hevc.rs +++ b/src/mux/codec/hevc.rs @@ -5,6 +5,7 @@ //! Each PES packet = one access unit = one frame. use super::coding::{CodingType, PictureInfo}; +use super::decodable::{Decodable, Need}; use super::startcode::{BitReader, find_start_code, skip_start_code}; use super::{CodecParser, Frame, PesPacket, pts_to_ns}; @@ -26,6 +27,15 @@ const SEI_CONTENT_LIGHT_LEVEL_INFO: u32 = 144; // Dolby Vision RPU NALs (type 62) pass through: only VPS/SPS/PPS/AUD are filtered. // IRAP types (keyframes): BLA, IDR, CRA const NAL_BLA_W_LP: u8 = 16; +const NAL_BLA_N_LP: u8 = 18; +const NAL_IDR_W_RADL: u8 = 19; +const NAL_IDR_N_LP: u8 = 20; +// Leading pictures (H.265 Table 7-1): RADL decode from their IRAP alone, RASL also +// reference pictures before it. +const NAL_RADL_N: u8 = 6; +const NAL_RADL_R: u8 = 7; +const NAL_RASL_N: u8 = 8; +const NAL_RASL_R: u8 = 9; const NAL_RSV_IRAP_VCL23: u8 = 23; // CRA_NUT (Clean Random Access). A CRA at a splice carries RASL pictures // referencing frames before the splice, gone after concatenation ("Could not @@ -115,17 +125,14 @@ pub struct HevcParser { /// ids are 0..=63 (H.265 §7.4.3.3); an out-of-range id is ignored. pps_num_extra: [Option; HEVC_MAX_PPS_COUNT], // Splice-aware CRA→BLA rewrite for a non-seamless BD clip boundary (first - // CRA_NUT -> BLA_W_LP so NoRaslOutput discards dangling RASL). Armed by PTS-backstep - // auto-detect in `parse`, or by the public `mark_clip_boundary` hook (no in-tree caller). + // CRA_NUT -> BLA_W_LP so NoRaslOutput discards dangling RASL). Armed by the public + // `mark_clip_boundary` hook (no in-tree caller); a CRA at a detected join is rewritten too. pending_clip_boundary: bool, - // Highest PES PTS seen, on a monotonic 64-bit timeline (33-bit PTS unwrapped - // across 2^33 wraps — see `pts_wrap_offset`). Auto-detects a non-seamless clip - // boundary when the caller never plumbs one in (see `BACKSTEP_TICKS`). - high_pts: Option, - // Accumulated 2^33-tick offset unwrapping raw PES PTS onto the monotonic - // `high_pts` timeline. Without it, a 33-bit PTS wrap (~26.5h) looks like a - // backward clip reset and false-arms the CRA→BLA rewrite, corrupting valid RASL. - pts_wrap_offset: i64, + /// Which pictures decode from the pictures this output holds; also detects a join (the + /// PTS timeline moves at an IRAP). + decodable: Decodable, + /// What each frame held by `reorder` needs, in decode order. + needs: std::collections::VecDeque, // HDR10 static metadata from prefix/suffix SEI: mastering display (137) and // content light level (144), captured independently and sticky (first wins). // `hdr10()` combines both only when present; SDR streams stay `None`, never fabricated. @@ -158,18 +165,6 @@ struct ContentLightLevel { max_pic_average_light_level: u16, } -// A backward PES-PTS step over this (90kHz ticks) marks a non-seamless BD clip -// boundary: each .m2ts clip has its own PTS base, resetting far more than any -// B-frame reorder window. Mirrors `DISCONTINUITY_BACKSTEP_NS` in `mux/timeline.rs`. -const BACKSTEP_TICKS: i64 = 270_000; - -// Enforced, not just described: 90kHz ticks -> ns is × 100_000 / 9, so this must -// equal `DISCONTINUITY_BACKSTEP_NS` exactly. Changing either constant alone fails the build. -const _: () = assert!( - BACKSTEP_TICKS * 100_000 / 9 == crate::mux::timeline::DISCONTINUITY_BACKSTEP_NS, - "HEVC BACKSTEP_TICKS must mirror mux::timeline::DISCONTINUITY_BACKSTEP_NS" -); - // Bytes reserved at the front of every assembled access unit so the keyframe // param-set re-assert (VPS+SPS+PPS, typically well under 1 KiB on BD/UHD // streams) can splice in without reallocating. Oversized sets just reallocate once. @@ -193,11 +188,6 @@ thread_local! { static GUARD_DROPS: std::cell::Cell = const { std::cell::Cell::new(0) }; } -// 33-bit 90kHz PTS wraps at 2^33 ticks (~26.5h). A backward step of ~2^33 is a -// wraparound (unwrap, add 2^33) not a clip reset (arbitrary sub-2^33 backward -// step); accepting steps within `PTS_WRAP_PERIOD`/2 of a full period separates the two cases. -const PTS_WRAP_PERIOD: i64 = 1 << 33; - impl Default for HevcParser { fn default() -> Self { Self::new() @@ -216,8 +206,8 @@ impl HevcParser { cur_pps: None, pps_num_extra: [None; HEVC_MAX_PPS_COUNT], pending_clip_boundary: false, - high_pts: None, - pts_wrap_offset: 0, + decodable: Decodable::new(true), + needs: std::collections::VecDeque::new(), sei_mastering: None, sei_content_light: None, reorder: None, @@ -229,19 +219,47 @@ impl HevcParser { pub(crate) fn with_ps_reorder(mut self, enabled: bool) -> Self { if enabled { self.reorder = Some(super::reorder::SparsePtsReorder::new()); + // Reconstructed PTS are approximate: only a step past the leading window is a join. + self.decodable = Decodable::new(false); } self } - /// Route a finished frame through the PTS reorderer when enabled, else emit - /// it directly (unchanged transport-stream behaviour). - fn finish(&mut self, explicit: Option, frame: Frame) -> Vec { + /// Route a finished frame through the PTS reorderer when enabled, then keep it + /// only if it decodes from what this output holds. + fn finish( + &mut self, + explicit: Option, + dts: Option, + frame: Frame, + need: Need, + ) -> Vec { match self.reorder.as_mut() { - Some(r) => r.push(explicit, frame), - None => vec![frame], + Some(r) => { + self.needs.push_back(need); + let out = r.push(explicit, frame); + self.decide(out) + } + None => self + .decodable + .admit(frame, need, explicit, dts) + .into_iter() + .collect(), } } + // Decide frames the reorderer released (decode order, display PTS assigned). + fn decide(&mut self, frames: Vec) -> Vec { + frames + .into_iter() + .filter_map(|f| { + let need = self.needs.pop_front().unwrap_or(Need::Nothing); + let pts = Some(f.pts_ns); + self.decodable.admit(f, need, pts, None) + }) + .collect() + } + // Combines mastering-display + content-light SEI into Hdr10Metadata, or `None` until BOTH // are seen — never a half-populated (confidently-wrong) HDR10 record. fn hdr10(&self) -> Option { @@ -304,12 +322,12 @@ impl HevcParser { } /// Mark that the NEXT IRAP this parser sees begins a NON-SEAMLESS BD clip - /// join (MPLS `connection_condition` 0x05 or 0x06). The first CRA at/after - /// this point is rewritten CRA_NUT (21) → BLA_W_LP (16) so a linear decoder - /// sets NoRaslOutput and discards the now-dangling RASL leading pictures. + /// join (MPLS `connection_condition` 0x01 on a PlayItem after the first). The first + /// CRA at/after this point is rewritten CRA_NUT (21) → BLA_W_LP (16) so a linear + /// decoder sets NoRaslOutput, and its RASL leading pictures are dropped. /// - /// MUST be called ONLY for connection_condition 0x05/0x06 — never for 0x01 (seamless/first - /// item) or within a single-clip title. + /// MUST NOT be called for connection_condition 0x05/0x06 (seamless), for the first + /// PlayItem, or within a single-clip title. pub fn mark_clip_boundary(&mut self) { self.pending_clip_boundary = true; } @@ -414,34 +432,17 @@ impl CodecParser for HevcParser { // player reorders). Use PTS not DTS: DTS presents B-frames in decode // order (visible judder) and breaks PTS-based seeking; fall back only if absent. let explicit_pts = pes.pts.or(pes.dts).map(pts_to_ns); + let dts = pes.pts.and(pes.dts).map(pts_to_ns); let pts_ns = explicit_pts.unwrap_or(0); - // Auto-detect a non-seamless clip boundary: mpls connection_condition - // isn't plumbed through the (threaded) mux pipeline, so a PTS backstep - // beyond `BACKSTEP_TICKS` on the unwrapped/high-water timeline arms the rewrite. - if let Some(raw_pts) = pes.pts { - // Unwrap onto the monotonic timeline: if the offset-adjusted value - // dropped ~2^33 below the high-water, the counter wrapped — add - // another period and re-check, rather than treat it as a clip reset. - let mut unwrapped = raw_pts + self.pts_wrap_offset; - if let Some(high) = self.high_pts - && high - unwrapped > PTS_WRAP_PERIOD / 2 - { - self.pts_wrap_offset += PTS_WRAP_PERIOD; - unwrapped += PTS_WRAP_PERIOD; - } - match self.high_pts { - Some(high) if unwrapped < high - BACKSTEP_TICKS => { - self.pending_clip_boundary = true; - self.high_pts = Some(unwrapped); - } - Some(high) => self.high_pts = Some(high.max(unwrapped)), - None => self.high_pts = Some(unwrapped), - } - } + // A join (mpls connection_condition isn't plumbed through the mux pipeline): an + // IRAP whose PTS does not continue the timeline. Its RASL reference another clip. + let joined = self.decodable.joined(explicit_pts, dts); let data = &pes.data; let mut keyframe = false; + // NAL type of the AU's IRAP slice, else its first coded slice: what it needs to decode. + let mut vcl_type: Option = None; // Set once the first CRA of a non-seamless join is seen in this AU. let mut bla_au = false; // Picture coding type, MEASURED from the first coded slice's header. @@ -479,6 +480,9 @@ impl CodecParser for HevcParser { // Measure coding type from the first coded slice (VCL NAL // 0..=31), only once the active PPS is known so the bit // offset to `slice_type` is exact; else decline (`None`), never guess. + if nal_type <= NAL_VCL_MAX && !keyframe { + vcl_type.get_or_insert(nal_type); + } if coding_type.is_none() && nal_type <= NAL_VCL_MAX { // Resolve num_extra from the PPS the slice REFERENCES (by // its slice_pic_parameter_set_id), not the last-active PPS. @@ -531,10 +535,11 @@ impl CodecParser for HevcParser { NAL_AUD => {} t if (NAL_BLA_W_LP..=NAL_RSV_IRAP_VCL23).contains(&t) => { keyframe = true; - // Splice-aware CRA→BLA: the first CRA_NUT after a - // non-seamless boundary becomes BLA_W_LP so NoRaslOutput - // drops RASL. Any IRAP clears the flag; only CRA is rewritten. - if self.pending_clip_boundary && t == NAL_CRA_NUT { + vcl_type = Some(t); + // Splice-aware CRA→BLA: a CRA at a join or a marked boundary + // becomes BLA_W_LP so NoRaslOutput drops RASL. Any IRAP clears + // the mark; only CRA is rewritten. + if (self.pending_clip_boundary || joined) && t == NAL_CRA_NUT { bla_au = true; } self.pending_clip_boundary = false; @@ -630,6 +635,27 @@ impl CodecParser for HevcParser { return Vec::new(); } + // RASL reference pictures before their IRAP (dropped unless the output holds them; + // always after a BLA); RADL only their IRAP. + let need = match vcl_type { + None => Need::Nothing, + Some(NAL_BLA_W_LP..=NAL_BLA_N_LP) => Need::Rap { + closed: false, + broken: true, + }, + Some(NAL_IDR_W_RADL | NAL_IDR_N_LP) => Need::Rap { + closed: true, + broken: false, + }, + Some(NAL_CRA_NUT..=NAL_RSV_IRAP_VCL23) => Need::Rap { + closed: false, + broken: bla_au, + }, + Some(NAL_RADL_N | NAL_RADL_R) => Need::Nothing, + Some(NAL_RASL_N | NAL_RASL_R) => Need::Leading, + Some(_) => Need::Anchor, + }; + // HDR10 static metadata is stamped onto every frame's PictureInfo once // both SEI messages are seen, riding the deferred-muxer path (reads it // from the first coded picture before the track header). `None` for SDR tracks. @@ -650,12 +676,15 @@ impl CodecParser for HevcParser { data: frame_data, duration_ns: None, }; - self.finish(explicit_pts, frame) + self.finish(explicit_pts, dts, frame, need) } fn flush(&mut self) -> Vec { match self.reorder.as_mut() { - Some(r) => r.flush(), + Some(r) => { + let out = r.flush(); + self.decide(out) + } None => Vec::new(), } } @@ -1636,7 +1665,7 @@ mod tests { }; let mut data = nal(NAL_PPS, pps_body(3)); data.extend_from_slice(&nal(1, slice_body)); - let frames = HevcParser::new().parse(&make_pes(data, Some(0))); + let frames = primed().parse(&make_pes(data, Some(0))); assert_eq!(frames.len(), 1); assert_eq!( frames[0].coding.expect("PictureInfo").coding_type(), @@ -1649,7 +1678,7 @@ mod tests { // decides the offset. let mut data0 = nal(NAL_PPS, pps_body(0)); data0.extend_from_slice(&nal(1, slice_body)); - let frames0 = HevcParser::new().parse(&make_pes(data0, Some(0))); + let frames0 = primed().parse(&make_pes(data0, Some(0))); assert_eq!( frames0[0].coding.expect("PictureInfo").coding_type(), CodingType::B, @@ -1693,7 +1722,7 @@ mod tests { data.extend_from_slice(&nal_bytes(NAL_PPS, &pps1)); data.extend_from_slice(&nal_bytes(1, &[slice])); - let frames = HevcParser::new().parse(&make_pes(data, Some(0))); + let frames = primed().parse(&make_pes(data, Some(0))); assert_eq!(frames.len(), 1); assert_eq!( frames[0].coding.expect("PictureInfo").coding_type(), @@ -1715,7 +1744,7 @@ mod tests { let slice64 = [0b1000_0001, 0b0000_0101, 0b1100_0000]; let mut data = nal_bytes(NAL_PPS, &pps64); data.extend_from_slice(&nal_bytes(1, &slice64)); - let frames = HevcParser::new().parse(&make_pes(data, Some(0))); + let frames = primed().parse(&make_pes(data, Some(0))); assert_eq!(frames.len(), 1); assert_eq!( frames[0].coding.map(|c| c.coding_type()), @@ -1730,13 +1759,21 @@ mod tests { assert_eq!(hevc_pps_id(&[0x44, 0x01, pps63[0], pps63[1]]), Some(63)); let mut data = nal_bytes(NAL_PPS, &pps63); data.extend_from_slice(&nal_bytes(1, &slice63)); - let frames = HevcParser::new().parse(&make_pes(data, Some(0))); + let frames = primed().parse(&make_pes(data, Some(0))); assert_eq!( frames[0].coding.map(|c| c.coding_type()), Some(CodingType::I) ); } + /// A parser that has emitted an IDR at PTS 0, so an inter picture after it decodes. + fn primed() -> HevcParser { + let mut p = HevcParser::new(); + let idr = p.parse(&make_pes(nal_bytes(19, &[0x80]), Some(0))); + assert!(idr[0].keyframe); + p + } + /// Build an Annex-B NAL (start code + 2-byte header + body bytes). fn nal_bytes(nal_type: u8, body: &[u8]) -> Vec { let mut v = vec![0x00, 0x00, 0x01]; @@ -1759,7 +1796,7 @@ mod tests { }; let src = crate::pes::SourcePos::at_byte(16384); let run = |slice_body: u8| { - let mut p = HevcParser::new(); + let mut p = primed(); let mut data = nal(NAL_PPS, 0xC0); // active PPS first (sets num_extra) data.extend_from_slice(&nal(1, slice_body)); // then the coded slice let mut pe = make_pes(data, Some(0)); @@ -1793,7 +1830,7 @@ mod tests { // No PPS seen → num_extra is unknown, so slice_type is NOT guessed; the // coding stays None (honestly absent) rather than risk a wrong offset. - let mut p = HevcParser::new(); + let mut p = primed(); let bare = p.parse(&make_pes(nal(1, 0xD8), Some(0))); assert!( bare[0].coding.is_none(), @@ -2368,12 +2405,10 @@ mod tests { } #[test] - #[ignore = "defect: a non-seamless join whose PTS jumps FORWARD is not detected, so its CRA stays CRA and its RASL decode against the previous clip"] fn a_forward_pts_join_onto_a_cra_does_not_hand_its_rasl_to_the_decoder() { - // Clip 1: CRA + trailing pictures at ~0 s. Clip 2 starts 100 s later in - // the source clock (e.g. a PlayItem that skips part of a clip, or a clip - // whose STC runs ahead) with a CRA whose RASL reference a picture of - // the skipped material. + // Clip 1: CRA + trailing pictures at ~0 s. Clip 2 starts 100 s later in the + // source clock (a PlayItem that skips part of a clip, or an STC running ahead) + // with a CRA whose RASL reference a picture of the skipped material. let mut p = HevcParser::new(); let mut frames = Vec::new(); frames.extend(p.parse(&make_pes(cra_au(&[0x01]), Some(0)))); @@ -2407,7 +2442,6 @@ mod tests { } #[test] - #[ignore = "defect: after a gap the ResyncGate resumes on a CRA and passes its RASL, whose reference the gate dropped"] fn a_gap_resync_onto_a_cra_does_not_hand_its_rasl_to_the_decoder() { // IDR, a TRAIL_R that follows lost data, then a CRA (same clock) whose RASL // reference the dropped TRAIL_R. The CRA is not first in the bitstream, so @@ -2435,7 +2469,6 @@ mod tests { } #[test] - #[ignore = "defect: pictures before the stream's first IRAP are emitted with no reference"] fn pictures_before_the_first_irap_are_dropped() { let mut p = HevcParser::new(); let mut frames = Vec::new(); @@ -2444,6 +2477,25 @@ mod tests { assert!(frames[0].keyframe, "the first emitted picture is the IRAP"); } + #[test] + fn a_cra_at_the_stream_start_drops_its_rasl_and_keeps_its_radl() { + // A conforming decoder discards these RASL itself; some hardware decoders do not. + let mut p = HevcParser::new(); + let pes = [ + make_pes(cra_au(&[0x01]), Some(3 * 3750)), + make_pes(nal_bytes(9, &[0xE0]), Some(0)), + make_pes(nal_bytes(8, &[0xE0]), Some(3750)), + make_pes(nal_bytes(7, &[0xE0]), Some(2 * 3750)), + make_pes(nal_bytes(1, &[0xD0]), Some(4 * 3750)), + ]; + let types: Vec = pes + .iter() + .flat_map(|x| p.parse(x)) + .map(|f| nal_type_of(&nals_of(&f.data)[0])) + .collect(); + assert_eq!(types, [NAL_CRA_NUT, 7, 1]); + } + /// Test 2: a CRA with NO boundary marker is left unchanged (CRA stays CRA). #[test] fn cra_without_boundary_unchanged() { @@ -2462,21 +2514,20 @@ mod tests { #[test] fn cra_at_auto_detected_pts_backstep_rewritten_to_bla() { let mut parser = HevcParser::new(); - // Clip 1: a CRA then a few trailing frames advancing the PTS watermark. - // PTS in 90 kHz ticks: 0, then ~1 h into the clip. - let one_hour = 90_000i64 * 3600; - parser.parse(&make_pes(cra_au(&[0x01]), Some(0))); - parser.parse(&make_pes(cra_au(&[0x02]), Some(one_hour))); - // In-clip B-frame dip: PTS steps back a few frames (< BACKSTEP_TICKS). - // Must NOT be mistaken for a clip boundary — this CRA stays CRA. - let dip = parser.parse(&make_pes(cra_au(&[0x03]), Some(one_hour - 3 * 3750))); + // Clip 1, ~1 h in: a CRA, then a picture decoded ahead and a B-frame dip (PTS + // stepping back below it) — reorder, not a boundary — then the next CRA continuing. + let t = 90_000i64 * 3600; + parser.parse(&make_pes(cra_au(&[0x01]), Some(t))); + parser.parse(&make_pes(nal_bytes(1, &[0xD0]), Some(t + 2 * 3750))); + parser.parse(&make_pes(nal_bytes(1, &[0xC0]), Some(t + 3750))); + let next = parser.parse(&make_pes(cra_au(&[0x03]), Some(t + 4 * 3750))); assert_eq!( - nal_type_of(&nals_of(&dip[0].data)[0]), + nal_type_of(&nals_of(&next[0].data)[0]), NAL_CRA_NUT, - "a sub-threshold B-frame PTS dip must not trigger the rewrite" + "a B-frame PTS dip must not trigger the rewrite" ); // Clip 2 splice: PES PTS resets to a new clip base far below the - // watermark (> BACKSTEP_TICKS backward). The opening CRA is rewritten. + // watermark. The opening CRA is rewritten. let splice = parser.parse(&make_pes(cra_au(&[0x04]), Some(0))); assert_eq!( nal_type_of(&nals_of(&splice[0].data)[0]), @@ -2484,7 +2535,7 @@ mod tests { "the splice CRA after a backward PTS reset must become BLA_W_LP" ); // One-shot: the NEXT clip-2 CRA (PTS advancing again) stays CRA. - let next = parser.parse(&make_pes(cra_au(&[0x05]), Some(90_000))); + let next = parser.parse(&make_pes(cra_au(&[0x05]), Some(3750))); assert_eq!( nal_type_of(&nals_of(&next[0].data)[0]), NAL_CRA_NUT, @@ -2499,13 +2550,13 @@ mod tests { let mut parser = HevcParser::new(); let period = 1i64 << 33; // Single clip, PTS climbing toward the 33-bit wrap. Start just below 2^33. - let near_wrap = period - 90_000; // ~1 s before the wrap point + let near_wrap = period - 2 * 3750; // two frames before the wrap point parser.parse(&make_pes(cra_au(&[0x01]), Some(near_wrap))); parser.parse(&make_pes(cra_au(&[0x02]), Some(near_wrap + 3750))); // The counter wraps: raw PTS resets to a small value, but this is the // SAME continuous clip, one frame later. A naive raw comparison sees a // ~2^33 backward step and false-arms the boundary. - let wrapped = parser.parse(&make_pes(cra_au(&[0x03]), Some(7500))); + let wrapped = parser.parse(&make_pes(cra_au(&[0x03]), Some(0))); assert_eq!( nal_type_of(&nals_of(&wrapped[0].data)[0]), NAL_CRA_NUT, @@ -2513,7 +2564,7 @@ mod tests { ); // Continue past the wrap: PTS keeps climbing from the new low base; still // one continuous clip, the CRA after must remain CRA. - let after = parser.parse(&make_pes(cra_au(&[0x04]), Some(11250))); + let after = parser.parse(&make_pes(cra_au(&[0x04]), Some(3750))); assert_eq!( nal_type_of(&nals_of(&after[0].data)[0]), NAL_CRA_NUT, @@ -2677,7 +2728,7 @@ mod tests { #[test] fn parse_trailing_not_keyframe() { - let mut parser = HevcParser::new(); + let mut parser = primed(); // TRAIL_R = type 1 let mut data = Vec::new(); @@ -2697,7 +2748,7 @@ mod tests { #[test] fn parse_tsa_not_keyframe() { - let mut parser = HevcParser::new(); + let mut parser = primed(); // TSA_N = type 2 let mut data = Vec::new(); @@ -2804,7 +2855,7 @@ mod tests { // it. #[test] fn redefined_pps_emitted_inline() { - let mut parser = HevcParser::new(); + let mut parser = primed(); let pps = |body: u8| { let mut v = vec![0x00, 0x00, 0x01]; v.extend_from_slice(&hevc_nal_header(34)); // PPS @@ -2875,7 +2926,7 @@ mod tests { // `00 00 01 00 00 01 `: two adjacent start codes leave the // in-between NAL empty after the trailing-zero strip. It must be skipped, // not written as a bare 0x00000000 length prefix (malformed to a decoder). - let mut parser = HevcParser::new(); + let mut parser = primed(); let mut data = Vec::new(); data.extend_from_slice(&[0x00, 0x00, 0x01]); // start code, empty NAL data.extend_from_slice(&[0x00, 0x00, 0x01]); // next start code @@ -2921,7 +2972,7 @@ mod tests { #[test] fn pts_conversion() { - let mut parser = HevcParser::new(); + let mut parser = primed(); let mut data = Vec::new(); data.extend_from_slice(&[0x00, 0x00, 0x01]); @@ -2940,7 +2991,7 @@ mod tests { #[test] fn pts_preferred_over_dts() { - let mut parser = HevcParser::new(); + let mut parser = primed(); let mut data = Vec::new(); data.extend_from_slice(&[0x00, 0x00, 0x01]); @@ -3899,7 +3950,7 @@ mod tests { #[test] fn type_15_just_below_irap_not_keyframe() { // Type 15 (RASL_R) is one below the IRAP range and must NOT be a keyframe. - let mut parser = HevcParser::new(); + let mut parser = primed(); let mut data = vec![0x00, 0x00, 0x01]; data.extend_from_slice(&hevc_nal_header(15)); data.extend_from_slice(&[0x10, 0x20]); @@ -3911,7 +3962,7 @@ mod tests { #[test] fn type_24_just_above_irap_not_keyframe() { // Type 24 (RSV_VCL24) is one above the IRAP range (..=23) → not keyframe. - let mut parser = HevcParser::new(); + let mut parser = primed(); let mut data = vec![0x00, 0x00, 0x01]; data.extend_from_slice(&hevc_nal_header(24)); data.extend_from_slice(&[0x10, 0x20]); @@ -3937,7 +3988,7 @@ mod tests { #[test] fn hevc_dts_fallback_when_pts_absent() { - let mut parser = HevcParser::new(); + let mut parser = primed(); let pes = PesPacket { source: None, pid: 0x1011, diff --git a/src/mux/codec/mod.rs b/src/mux/codec/mod.rs index 22b735fe..fd3cc860 100644 --- a/src/mux/codec/mod.rs +++ b/src/mux/codec/mod.rs @@ -15,6 +15,8 @@ mod audio_frames; /// Codec-agnostic per-picture coding carrier (`PictureInfo` + accessors). pub mod coding; pub(crate) mod crc; +/// Which pictures decode from the pictures the output holds (start, join, gap). +pub(crate) mod decodable; pub(crate) mod dropgate; /// DTS / DTS-HD elementary-stream parser. pub mod dts; @@ -455,6 +457,7 @@ mod provenance_guard { const NON_PARSER_MODULES: &[&str] = &[ "coding", "crc", + "decodable", "dropgate", "mp2_channels", "pesbuf", diff --git a/src/mux/codec/mpeg2.rs b/src/mux/codec/mpeg2.rs index dd7596bd..cd4f022d 100644 --- a/src/mux/codec/mpeg2.rs +++ b/src/mux/codec/mpeg2.rs @@ -115,6 +115,8 @@ pub struct Mpeg2Parser { last_kept: bool, /// A dropped picture's discontinuity flag, carried to the next emitted frame. carry_discontinuity: bool, + /// A discontinuity was seen; `refs` resets at the I-picture the `ResyncGate` resumes on. + gap: bool, } /// GOP header flags (ISO/IEC 13818-2 §6.3.8). @@ -168,6 +170,7 @@ impl Mpeg2Parser { gop_anchor_tr: None, last_kept: false, carry_discontinuity: false, + gap: false, } } @@ -381,6 +384,12 @@ impl Mpeg2Parser { self.refs = 0; } } + // After a gap the gate drops (and counts) up to the next I, which holds none before it. + self.gap |= bp.frame.discontinuity; + if self.gap && bp.frame.keyframe { + self.gap = false; + self.refs = 0; + } if bp.second_field { keep.push(self.last_kept); continue; @@ -1056,7 +1065,6 @@ mod tests { } #[test] - #[ignore = "defect: after a gap the ResyncGate resumes on an open GOP's I-picture and passes its leading B-pictures, whose forward reference the gate dropped"] fn a_gap_resync_onto_an_open_gop_emits_no_leading_b_pictures() { // GOP 1 (closed) loses data before P5: P5 and the Bs after it reference // the lost picture. GOP 2 is open and PTS-continuous (no join): its leading diff --git a/src/mux/codec/vc1.rs b/src/mux/codec/vc1.rs index 6a8f0b40..18398de1 100644 --- a/src/mux/codec/vc1.rs +++ b/src/mux/codec/vc1.rs @@ -7,6 +7,7 @@ //! (0x0F) in the PES, per the BD VC-1 convention (see `parse`). use super::coding::{CodingType, PictureInfo}; +use super::decodable::{Decodable, Need}; use super::startcode::{BitReader, find_start_code}; use super::{CodecParser, Frame, PesPacket, pts_to_ns}; @@ -74,6 +75,47 @@ fn vc1_frame_coding_type(frame_rbsp: &[u8], seq_header: Option<&[u8]>) -> Option vc1_progressive_ptype(&mut BitReader::new(frame_rbsp)) } +/// What an advanced-profile picture predicts from, for decodability: the PTYPE VLC, or for an +/// interlaced sequence FCM then PTYPE or a field pair's FPTYPE (SMPTE 421M §7.1.1.4; FCM and +/// FPTYPE as ffmpeg `vc1.c` reads them). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Vc1Pic { + /// I, or an I/I or I/P field pair. + Intra, + /// P, skipped, or a P/I or P/P field pair. + Predicted, + /// B, or a field pair holding a B field. + Bi, + /// BI (intra, never a reference), or a BI/BI field pair. + IntraB, +} + +fn vc1_picture(frame_rbsp: &[u8], seq_header: Option<&[u8]>) -> Option { + let interlace = parse_vc1_interlace(seq_header?)?; + let mut br = BitReader::new(frame_rbsp); + // FCM: 0 progressive, 10 frame-interlace, 11 field-interlace. + if interlace && br.read_bit()? == 1 && br.read_bit()? == 1 { + // FPTYPE: bit 2 = B/BI pair, bit 1 = first field P (or BI), bit 0 = second field. + return Some(match br.read_bits(3)? { + 0 | 1 => Vc1Pic::Intra, + 2 | 3 => Vc1Pic::Predicted, + 7 => Vc1Pic::IntraB, + _ => Vc1Pic::Bi, + }); + } + // PTYPE: 0 P, 10 B, 110 I, 1110 BI, 1111 skipped (P). + let mut ones = 0; + while ones < 4 && br.read_bit()? == 1 { + ones += 1; + } + Some(match ones { + 1 => Vc1Pic::Bi, + 2 => Vc1Pic::Intra, + 3 => Vc1Pic::IntraB, + _ => Vc1Pic::Predicted, + }) +} + pub struct Vc1Parser { // First-seen seq_header + entry_point seed MKV codecPrivate. A redefined // body must be emitted IN-BAND at each occurrence, and at every keyframe @@ -91,6 +133,12 @@ pub struct Vc1Parser { /// (HD-DVD EVO) path where the source stamps a PTS once per GOP. `None` on /// the BD/UHD transport path, which carries a per-frame PTS. reorder: Option, + /// Which pictures decode from the pictures this output holds. + decodable: Decodable, + /// What each frame held by `reorder` needs, in decode order. + needs: std::collections::VecDeque, + /// Whether an anchor followed the last entry-point picture: a B before one is leading. + anchor_since_entry: bool, } impl Default for Vc1Parser { @@ -109,6 +157,9 @@ impl Vc1Parser { width: 1920, height: 1080, reorder: None, + decodable: Decodable::new(true), + needs: std::collections::VecDeque::new(), + anchor_since_entry: false, } } @@ -117,18 +168,46 @@ impl Vc1Parser { pub(crate) fn with_ps_reorder(mut self, enabled: bool) -> Self { if enabled { self.reorder = Some(super::reorder::SparsePtsReorder::new()); + // Reconstructed PTS are approximate: only a step past the leading window is a join. + self.decodable = Decodable::new(false); } self } - /// Route a finished frame through the PTS reorderer when enabled, else emit - /// it directly (unchanged transport-stream behaviour). - fn finish(&mut self, explicit: Option, frame: Frame) -> Vec { + /// Route a finished frame through the PTS reorderer when enabled, then keep it + /// only if it decodes from what this output holds. + fn finish( + &mut self, + explicit: Option, + dts: Option, + frame: Frame, + need: Need, + ) -> Vec { match self.reorder.as_mut() { - Some(r) => r.push(explicit, frame), - None => vec![frame], + Some(r) => { + self.needs.push_back(need); + let out = r.push(explicit, frame); + self.decide(out) + } + None => self + .decodable + .admit(frame, need, explicit, dts) + .into_iter() + .collect(), } } + + // Decide frames the reorderer released (decode order, display PTS assigned). + fn decide(&mut self, frames: Vec) -> Vec { + frames + .into_iter() + .filter_map(|f| { + let need = self.needs.pop_front().unwrap_or(Need::Nothing); + let pts = Some(f.pts_ns); + self.decodable.admit(f, need, pts, None) + }) + .collect() + } } // Handle a seq_header or entry_point start-code unit (Annex B raw bytes). Strips the first-seen @@ -163,9 +242,12 @@ impl CodecParser for Vc1Parser { // DTS presents B-frames in decode order (judder, broken seeking). // Fall back to DTS only if PTS is absent. let explicit_pts = pes.pts.or(pes.dts).map(pts_to_ns); + let dts = pes.pts.and(pes.dts).map(pts_to_ns); let ts_ns = explicit_pts.unwrap_or(0); let mut has_seq_header = false; let mut has_entry_point = false; + // BROKEN_LINK and CLOSED_ENTRY: the entry-point header's first two bits (§6.2.1). + let mut entry_flags: Option<(bool, bool)> = None; let mut frame_start: Option = None; // Track whether this AU carried a redefined (in-band) copy of each // header type, in separate temporaries so the keyframe prefix can be @@ -200,6 +282,8 @@ impl CodecParser for Vc1Parser { } SC_ENTRY_POINT => { let end = find_start_code(data, pos + 4).unwrap_or(data.len()); + let b = data.get(pos + 4).copied().unwrap_or(0); + entry_flags = Some((b & 0x80 != 0, b & 0x40 != 0)); if let Some(v) = handle_header( &mut self.entry_point, &mut self.cur_entry_point, @@ -285,6 +369,30 @@ impl CodecParser for Vc1Parser { vc1_frame_coding_type(data.get(fs + 4..)?, self.cur_seq_header.as_deref()) }); + // An entry-point picture opens a segment; a B before the segment's next anchor is + // leading (with CLOSED_ENTRY 0 it may predict from the anchor before the entry point). + let picture = frame_start + .and_then(|fs| vc1_picture(data.get(fs + 4..)?, self.cur_seq_header.as_deref())); + let need = match picture { + None => Need::Nothing, + Some(_) if keyframe || entry_flags.is_some() => { + self.anchor_since_entry = false; + let (broken, closed) = entry_flags.unwrap_or((false, false)); + Need::Rap { closed, broken } + } + Some(Vc1Pic::Intra) => { + self.anchor_since_entry = true; + Need::Intra + } + Some(Vc1Pic::Predicted) => { + self.anchor_since_entry = true; + Need::Anchor + } + Some(Vc1Pic::Bi) if self.anchor_since_entry => Need::Trailing, + Some(Vc1Pic::Bi) => Need::Leading, + Some(Vc1Pic::IntraB) => Need::Nothing, + }; + let frame = Frame { // Coding-type only: VC-1 field order is not decoded here, so // field_order() stays None — honestly absent, never guessed. @@ -298,12 +406,15 @@ impl CodecParser for Vc1Parser { data: frame_data, duration_ns: None, }; - self.finish(explicit_pts, frame) + self.finish(explicit_pts, dts, frame, need) } fn flush(&mut self) -> Vec { match self.reorder.as_mut() { - Some(r) => r.flush(), + Some(r) => { + let out = r.flush(); + self.decide(out) + } None => Vec::new(), } } @@ -586,7 +697,6 @@ mod tests { } #[test] - #[ignore = "defect: an open entry point (CLOSED_ENTRY=0) at the stream start emits its leading B-pictures, whose forward reference precedes the title"] fn an_open_entry_point_at_the_stream_start_drops_its_leading_b_pictures() { assert_eq!( coding_types(entry_gop(false, false)), @@ -594,6 +704,39 @@ mod tests { ); } + #[test] + fn interlaced_pictures_are_judged_from_fcm_and_fptype() { + // INTERLACE = 1. FCM '11' + FPTYPE: '000' I/I, '011' P/P, '100' B/B, '111' BI/BI; + // FCM '10' (frame interlace) + PTYPE '110' I; FCM '0' + PTYPE '10' B. + let seq_int = [0x00, 0x00, 0x01, SC_SEQUENCE_HEADER, 0xC0, 0, 0, 0, 0, 0x40]; + let pic = |b: u8| vc1_picture(&[b], Some(&seq_int)); + assert_eq!(pic(0b1100_0000), Some(Vc1Pic::Intra)); + assert_eq!(pic(0b1101_1000), Some(Vc1Pic::Predicted)); + assert_eq!(pic(0b1110_0000), Some(Vc1Pic::Bi)); + assert_eq!(pic(0b1111_1000), Some(Vc1Pic::IntraB)); + assert_eq!(pic(0b1011_0000), Some(Vc1Pic::Intra)); + assert_eq!(pic(0b0100_0000), Some(Vc1Pic::Bi)); + } + + #[test] + fn an_interlaced_open_entry_point_at_the_stream_start_drops_its_leading_b_field_pairs() { + let seq = [0x00, 0x00, 0x01, SC_SEQUENCE_HEADER, 0xC0, 0, 0, 0, 0, 0x40]; + let ep = [0x00, 0x00, 0x01, SC_ENTRY_POINT, 0x00, 0x00]; + let pic = |b: u8| vec![0x00, 0x00, 0x01, SC_FRAME, b]; + let aus = [ + [&seq[..], &ep[..], &pic(0b1100_1000)].concat(), // I/P field pair + pic(0b1110_0000), // B/B, leading + pic(0b1101_1000), // P/P + pic(0b1110_0000), // B/B, trailing + ]; + let mut p = Vc1Parser::new(); + let n: Vec = aus + .into_iter() + .map(|au| p.parse(&make_pes(au, Some(0))).len()) + .collect(); + assert_eq!(n, [1, 0, 1, 1]); + } + #[test] fn a_closed_entry_point_at_the_stream_start_keeps_every_picture() { assert_eq!( @@ -603,7 +746,6 @@ mod tests { } #[test] - #[ignore = "defect: BROKEN_LINK mid-stream is not read; the Bs it marks undecodable are emitted"] fn a_broken_link_drops_the_leading_b_pictures_mid_stream() { let mut aus = entry_gop(false, true); aus.extend(entry_gop(true, false)); @@ -621,7 +763,6 @@ mod tests { } #[test] - #[ignore = "defect: after a gap the ResyncGate resumes on an open entry point and passes its leading B-pictures, whose forward reference the gate dropped"] fn a_gap_resync_onto_an_open_entry_point_emits_no_leading_b_pictures() { let mut aus = entry_gop(false, true); aus.extend(entry_gop(false, false)); diff --git a/src/mux/mkvstream.rs b/src/mux/mkvstream.rs index 9497143a..32182875 100644 --- a/src/mux/mkvstream.rs +++ b/src/mux/mkvstream.rs @@ -3118,6 +3118,24 @@ mod tests { assert!(m.flush().iter().all(|(_, add)| add.is_none())); } + // The base parser drops a leading picture it cannot decode; the dependent access unit + // of that PTS pairs with no base and is never written. + #[test] + fn mvc_merge_drops_the_dependent_of_a_dropped_base() { + let mut m = empty_merge(); + assert!( + m.ingest(&mvc_frame(2, 1, false, lp(&[&DEP_SLICE]))) + .is_empty() + ); + let dep = lp(&[&DEP_SLICE, &DEP_PPS]); + assert!(m.ingest(&mvc_frame(2, 3, false, dep.clone())).is_empty()); + let e = m.ingest(&mvc_frame(0, 3, true, vec![0x65, 1])); + assert_eq!(e.len(), 1); + assert_eq!(e[0].1.as_deref(), Some(dep.as_slice())); + assert!(m.flush().is_empty()); + assert_eq!(m.orphan_deps, 1); + } + #[test] fn mvc_merge_dep_overflow_drops_old_keeps_newest() { let mut m = empty_merge();