audit: byte caps on GOP buffers, opener-scan resume, honest video codec

Round-6 findings from the 10-phase release audit:

- Wire the documented MAX_PENDING_BYTES byte cap into the MPEG-2 GOP
  buffer (it was dead code) and add an equivalent MAX_GOP_BYTES cap to the
  sparse-PTS reorder, so a crafted stream of few-but-huge access units
  cannot over-allocate — both were bounded only by frame count before.
- probe_evo_streams defaulted an unsniffable HD-DVD video stream to H.264,
  which mis-parses a VC-1 (or still-encrypted) clip into a corrupt track.
  Emit the video stream only when the codec is actually identified — the
  honest outcome, matching the audio path (a real clear clip always carries
  its sequence header at the head).
- Resume the AU-opener search from a cursor (like the boundary search), so
  a long unsynced junk run is O(bytes), not O(buffer) per push.
- Mark mpeg2's now-dead MAX_AU_BUFFER test-only; restore #[doc(hidden)] on
  the aacs probe harness module.
- Add regression tests: the 0xFD video-routing guard, the FMTS-is-UHD key
  state, and the GOP byte caps.

src/mux/codec/mpeg2.rs

@@ -46,10 +46,10 @@ const PICTURE_CODE: u8 = 0x00;
 /// Picture coding type: I-frame.
 const PICTURE_TYPE_I: u8 = 1;
 
-/// Hard cap on the access-unit reassembly buffer. A real MPEG-2 frame is well
-/// under 1 MiB (DVD I-frames ~100 KB); past this cap a corrupt stream that
-/// never produces a second access-unit boundary is force-flushed as a single
-/// frame rather than driving unbounded allocation.
+/// The access-unit reassembly cap now lives in [`crate::mux::au_assembly`] (the
+/// `AuAssembler` owns cross-PES buffering); this mirror exists only so the
+/// force-flush test below can size an over-cap fixture against the same bound.
+#[cfg(test)]
 const MAX_AU_BUFFER: usize = 8 * 1024 * 1024;
 
 /// Cap on frames held awaiting the first PES PTS anchor. A DVD stamps a PTS in
@@ -259,8 +259,11 @@ impl Mpeg2Parser {
             },
         });
         // Safety cap: a stream with no GOP/sequence boundaries would buffer
-        // unbounded. Force-flush a pathologically long run as its own GOP.
-        if self.gop_buf.len() >= MAX_PENDING_FRAMES {
+        // unbounded. Force-flush a pathologically long run as its own GOP —
+        // bounded by BOTH the frame count and the total buffered bytes, so a
+        // crafted stream of few-but-huge pictures cannot over-allocate either.
+        let gop_bytes: usize = self.gop_buf.iter().map(|p| p.frame.data.len()).sum();
+        if self.gop_buf.len() >= MAX_PENDING_FRAMES || gop_bytes >= MAX_PENDING_BYTES {
             self.flush_gop(out);
         }
     }
@@ -1402,13 +1405,12 @@ mod tests {
         let mut data = make_picture_header(PICTURE_TYPE_I);
         // > MAX_AU_BUFFER of slice bytes with no following picture/seq/GOP.
         data.extend(std::iter::repeat_n(0xAA, MAX_AU_BUFFER + 1024));
-        let frames = parser.parse(&make_pes(data, Some(0)));
-        assert!(
-            frames.is_empty(),
-            "over-cap AU is force-COMPLETED (bounded) but buffered in its GOP"
-        );
-        let frames = parser.flush();
-        assert_eq!(frames.len(), 1, "force-flushed at EOF, not dropped");
+        // The AU assembler force-completes the ~8 MiB AU (no boundary), and the
+        // GOP byte cap (MAX_PENDING_BYTES) then force-flushes that oversized GOP
+        // during parse rather than buffering it unbounded.
+        let mut frames = parser.parse(&make_pes(data, Some(0)));
+        frames.extend(parser.flush());
+        assert_eq!(frames.len(), 1, "over-cap AU force-flushed, not dropped");
         assert!(frames[0].keyframe);
     }
 

src/mux/codec/reorder.rs

@@ -44,6 +44,13 @@ const FALLBACK_FRAME_DUR_NS: i64 = 1_001_000_000 / 24;
 /// reassembly buffer grows unbounded on disc-controlled input.
 const MAX_GOP_FRAMES: usize = 600;
 
+/// Byte cap on the buffered GOP, complementing [`MAX_GOP_FRAMES`]. A GOP holds a
+/// couple hundred MB at most in practice; this force-completes a run of
+/// few-but-huge access units so a crafted/corrupt stream cannot over-allocate
+/// (the AU assembler caps each frame at 8 MiB, so 600 frames alone could reach
+/// ~5 GiB without this).
+const MAX_GOP_BYTES: usize = 64 * 1024 * 1024;
+
 /// One buffered coded picture awaiting its GOP's completion.
 struct Pending {
     /// Explicit PES PTS (ns) for this AU, or `None` when the source omitted it.
@@ -72,6 +79,9 @@ struct Gop {
 pub(crate) struct SparsePtsReorder {
     /// Frames of the GOP currently accumulating, in decode order.
     cur: Vec<Pending>,
+    /// Total `data` bytes buffered in `cur` — the byte-cap counter, reset each
+    /// time `cur` is drained into a completed GOP.
+    cur_bytes: usize,
     /// The previously-completed GOP, held one step so its duration can be
     /// calibrated from the next GOP's anchor before it is emitted.
     held: Option<Gop>,
@@ -86,6 +96,7 @@ impl SparsePtsReorder {
     pub(crate) fn new() -> Self {
         Self {
             cur: Vec::new(),
+            cur_bytes: 0,
             held: None,
             dur_ns: 0,
             next_start_ns: 0,
@@ -102,11 +113,15 @@ impl SparsePtsReorder {
         // A keyframe opens a new GOP: the picture already accumulated in `cur` is
         // a complete GOP. Complete it (this frame belongs to the NEW GOP). Also
         // force-complete a pathologically long run that never signalled a
-        // keyframe, so a crafted/corrupt stream cannot buffer without bound.
+        // keyframe — bounded by BOTH frame count and total buffered bytes, so a
+        // crafted/corrupt stream of few-but-huge access units cannot buffer
+        // without bound.
         let mut out = Vec::new();
-        if (frame.keyframe || self.cur.len() >= MAX_GOP_FRAMES) && !self.cur.is_empty() {
+        let over_cap = self.cur.len() >= MAX_GOP_FRAMES || self.cur_bytes >= MAX_GOP_BYTES;
+        if (frame.keyframe || over_cap) && !self.cur.is_empty() {
             out = self.complete_current_gop();
         }
+        self.cur_bytes += frame.data.len();
         self.cur.push(Pending {
             explicit,
             ctype,
@@ -131,6 +146,7 @@ impl SparsePtsReorder {
             return Vec::new();
         }
         let pend = std::mem::take(&mut self.cur);
+        self.cur_bytes = 0;
         let dispidx = display_indices(pend.iter().map(|p| p.ctype));
         let count = pend.len() as i64;
         let anchor = pend
@@ -310,6 +326,29 @@ mod tests {
         );
     }
 
+    #[test]
+    fn force_flushes_a_gop_that_exceeds_the_byte_cap() {
+        use CodingType::*;
+        // Few-but-huge access units with no keyframe must not accumulate past the
+        // byte cap: a handful of ~MAX_GOP_BYTES/4-sized frames force-completes the
+        // GOP well before the frame-count cap, bounding memory.
+        let big = MAX_GOP_BYTES / 4 + 1;
+        let mut r = SparsePtsReorder::new();
+        let mut emitted = 0usize;
+        // Enough huge frames to trigger several byte-cap completions (a GOP is
+        // held one step for duration calibration, so the first emit lands after
+        // the second cap fires) — well under the 600-frame count cap.
+        for i in 0..16 {
+            let mut f = frame(P, false);
+            f.data = vec![0u8; big];
+            emitted += r.push((i == 0).then_some(0), f).len();
+        }
+        assert!(
+            emitted >= 1,
+            "byte cap force-flushed (emitted {emitted}) before the frame-count cap"
+        );
+    }
+
     #[test]
     fn force_flushes_a_gop_that_never_signals_a_keyframe() {
         use CodingType::*;