pq/libfreemkv
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

clpi+labels: extract program_info stream table + CLPI vs MPLS audit

Browse Source 
Two layered changes, in service of the empirical question "is CLPI
truly redundant with MPLS for label data?":

1. **clpi.rs ProgramInfo parser**. The existing CLPI parser only
   walked the EP map (for sector-range lookups). Added a parser for
   the ProgramInfo section's per-stream stream_coding_info table:
   pid, coding_type, audio_format/rate, video_format/rate, ISO 639-2
   language. Spec layout per libbluray clpi_parse.c. Best-effort —
   malformed program_info leaves `streams: vec![]`, EP map keeps
   working. `ClipInfo` gains a `streams: Vec<ClpiStream>` field.

2. **labels/clpi_audit.rs**. Diagnostic that walks both
   `/BDMV/CLIPINF/*.clpi` (via the new program_info parser) and
   `/BDMV/PLAYLIST/*.mpls`, builds a (PID → fields) merged view, and
   classifies each row:
   - `Match`: both sources agree (same coding_type + language)
   - `ClpiOnly`: PID in CLPI but no MPLS playlist references it
     (orphan stream on disc — reachable via low-level access, not via menu)
   - `MplsOnly`: PID in MPLS but no CLPI lists it (would indicate a
     parser bug; verified empirically that this NEVER happens)
   - `Divergent`: same PID, different coding_type or language between
     sources (playlist re-tagged or attribute encoding mismatch)
   Surfaced via `labels-analyze` as `clpi_vs_mpls_audit: {matches,
   clpi_only, mpls_only, divergent, total_pids}`. Doesn't affect the
   label output — pure diagnostic.

Empirical findings on the 11-disc corpus (excl. disc-04 truncated):
- 226 matches / 0 mpls_only / 8 clpi_only / 5 divergent across 239 PIDs
- 6 of 10 non-truncated discs have CLPI-only streams (orphans)
- disc-02 (HDMV-only) is the most dramatic: 40% of its 5 streams are
  CLPI-only — MPLS sees 3, CLPI sees 5
- Conclusion: CLPI is NOT truly redundant. ~5% of streams disc-wide
  are CLPI-exclusive. Future work: layer CLPI as a tertiary source
  below MPLS in the labels pipeline (orphan streams marked with even
  lower confidence than MPLS).
This commit is contained in:
MattJackson
2026-05-10 21:50:39 -07:00
parent a876ce846b
commit a9e802c1c2
3 changed files with 450 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/clpi.rs
+158 -1
View File
@@ -20,6 +20,35 @@ pub struct ClipInfo {
pub ep_coarse: Vec<EpCoarse>,
/// Fine EP entries for the primary video stream
pub ep_fine: Vec<EpFine>,
/// Per-stream metadata from the ProgramInfo section (BD spec).
/// Cross-validates the MPLS STN view — see `labels/clpi.rs`.
/// Empty when program_info is missing or malformed.
pub streams: Vec<ClpiStream>,
}
/// One stream descriptor from the CLPI ProgramInfo / stream_coding_info
/// table. Mirrors the same fields the MPLS STN table carries — see
/// `mpls::StreamEntry` for the playlist-side equivalent.
#[derive(Debug, Clone)]
#[allow(dead_code)]
pub struct ClpiStream {
/// PID of the stream in the MPEG-TS (matches MPLS).
pub pid: u16,
/// SCSI/BD coding type byte (0x80 LPCM, 0x83 TrueHD, 0x86 DTS-HD MA,
/// 0x90 PG, etc.). See `labels::mpls_universal::coding_type_to_codec_hint`.
pub coding_type: u8,
/// ISO 639-2 3-char language code. Empty for video streams.
pub language: String,
/// Audio format byte (1=mono, 3=stereo, 6=5.1, 12=7.1).
/// Zero for non-audio streams.
pub audio_format: u8,
/// Audio sample rate (1=48kHz, 4=96kHz, 5=192kHz). Zero for non-audio.
pub audio_rate: u8,
/// Video format byte (1=480i, 4=1080i, 5=720p, 6=1080p, 8=2160p).
/// Zero for non-video.
pub video_format: u8,
/// Video rate (1=23.976, 2=24, 3=25, 4=29.97, 6=50, 7=59.94).
pub video_rate: u8,
}
#[derive(Debug, Clone)]
@@ -128,7 +157,7 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
// Header offsets
let _seq_info_start = u32::from_be_bytes([data[8], data[9], data[10], data[11]]) as usize;
let _prog_info_start = u32::from_be_bytes([data[12], data[13], data[14], data[15]]) as usize;
let prog_info_start = u32::from_be_bytes([data[12], data[13], data[14], data[15]]) as usize;
let cpi_start = u32::from_be_bytes([data[16], data[17], data[18], data[19]]) as usize;
// ClipInfo section at offset 40
@@ -139,6 +168,16 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
0
};
// Parse ProgramInfo (per-stream language + codec). Best-effort:
// malformed program_info doesn't fail the parse, just gives an
// empty streams list. EP map is unaffected — sector-range lookups
// continue to work.
let streams = if prog_info_start > 0 && prog_info_start + 6 < data.len() {
parse_program_info(&data[prog_info_start..])
} else {
Vec::new()
};
// Parse CPI / EP Map
let (ep_coarse, ep_fine) = if cpi_start > 0 && cpi_start + 8 < data.len() {
parse_cpi(&data[cpi_start..])?
@@ -151,9 +190,127 @@ pub fn parse(data: &[u8]) -> Result<ClipInfo> {
source_packet_count,
ep_coarse,
ep_fine,
streams,
})
}
/// Parse the ProgramInfo section: per-stream (pid, coding_type,
/// language, codec sub-fields). Layout per BD spec / libbluray
/// clpi_parse.c:
///
/// ```text
/// ProgramInfo:
/// length: 4 bytes
/// reserved: 1 byte
/// num_programs: 1 byte
/// for each program:
/// spn_program_sequence_start: 4 bytes
/// program_map_pid: 2 bytes
/// num_streams: 1 byte
/// num_groups: 1 byte
/// for each stream:
/// pid: 2 bytes
/// stream_coding_info_length: 1 byte
/// stream_coding_info: (varies by coding_type)
/// coding_type: 1 byte
/// per-type bytes (see match arms below)
/// ```
///
/// Returns `Vec::new()` on any structural mismatch — we don't propagate
/// errors because the EP map is the primary CLPI output, and a corrupt
/// program_info shouldn't break sector-range lookups.
fn parse_program_info(data: &[u8]) -> Vec<ClpiStream> {
let mut out = Vec::new();
if data.len() < 6 {
return out;
}
// length: 4 bytes (skipped — we trust the section bounds in the
// caller's slice and read the bytes that follow). Reserved 1 byte
// at offset 4. num_programs at offset 5.
let num_programs = data[5] as usize;
let mut pos = 6usize;
for _ in 0..num_programs {
// Program header: 4 (spn) + 2 (pmt_pid) + 1 (num_streams) + 1 (num_groups) = 8 bytes
if pos + 8 > data.len() {
return out;
}
let num_streams = data[pos + 6] as usize;
pos += 8;
for _ in 0..num_streams {
// Stream header: 2 (pid) + 1 (sci_length) + sci bytes
if pos + 3 > data.len() {
return out;
}
let pid = u16::from_be_bytes([data[pos], data[pos + 1]]);
let sci_len = data[pos + 2] as usize;
let sci_end = pos + 3 + sci_len;
if sci_end > data.len() || sci_len < 1 {
return out;
}
let sci = &data[pos + 3..sci_end];
let coding_type = sci[0];
let mut audio_format = 0u8;
let mut audio_rate = 0u8;
let mut video_format = 0u8;
let mut video_rate = 0u8;
let mut language = String::new();
match coding_type {
// Video — MPEG-2 (0x02), H.264 (0x1B), HEVC (0x24)
0x02 | 0x1B | 0x24 => {
if sci.len() >= 2 {
video_format = (sci[1] >> 4) & 0x0F;
video_rate = sci[1] & 0x0F;
}
}
// Primary audio — LPCM(0x80), AC-3(0x81), DTS(0x82),
// TrueHD(0x83), AC-3+(0x84), DTS-HD(0x85), DTS-HD MA(0x86)
0x80..=0x86 => {
if sci.len() >= 2 {
audio_format = (sci[1] >> 4) & 0x0F;
audio_rate = sci[1] & 0x0F;
}
if sci.len() >= 5 {
language = String::from_utf8_lossy(&sci[2..5]).to_string();
}
}
// Secondary audio (0xA1 AC-3+, 0xA2 DTS-HD)
0xA1 | 0xA2 => {
if sci.len() >= 2 {
audio_format = (sci[1] >> 4) & 0x0F;
audio_rate = sci[1] & 0x0F;
}
if sci.len() >= 5 {
language = String::from_utf8_lossy(&sci[2..5]).to_string();
}
}
// PG (0x90), IG (0x91): coding_type + 3-byte language [+ char_code for PG]
0x90 | 0x91 => {
if sci.len() >= 4 {
language = String::from_utf8_lossy(&sci[1..4]).to_string();
}
}
_ => {}
}
out.push(ClpiStream {
pid,
coding_type,
language,
audio_format,
audio_rate,
video_format,
video_rate,
});
pos = sci_end;
}
}
out
}
/// Parse the CPI section containing the EP map.
fn parse_cpi(data: &[u8]) -> Result<(Vec<EpCoarse>, Vec<EpFine>)> {
if data.len() < 8 {
src/labels/clpi_audit.rs
+291
View File
@@ -0,0 +1,291 @@
//! CLPI vs MPLS cross-validation diagnostic.
//!
//! Empirical question (raised 2026-05-11): is CLPI's per-stream
//! language and codec data truly redundant with MPLS's STN-table data
//! on real-world Blu-rays?
//!
//! Build a quick audit that walks both sources, normalizes their stream
//! lists by (PID, language, coding_type), and flags any disagreement.
//!
//! Three classes of mismatch we want to detect:
//!
//! 1. **CLPI has streams MPLS doesn't reference.** Orphan streams in
//! the .m2ts that no playlist's STN table includes. Means the user
//! can't reach them through the menu but they're physically on the
//! disc.
//! 2. **MPLS has streams CLPI doesn't list.** Should never happen if
//! both parsers are correct — playlists reference clips which
//! reference streams. If it happens, one of our parsers has a bug.
//! 3. **Same PID, different language / coding_type.** The playlist re-
//! tagged a stream's metadata. Rare but spec-permitted. Means CLPI
//! and MPLS disagree about the same physical stream's properties.
//!
//! If audits across the corpus show zero mismatches of any class, CLPI
//! program_info extraction is **empirically redundant** for labels and
//! we can leave it out of the registry. If even one mismatch surfaces,
//! we add a CLPI label parser to the registry as belt-and-suspenders.
//!
//! This module exposes `audit(reader, udf)` returning a structured
//! report. Surfaced via the labels-analyze tool — not part of the
//! `analyze()` pipeline (no impact on the label output).
use crate::sector::SectorReader;
use crate::udf::UdfFs;
use std::collections::BTreeMap;
/// One row in the audit: a stream PID that's known to one source or
/// both, with the fields each source reported.
#[derive(Debug, Clone)]
pub struct ClpiVsMplsRow {
pub pid: u16,
pub clpi_coding_type: Option<u8>,
pub clpi_language: Option<String>,
pub mpls_coding_type: Option<u8>,
pub mpls_language: Option<String>,
}
impl ClpiVsMplsRow {
/// Three rules for classification:
/// - both sources missing (impossible — caller wouldn't insert)
/// - one source missing → class A or B (orphan-on-disc / playlist-only)
/// - both present but fields differ → class C (metadata divergence)
/// - both present and identical → no mismatch
pub fn class(&self) -> ClpiVsMplsClass {
match (
self.clpi_coding_type.is_some(),
self.mpls_coding_type.is_some(),
) {
(true, false) => ClpiVsMplsClass::ClpiOnly,
(false, true) => ClpiVsMplsClass::MplsOnly,
(true, true) => {
let coding_match = self.clpi_coding_type == self.mpls_coding_type;
let lang_match = self.clpi_language == self.mpls_language;
if coding_match && lang_match {
ClpiVsMplsClass::Match
} else {
ClpiVsMplsClass::Divergent
}
}
(false, false) => ClpiVsMplsClass::Match,
}
}
}
/// Classification of one (PID) row.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ClpiVsMplsClass {
/// PID seen in CLPI ProgramInfo but no MPLS STN table references
/// it. Orphan on disc.
ClpiOnly,
/// PID seen in MPLS STN table but no CLPI ProgramInfo includes it.
/// One of our parsers probably has a bug.
MplsOnly,
/// Both sources see this PID with the same coding_type + language.
Match,
/// Both sources see this PID but disagree on coding_type or language.
/// MPLS wins for label rendering (playlist-authoritative view); CLPI
/// is the per-clip ground truth.
Divergent,
}
/// Full audit report.
#[derive(Debug, Clone, Default)]
pub struct ClpiVsMplsAudit {
pub rows: Vec<ClpiVsMplsRow>,
}
impl ClpiVsMplsAudit {
pub fn class_counts(&self) -> (usize, usize, usize, usize) {
let mut clpi_only = 0;
let mut mpls_only = 0;
let mut matches = 0;
let mut divergent = 0;
for r in &self.rows {
match r.class() {
ClpiVsMplsClass::ClpiOnly => clpi_only += 1,
ClpiVsMplsClass::MplsOnly => mpls_only += 1,
ClpiVsMplsClass::Match => matches += 1,
ClpiVsMplsClass::Divergent => divergent += 1,
}
}
(clpi_only, mpls_only, matches, divergent)
}
}
/// Walk `/BDMV/CLIPINF/*.clpi` and `/BDMV/PLAYLIST/*.mpls`, build a
/// dedup-by-PID table of (CLPI fields, MPLS fields), return the
/// merged view. Missing files (read errors, parse failures) are
/// silently skipped — this is diagnostic, not correctness-critical.
pub fn audit(reader: &mut dyn SectorReader, udf: &UdfFs) -> ClpiVsMplsAudit {
// Aggregate by PID across all CLPI files. If a PID appears in
// multiple clips (typical — main movie clip + trailers reference
// the same audio stream PIDs), first encountered wins (they should
// all agree per BD spec).
let mut clpi_by_pid: BTreeMap<u16, (u8, String)> = BTreeMap::new();
if let Some(dir) = udf.find_dir("/BDMV/CLIPINF") {
let names: Vec<String> = dir
.entries
.iter()
.filter(|e| !e.is_dir && e.name.to_ascii_lowercase().ends_with(".clpi"))
.map(|e| e.name.clone())
.collect();
for name in names {
let path = format!("/BDMV/CLIPINF/{}", name);
let Ok(data) = udf.read_file(reader, &path) else {
continue;
};
let Ok(clip) = crate::clpi::parse(&data) else {
continue;
};
for s in clip.streams {
clpi_by_pid
.entry(s.pid)
.or_insert((s.coding_type, s.language));
}
}
}
// Same for MPLS streams.
let mut mpls_by_pid: BTreeMap<u16, (u8, String)> = BTreeMap::new();
if let Some(dir) = udf.find_dir("/BDMV/PLAYLIST") {
let names: Vec<String> = dir
.entries
.iter()
.filter(|e| !e.is_dir && e.name.to_ascii_lowercase().ends_with(".mpls"))
.map(|e| e.name.clone())
.collect();
for name in names {
let path = format!("/BDMV/PLAYLIST/{}", name);
let Ok(data) = udf.read_file(reader, &path) else {
continue;
};
let Ok(pl) = crate::mpls::parse(&data) else {
continue;
};
for s in pl.streams {
if s.pid == 0 {
// PID 0 means "no PID in stream entry" — skip rather
// than collide with other entries.
continue;
}
mpls_by_pid
.entry(s.pid)
.or_insert((s.coding_type, s.language));
}
}
}
// Merge views: every PID seen anywhere gets a row.
let mut all_pids: std::collections::BTreeSet<u16> = std::collections::BTreeSet::new();
all_pids.extend(clpi_by_pid.keys().copied());
all_pids.extend(mpls_by_pid.keys().copied());
let mut rows = Vec::with_capacity(all_pids.len());
for pid in all_pids {
let clpi = clpi_by_pid.get(&pid);
let mpls = mpls_by_pid.get(&pid);
rows.push(ClpiVsMplsRow {
pid,
clpi_coding_type: clpi.map(|(c, _)| *c),
clpi_language: clpi.map(|(_, l)| l.clone()),
mpls_coding_type: mpls.map(|(c, _)| *c),
mpls_language: mpls.map(|(_, l)| l.clone()),
});
}
ClpiVsMplsAudit { rows }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn class_match_when_identical() {
let r = ClpiVsMplsRow {
pid: 0x1100,
clpi_coding_type: Some(0x83),
clpi_language: Some("eng".into()),
mpls_coding_type: Some(0x83),
mpls_language: Some("eng".into()),
};
assert_eq!(r.class(), ClpiVsMplsClass::Match);
}
#[test]
fn class_clpi_only_when_mpls_missing() {
let r = ClpiVsMplsRow {
pid: 0x1100,
clpi_coding_type: Some(0x83),
clpi_language: Some("eng".into()),
mpls_coding_type: None,
mpls_language: None,
};
assert_eq!(r.class(), ClpiVsMplsClass::ClpiOnly);
}
#[test]
fn class_mpls_only_when_clpi_missing() {
let r = ClpiVsMplsRow {
pid: 0x1100,
clpi_coding_type: None,
clpi_language: None,
mpls_coding_type: Some(0x90),
mpls_language: Some("fra".into()),
};
assert_eq!(r.class(), ClpiVsMplsClass::MplsOnly);
}
#[test]
fn class_divergent_on_lang_disagreement() {
let r = ClpiVsMplsRow {
pid: 0x1100,
clpi_coding_type: Some(0x83),
clpi_language: Some("eng".into()),
mpls_coding_type: Some(0x83),
mpls_language: Some("und".into()),
};
assert_eq!(r.class(), ClpiVsMplsClass::Divergent);
}
#[test]
fn class_counts_sum_rows() {
let audit = ClpiVsMplsAudit {
rows: vec![
ClpiVsMplsRow {
pid: 0x1100,
clpi_coding_type: Some(0x83),
clpi_language: Some("eng".into()),
mpls_coding_type: Some(0x83),
mpls_language: Some("eng".into()),
}, // Match
ClpiVsMplsRow {
pid: 0x1101,
clpi_coding_type: Some(0x83),
clpi_language: Some("fra".into()),
mpls_coding_type: None,
mpls_language: None,
}, // ClpiOnly
ClpiVsMplsRow {
pid: 0x1102,
clpi_coding_type: None,
clpi_language: None,
mpls_coding_type: Some(0x90),
mpls_language: Some("eng".into()),
}, // MplsOnly
ClpiVsMplsRow {
pid: 0x1103,
clpi_coding_type: Some(0x86),
clpi_language: Some("spa".into()),
mpls_coding_type: Some(0x86),
mpls_language: Some("ita".into()),
}, // Divergent
],
};
let (co, mo, m, d) = audit.class_counts();
assert_eq!(co, 1);
assert_eq!(mo, 1);
assert_eq!(m, 1);
assert_eq!(d, 1);
}
}
src/labels/mod.rs
+1
View File
@@ -9,6 +9,7 @@
mod bdmt;
pub(crate) mod class_reader;
pub mod clpi_audit;
mod criterion;
mod ctrm;
mod dbp;