mirror of
https://github.com/freemkv/libfreemkv.git
synced 2026-10-07 18:13:10 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
84e0c2ca69 | ||
|
|
f4a881b25d | ||
|
|
1edda7fedb | ||
|
|
3dd71a0650 | ||
|
|
579c94bd23 | ||
|
|
3b2a68fe7a | ||
|
|
5b56f112a0 | ||
|
|
f27e4c3088 | ||
|
|
b3576f2ae6 | ||
|
|
39f2d0e992 | ||
|
|
50867516b8 | ||
|
|
193e7c1680 | ||
|
|
b6463729e8 | ||
|
|
dffee56102 | ||
|
|
d8f27cdee0 | ||
|
|
fc13268dc6 | ||
|
|
d9b55f02d0 | ||
|
|
35d66d2059 | ||
|
|
d703ce439b | ||
|
|
6112a26d15 | ||
|
|
2a33364253 | ||
|
|
fa3872ddcc | ||
|
|
c427389f36 | ||
|
|
6980f562df | ||
|
|
e3b2c9d850 | ||
|
|
a383200ef1 | ||
|
|
51d15f551e | ||
|
|
523af461a5 | ||
|
|
695b65149e | ||
|
|
d3f9560689 | ||
|
|
1fa5a7d27f | ||
|
|
55219070f1 | ||
|
|
a34c419521 | ||
|
|
b179846f5d | ||
|
|
e110e80e6e | ||
|
|
597fa34099 | ||
|
|
52eeb949e3 | ||
|
|
6ca62b8411 | ||
|
|
04195c27d8 | ||
|
|
5a8f8e54e1 | ||
|
|
1a2c830cf6 | ||
|
|
637c2bfe36 | ||
|
|
0d28357b4d | ||
|
|
5495332f07 | ||
|
|
e22fc6fd47 | ||
|
|
5b98c13e47 | ||
|
|
8f8f1a62a2 | ||
|
|
e3cfd27942 | ||
|
|
aa12bdad62 | ||
|
|
ef3895cdc5 | ||
|
|
2dcf969ac8 | ||
|
|
005f887bf9 | ||
|
|
d7243a6044 | ||
|
|
f1926c38dc | ||
|
|
4709a73c80 | ||
|
|
55bd1ee868 | ||
|
|
5285e8b6ec | ||
|
|
c5af61e46b | ||
|
|
ed73c1ab68 | ||
|
|
5ce2df0888 | ||
|
|
ea591a36a5 |
+7
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "libfreemkv"
|
||||
version = "0.18.22"
|
||||
version = "0.23.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.86"
|
||||
license = "AGPL-3.0-only"
|
||||
@@ -28,6 +28,12 @@ base64 = "0.22.1"
|
||||
# under project docs ("Acceptable strings: debug/trace logging"). Consumers (autorip)
|
||||
# wire a tracing subscriber and pipe events into the JSONL debug log.
|
||||
tracing = "0.1"
|
||||
# Bounded MPSC channel with kernel-wakeup send_timeout. Used by `io::pipeline`
|
||||
# so the halt-aware send/finish loops can BLOCK on consumer drain instead of
|
||||
# polling — the 50 ms poll cadence of the previous mpsc-based impl capped mux
|
||||
# throughput at ~1 MB/s (see (internal)/memory/
|
||||
# feedback_send_with_halt_poll_throttle.md, 0.21.7).
|
||||
crossbeam-channel = "0.5"
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
+2
-2
@@ -16,7 +16,7 @@ pub mod lfsr;
|
||||
pub(crate) mod tables;
|
||||
|
||||
use crate::disc::Extent;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
|
||||
/// CSS decryption state for a DVD title.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -34,7 +34,7 @@ pub struct CssState {
|
||||
/// 0x80 (start of the encrypted region). This only happens when a new PES
|
||||
/// packet begins at exactly sector offset 128. We scan up to 50000
|
||||
/// scrambled sectors sequentially across all extents.
|
||||
pub fn crack_key(reader: &mut dyn SectorReader, extents: &[Extent]) -> Option<CssState> {
|
||||
pub fn crack_key(reader: &mut dyn SectorSource, extents: &[Extent]) -> Option<CssState> {
|
||||
let mut tried = 0u32;
|
||||
let max_tries = 50_000;
|
||||
|
||||
|
||||
+4
-4
@@ -3,13 +3,13 @@
|
||||
use super::*;
|
||||
use crate::clpi;
|
||||
use crate::mpls;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf;
|
||||
|
||||
impl Disc {
|
||||
/// Scan Blu-ray titles from MPLS playlists.
|
||||
pub(super) fn scan_bluray_titles(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
) -> Vec<DiscTitle> {
|
||||
let mut titles = Vec::new();
|
||||
@@ -31,7 +31,7 @@ impl Disc {
|
||||
}
|
||||
|
||||
pub(super) fn parse_playlist(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
filename: &str,
|
||||
data: &[u8],
|
||||
@@ -199,7 +199,7 @@ impl Disc {
|
||||
/// Prefers English, falls back to first available language.
|
||||
/// Returns None if META directory is empty or XML has no usable title.
|
||||
pub(super) fn read_meta_title(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
) -> Option<String> {
|
||||
let meta_dir = udf_fs.find_dir("/BDMV/META")?;
|
||||
|
||||
+2
-2
@@ -2,13 +2,13 @@
|
||||
|
||||
use super::*;
|
||||
use crate::ifo;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf;
|
||||
|
||||
impl Disc {
|
||||
/// Scan DVD titles from IFO files (VIDEO_TS.IFO + VTS_XX_0.IFO).
|
||||
pub(super) fn scan_dvd_titles(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf_fs: &udf::UdfFs,
|
||||
) -> Vec<DiscTitle> {
|
||||
let dvd_info = match ifo::parse_vmg(reader, udf_fs) {
|
||||
|
||||
+3
-3
@@ -2,7 +2,7 @@
|
||||
|
||||
use super::*;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf;
|
||||
|
||||
/// Result of SCSI AACS handshake (ECDH authentication).
|
||||
@@ -113,12 +113,12 @@ impl Disc {
|
||||
|
||||
/// Resolve disc encryption — AACS 1.0, AACS 2.0, CSS, or none.
|
||||
///
|
||||
/// Reads AACS files from UDF (via SectorReader), resolves keys through
|
||||
/// Reads AACS files from UDF (via SectorSource), resolves keys through
|
||||
/// whatever path works: KEYDB VUK lookup, media key derivation, processing
|
||||
/// keys, device keys. Uses handshake result (volume ID, bus key) if available.
|
||||
pub(super) fn resolve_encryption(
|
||||
udf_fs: &udf::UdfFs,
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
keydb_path: &std::path::Path,
|
||||
handshake: Option<&HandshakeResult>,
|
||||
) -> Result<AacsState> {
|
||||
|
||||
+11
-1243
File diff suppressed because it is too large
Load Diff
+1528
-3
File diff suppressed because it is too large
Load Diff
+29
-1
@@ -186,6 +186,22 @@ impl ReadCtx {
|
||||
/// marginal media is part of the job, and the fast-jump
|
||||
/// threshold is loose so we don't bail too early on a range that
|
||||
/// has scattered good sectors mixed in.
|
||||
///
|
||||
/// `damage_threshold_pct = 6` mirrors `disc/patch.rs`'s
|
||||
/// `PASSN_DAMAGE_THRESHOLD_PCT`. Pass N triggers the damage-skip
|
||||
/// at half the density Pass 1 uses (Pass 1 = 12%) because the
|
||||
/// patch loop's whole job is to chip away at bad ranges — being
|
||||
/// more eager to skip clustered bad sectors converges faster on
|
||||
/// the recoverable good sectors inside a range. The patch-side
|
||||
/// `compute_damage_skip` reads its threshold from
|
||||
/// `PASSN_DAMAGE_THRESHOLD_PCT`; keep the two in sync until the
|
||||
/// patch loop's damage-skip is unified with `handle_read_error`'s
|
||||
/// jump path. (v0.20.8 unification attempt found the unification
|
||||
/// itself blocked on the size-aware `range_remaining/4` cap that
|
||||
/// lives in `compute_damage_skip` but not in
|
||||
/// `handle_read_error::JumpAhead` — see
|
||||
/// `tests/passn_handler_ab.rs` for the A/B fixture that pins
|
||||
/// the divergence point.)
|
||||
pub fn for_patch(batch: u16) -> Self {
|
||||
Self {
|
||||
batch,
|
||||
@@ -194,7 +210,7 @@ impl ReadCtx {
|
||||
consecutive_outer_failures: 0,
|
||||
damage_window: Vec::with_capacity(16),
|
||||
damage_window_max: 16,
|
||||
damage_threshold_pct: 12,
|
||||
damage_threshold_pct: PATCH_DAMAGE_THRESHOLD_PCT,
|
||||
// Pass N is allowed to grind: window-based jump only,
|
||||
// matching the historical behaviour for patch passes.
|
||||
fast_jump_threshold: u64::MAX,
|
||||
@@ -401,6 +417,18 @@ const WEDGE_ABORT_THRESHOLD: u64 = 16;
|
||||
/// next range.
|
||||
const WEDGE_PASS_N_SKIP_SECTORS: u64 = 64;
|
||||
|
||||
/// Single source of truth for the Pass-N damage-window threshold.
|
||||
/// Both [`ReadCtx::for_patch`] and `disc::patch::compute_damage_skip`
|
||||
/// reference this constant so the two damage-skip paths cannot drift.
|
||||
///
|
||||
/// 6% means: with a 16-entry sliding window, the damage-skip fires
|
||||
/// once 1 out of 16 recent reads has failed. Pass 1 uses a 12%
|
||||
/// threshold via `damage_threshold_pct` on `for_sweep`; Pass N is
|
||||
/// twice as eager because patch's whole job is to converge on the
|
||||
/// bad sub-zones inside a NonTrimmed range — a faster trigger
|
||||
/// produces tighter convergence in fewer iterations.
|
||||
pub const PATCH_DAMAGE_THRESHOLD_PCT: usize = 6;
|
||||
|
||||
/// THE single error-handling entry point. Updates `ctx`, returns the
|
||||
/// action the caller must apply.
|
||||
///
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@
|
||||
//! - Mapfile is single-writer (consumer-only). No locking.
|
||||
//! - All `read_error::ReadCtx` state stays on the producer thread.
|
||||
//! - `set_speed` calls happen on the producer thread (same thread that
|
||||
//! owns the `SectorReader`). No new SCSI concurrency.
|
||||
//! owns the `SectorSource`). No new SCSI concurrency.
|
||||
//! - Per-iteration ordering of file-write → mapfile-record is kept
|
||||
//! intact in the consumer (write before record), so the on-disk
|
||||
//! invariant "mapfile only marks Finished what the file has
|
||||
|
||||
+2
-2
@@ -29,7 +29,7 @@ use crate::platform::PlatformDriver;
|
||||
use crate::platform::mt1959::Mt1959;
|
||||
use crate::profile::{self, DriveProfile};
|
||||
use crate::scsi::ScsiTransport;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
@@ -707,7 +707,7 @@ fn open_block_device_for_sg(sg_path: &Path) -> Option<std::os::unix::io::RawFd>
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for Drive {
|
||||
impl SectorSource for Drive {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
|
||||
+3
-3
@@ -9,7 +9,7 @@
|
||||
|
||||
use crate::disc::{Codec, Resolution};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
// ── Public types ────────────────────────────────────────────────────────────
|
||||
@@ -184,7 +184,7 @@ fn bcd_byte(b: u8) -> u32 {
|
||||
///
|
||||
/// Reads the VMG (Video Manager) to discover title sets, then reads each
|
||||
/// VTS IFO to extract PGC chains, cell addresses, and stream attributes.
|
||||
pub fn parse_vmg(reader: &mut dyn SectorReader, udf: &UdfFs) -> Result<DvdInfo> {
|
||||
pub fn parse_vmg(reader: &mut dyn SectorSource, udf: &UdfFs) -> Result<DvdInfo> {
|
||||
let vmg_data = udf.read_file(reader, "/VIDEO_TS/VIDEO_TS.IFO")?;
|
||||
|
||||
// Validate VMG magic
|
||||
@@ -263,7 +263,7 @@ pub fn parse_vmg(reader: &mut dyn SectorReader, udf: &UdfFs) -> Result<DvdInfo>
|
||||
///
|
||||
/// `titles_info` is a list of (chapter_count, vts_title_number) from TT_SRPT.
|
||||
fn parse_vts(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf: &UdfFs,
|
||||
vts_number: u8,
|
||||
titles_info: &[(u16, u8)],
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
//! Bounded-syscall primitive: run a (potentially-blocking) operation
|
||||
//! on a worker thread, with a hard wall-clock deadline and an optional
|
||||
//! cooperative [`Halt`] poll. The calling thread is never trapped
|
||||
//! inside a kernel call.
|
||||
//!
|
||||
//! ## Why this exists
|
||||
//!
|
||||
//! [`crate::halt::Halt`] is cooperative: callers poll
|
||||
//! `is_cancelled()`. It cannot reach inside a syscall the kernel
|
||||
//! currently owns the thread for — `libc::sync_file_range`,
|
||||
//! `libc::fsync`, `File::write` on NFS, and so on. `/api/stop` from
|
||||
//! autorip therefore can't unstick a thread sitting in such a syscall.
|
||||
//!
|
||||
//! [`bounded_syscall`] is the escape hatch: it runs `op` on a fresh
|
||||
//! worker thread, then `recv_timeout`s on a rendezvous channel for the
|
||||
//! result. The wait is broken into ~250 ms slices so the calling
|
||||
//! thread can poll the supplied [`Halt`] in between. If the deadline
|
||||
//! elapses or the halt fires, the worker is intentionally leaked — the
|
||||
//! syscall will unwind whenever the kernel decides, or at process
|
||||
//! exit, but the caller is free to fall back to a degraded code path
|
||||
//! (skip the sync, log loudly, etc.).
|
||||
//!
|
||||
//! ## Trade-offs
|
||||
//!
|
||||
//! - **Thread per call.** Cheap (`std::thread::spawn` is < 100 µs on
|
||||
//! Linux/macOS), but not free. Use on coarse-grained finalisation
|
||||
//! syscalls (`sync_all`, `sync_file_range(WAIT_AFTER)`), not on hot
|
||||
//! inner-loop writes.
|
||||
//! - **Leak on timeout.** A wedged syscall keeps a kernel slot and a
|
||||
//! user-space thread around for the rest of the process's life.
|
||||
//! Bounded by the number of independent rip/mux sessions, which is
|
||||
//! one per disc. The alternative — trapping the caller forever —
|
||||
//! defeats the entire purpose of `/api/stop`.
|
||||
//! - **Halt granularity ~250 ms.** Halt observation is not instant;
|
||||
//! it's the worst-case latency of the `recv_timeout` slice. Good
|
||||
//! enough for human-driven stop requests; not suitable for hard
|
||||
//! real-time deadlines.
|
||||
//!
|
||||
//! ## Single source of truth
|
||||
//!
|
||||
//! Do NOT inline this pattern. Every blocking-syscall wrapper in the
|
||||
//! rip + mux pipeline calls this helper, so changes (e.g. swapping the
|
||||
//! channel impl, adjusting the poll slice, adding metrics) land in one
|
||||
//! place.
|
||||
//!
|
||||
//! ## Platform
|
||||
//!
|
||||
//! Pure `std::thread` + `std::sync::mpsc`. No `cfg(target_os)` needed
|
||||
//! here — the helper itself is platform-agnostic. Callers that wrap
|
||||
//! Linux-only syscalls (`sync_file_range`) still need their own
|
||||
//! `#[cfg(target_os = "linux")]` gates; this helper does not.
|
||||
|
||||
use std::sync::mpsc::{RecvTimeoutError, sync_channel};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::halt::Halt;
|
||||
|
||||
/// Granularity of the halt poll. The receive loop wakes every
|
||||
/// [`POLL_INTERVAL`] to (a) check the [`Halt`] token, then (b) check
|
||||
/// the overall deadline, then go back to waiting. 250 ms is a
|
||||
/// pragmatic balance: short enough that human-driven `/api/stop` feels
|
||||
/// responsive (< 0.5 s p99), long enough that the polling overhead is
|
||||
/// negligible against multi-second syscalls.
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(250);
|
||||
|
||||
/// Failure outcome from a bounded syscall wrapper.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum BoundedError {
|
||||
/// The user-visible halt token fired during the wait. The worker
|
||||
/// thread is intentionally leaked — the caller should fall back to
|
||||
/// a degraded code path rather than waiting on the syscall to
|
||||
/// return.
|
||||
Halted,
|
||||
/// The deadline elapsed before the syscall returned. Same leak
|
||||
/// semantics as `Halted`.
|
||||
Timeout,
|
||||
/// The worker thread panicked, or its sender disconnected before
|
||||
/// sending a result. Treat as a benign no-op (callers usually
|
||||
/// log and continue) rather than a hard error — by definition no
|
||||
/// syscall observably ran to completion in this case.
|
||||
WorkerLost,
|
||||
}
|
||||
|
||||
/// Run a (potentially-blocking) operation on a worker thread with a
|
||||
/// deadline and an optional cooperative halt-token poll. Returns the
|
||||
/// operation's result if it completes within `timeout`; otherwise one
|
||||
/// of [`BoundedError::Halted`] / [`BoundedError::Timeout`] /
|
||||
/// [`BoundedError::WorkerLost`].
|
||||
///
|
||||
/// On `Halted` / `Timeout` the worker thread is intentionally leaked:
|
||||
/// the syscall will unwind whenever the kernel decides, or when the
|
||||
/// process exits. The calling thread is never trapped inside a kernel
|
||||
/// call.
|
||||
///
|
||||
/// `halt` is polled at [`POLL_INTERVAL`] granularity. Pass `None` for
|
||||
/// callers that don't (yet) have a halt token plumbed through —
|
||||
/// behaviour degrades to deadline-only, matching the 0.20.5
|
||||
/// `wait_after_with_timeout` shape this helper generalises.
|
||||
///
|
||||
/// `op` returns `R: Send + 'static`. The closure must own everything
|
||||
/// it touches because it may outlive this call (timeout / halt cases).
|
||||
pub(crate) fn bounded_syscall<F, R>(
|
||||
halt: Option<&Halt>,
|
||||
timeout: Duration,
|
||||
op: F,
|
||||
) -> Result<R, BoundedError>
|
||||
where
|
||||
F: FnOnce() -> R + Send + 'static,
|
||||
R: Send + 'static,
|
||||
{
|
||||
// Rendezvous channel: the worker sends exactly one value (the
|
||||
// op's return) and then exits. Capacity-0 means the send blocks
|
||||
// until we receive — fine on the happy path; on the timeout /
|
||||
// halt path the receiver is dropped and the worker's send
|
||||
// returns Err, which the worker ignores.
|
||||
let (tx, rx) = sync_channel::<R>(0);
|
||||
let _ = thread::Builder::new()
|
||||
.name("freemkv-bounded-syscall".into())
|
||||
.spawn(move || {
|
||||
// Ignore the send error: if we time out (or get halted)
|
||||
// before the worker finishes, the receiver is dropped
|
||||
// and `tx.send` returns Err. Either way, the worker has
|
||||
// nothing more to do.
|
||||
let _ = tx.send(op());
|
||||
});
|
||||
|
||||
let deadline = Instant::now() + timeout;
|
||||
loop {
|
||||
let now = Instant::now();
|
||||
let remaining = deadline.saturating_duration_since(now);
|
||||
let slice = remaining.min(POLL_INTERVAL);
|
||||
match rx.recv_timeout(slice) {
|
||||
Ok(v) => return Ok(v),
|
||||
Err(RecvTimeoutError::Timeout) => {
|
||||
if let Some(h) = halt {
|
||||
if h.is_cancelled() {
|
||||
return Err(BoundedError::Halted);
|
||||
}
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(BoundedError::Timeout);
|
||||
}
|
||||
// Otherwise: another slice.
|
||||
}
|
||||
Err(RecvTimeoutError::Disconnected) => {
|
||||
// Worker thread spawn failed, or it panicked before
|
||||
// sending. Caller treats this as "no syscall ran" —
|
||||
// typically a no-op + log.
|
||||
return Err(BoundedError::WorkerLost);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
#[test]
|
||||
fn op_completes_quickly() {
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), || 42u32);
|
||||
assert!(matches!(r, Ok(42)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn op_exceeds_timeout() {
|
||||
// Op sleeps longer than the deadline → Timeout.
|
||||
let r = bounded_syscall(None, Duration::from_millis(300), || {
|
||||
thread::sleep(Duration::from_secs(2));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Timeout)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_fires_during_wait() {
|
||||
let halt = Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
// Flip the halt from a side thread after ~300 ms — long
|
||||
// enough that the receive loop has rolled at least one
|
||||
// 250 ms slice and is sitting in `recv_timeout` again when
|
||||
// the bit flips.
|
||||
thread::spawn(move || {
|
||||
thread::sleep(Duration::from_millis(300));
|
||||
halt2.cancel();
|
||||
});
|
||||
let r = bounded_syscall(Some(&halt), Duration::from_secs(5), || {
|
||||
thread::sleep(Duration::from_secs(5));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Halted)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_panics() {
|
||||
// Worker panics → sender drops without sending → recv sees
|
||||
// Disconnected → WorkerLost. We use an explicit panic in the
|
||||
// op closure rather than `panic!()` from inside the channel
|
||||
// machinery; the spawned thread's panic is contained (no
|
||||
// process abort) because we don't `.join()` it.
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), || -> u32 {
|
||||
panic!("intentional test panic");
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::WorkerLost)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn halt_already_set_before_call_still_returns_halted() {
|
||||
// Halt observed on the very first poll slice. The op blocks
|
||||
// forever; we must not wait the full timeout to notice the
|
||||
// halt is already set.
|
||||
let halt = Halt::new();
|
||||
halt.cancel();
|
||||
let started = Instant::now();
|
||||
let r = bounded_syscall(Some(&halt), Duration::from_secs(10), || {
|
||||
thread::sleep(Duration::from_secs(10));
|
||||
0u32
|
||||
});
|
||||
assert!(matches!(r, Err(BoundedError::Halted)));
|
||||
// Should bail out within ~1 s; allow 2 s of slack for slow
|
||||
// CI hosts.
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(2),
|
||||
"halt-already-set took {:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ok_path_takes_no_halt_token() {
|
||||
// Sanity: the `None` halt path is the documented zero-config
|
||||
// form (matches the 0.20.5 `wait_after_with_timeout`
|
||||
// behaviour). Op returns immediately; we must observe Ok.
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let f2 = flag.clone();
|
||||
let r = bounded_syscall(None, Duration::from_secs(2), move || {
|
||||
f2.store(true, Ordering::Relaxed);
|
||||
"ok"
|
||||
});
|
||||
assert!(matches!(r, Ok("ok")));
|
||||
assert!(flag.load(Ordering::Relaxed));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
//! Byte-sized bounded producer/consumer channel.
|
||||
//!
|
||||
//! Wraps `std::sync::mpsc::sync_channel` with a byte-accounting
|
||||
//! `Mutex<usize> + Condvar` cap. Sender blocks (cooperatively) when
|
||||
//! `used_bytes + item.byte_size() > capacity_bytes`. Receiver
|
||||
//! decrements `used_bytes` when it takes the item.
|
||||
//!
|
||||
//! Why: the existing producer→consumer channel between `DiscStream`
|
||||
//! (PES producer) and `MuxSink` (PES consumer) is bounded by frame
|
||||
//! count. Frame sizes vary 100× between metadata and keyframes, so a
|
||||
//! count-based cap either starves on small frames or buffers far too
|
||||
//! much memory on big ones. Byte-sized accounting sizes the buffer for
|
||||
//! the worst-case input stall (NFS read p99 ≈ 1–2 s × ~15 MB/s peak
|
||||
//! compressed bitrate ≈ ~30 MB) directly.
|
||||
//!
|
||||
//! The underlying mpsc channel is created with a very large slot count
|
||||
//! so the byte cap (not the slot count) is the real backpressure. Slot
|
||||
//! count is only there to give the kernel a small chunk to wake on.
|
||||
//!
|
||||
//! See `(internal)/memory/project_buffering_architecture.md` §
|
||||
//! Pipeline channel — sizing.
|
||||
|
||||
use std::sync::mpsc::{Receiver as MpscReceiver, RecvError, SendError, SyncSender, sync_channel};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
|
||||
/// Default byte cap for the muxer's input channel. Sized to hide a
|
||||
/// worst-case ~2 s NFS read refill at UHD peak compressed bitrate
|
||||
/// (~15 MB/s); 64 MiB gives headroom. Tweakable; not magic.
|
||||
pub const BYTE_CHANNEL_DEFAULT_CAPACITY: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// Slot capacity of the inner `sync_channel`. Large so the byte cap is
|
||||
/// the real backpressure mechanism — the mpsc slot count only exists
|
||||
/// to give the kernel a chunk to wake on. PES frames are typically
|
||||
/// ~700 B each, so 64 MiB ≈ 90 k frames; 200 k is comfortable headroom.
|
||||
const INNER_SLOT_CAPACITY: usize = 200_000;
|
||||
|
||||
/// Anything whose in-memory cost can be accounted by a single
|
||||
/// `usize`. Implement on the item type sent through [`Sender`].
|
||||
pub trait HasByteSize {
|
||||
/// Bytes this item contributes to the channel's used budget.
|
||||
/// Must be > 0 to make progress (a 0-byte item would never
|
||||
/// block the sender no matter the cap; see send_blocks_at_capacity
|
||||
/// test).
|
||||
fn byte_size(&self) -> usize;
|
||||
}
|
||||
|
||||
impl HasByteSize for crate::pes::PesFrame {
|
||||
fn byte_size(&self) -> usize {
|
||||
// Frame data + the fixed header overhead the serializer
|
||||
// writes (track + pts + keyframe + len). The `Vec<u8>` heap
|
||||
// allocation also has alloc-header overhead but that's
|
||||
// <0.1 % at typical frame sizes — folding it in would just
|
||||
// add noise to the budget.
|
||||
self.data.len() + 14
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared book-keeping between [`Sender`] and [`Receiver`]. Wrapped in
|
||||
/// an `Arc` because both halves hold it independently.
|
||||
struct Accounting {
|
||||
used: Mutex<usize>,
|
||||
cv: Condvar,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
/// Send half of the byte-bounded channel.
|
||||
///
|
||||
/// `send` blocks (on a `Condvar`) when adding the item would push
|
||||
/// `used_bytes` past `capacity_bytes`. Unblocks when the receiver
|
||||
/// `recv`s items out and notifies. Returns `Err(item)` if the
|
||||
/// receiver has been dropped — mirrors `mpsc::SyncSender::send`.
|
||||
pub struct Sender<T: HasByteSize> {
|
||||
tx: SyncSender<T>,
|
||||
acct: Arc<Accounting>,
|
||||
}
|
||||
|
||||
impl<T: HasByteSize> Clone for Sender<T> {
|
||||
fn clone(&self) -> Self {
|
||||
Sender {
|
||||
tx: self.tx.clone(),
|
||||
acct: self.acct.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: HasByteSize> Sender<T> {
|
||||
/// Push one item. Blocks until adding it would not exceed the
|
||||
/// capacity, then sends through the inner mpsc channel.
|
||||
pub fn send(&self, item: T) -> Result<(), SendError<T>> {
|
||||
let sz = item.byte_size();
|
||||
// Reserve capacity first. The reservation is observable to
|
||||
// other senders via `used`; only after we win the slot do we
|
||||
// hand the item to the inner mpsc channel. That ordering means
|
||||
// `used` is always a conservative upper bound on what's in the
|
||||
// mpsc queue + about-to-be-sent.
|
||||
{
|
||||
let mut used = self.acct.used.lock().expect("byte_channel poisoned");
|
||||
// An item bigger than the whole capacity will never fit; let
|
||||
// it through anyway as a one-shot reservation, otherwise the
|
||||
// sender deadlocks forever waiting for `used == 0` AND
|
||||
// nothing in flight. The receiver will drain it on the
|
||||
// other side. Same behaviour as `std::sync::mpsc` for
|
||||
// arbitrarily large messages.
|
||||
while *used + sz > self.acct.capacity && *used > 0 {
|
||||
used = self.acct.cv.wait(used).expect("byte_channel cv poisoned");
|
||||
}
|
||||
*used += sz;
|
||||
}
|
||||
match self.tx.send(item) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(SendError(returned)) => {
|
||||
// Receiver dropped — refund the reservation so a later
|
||||
// sender on a clone doesn't observe phantom used bytes
|
||||
// (the receiver is gone so nobody will decrement).
|
||||
let mut used = self.acct.used.lock().expect("byte_channel poisoned");
|
||||
*used = used.saturating_sub(sz);
|
||||
self.acct.cv.notify_all();
|
||||
Err(SendError(returned))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Receive half of the byte-bounded channel.
|
||||
///
|
||||
/// `recv` blocks on the inner mpsc until an item is available, then
|
||||
/// decrements the byte-accounting and wakes any sender waiting on
|
||||
/// capacity.
|
||||
pub struct Receiver<T: HasByteSize> {
|
||||
rx: MpscReceiver<T>,
|
||||
acct: Arc<Accounting>,
|
||||
}
|
||||
|
||||
impl<T: HasByteSize> Receiver<T> {
|
||||
/// Take the next item. Returns `Err(RecvError)` when all senders
|
||||
/// have been dropped and the channel is empty.
|
||||
pub fn recv(&self) -> Result<T, RecvError> {
|
||||
let item = self.rx.recv()?;
|
||||
let sz = item.byte_size();
|
||||
let mut used = self.acct.used.lock().expect("byte_channel poisoned");
|
||||
*used = used.saturating_sub(sz);
|
||||
// Notify all so multi-sender setups wake every blocked sender,
|
||||
// not just one. Wasted wakeups are cheap; missed wakeups would
|
||||
// be a deadlock.
|
||||
self.acct.cv.notify_all();
|
||||
Ok(item)
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a byte-bounded channel with the given capacity in bytes.
|
||||
/// Returns a `(Sender, Receiver)` pair; clone the `Sender` for
|
||||
/// multi-producer setups.
|
||||
pub fn channel<T: HasByteSize>(capacity_bytes: usize) -> (Sender<T>, Receiver<T>) {
|
||||
let (tx, rx) = sync_channel::<T>(INNER_SLOT_CAPACITY);
|
||||
let acct = Arc::new(Accounting {
|
||||
used: Mutex::new(0),
|
||||
cv: Condvar::new(),
|
||||
capacity: capacity_bytes,
|
||||
});
|
||||
(
|
||||
Sender {
|
||||
tx,
|
||||
acct: acct.clone(),
|
||||
},
|
||||
Receiver { rx, acct },
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Test payload — its `byte_size` returns whatever we passed at
|
||||
/// construction so capacity math is exact and predictable.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
struct Item {
|
||||
sz: usize,
|
||||
tag: u32,
|
||||
}
|
||||
|
||||
impl HasByteSize for Item {
|
||||
fn byte_size(&self) -> usize {
|
||||
self.sz
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_recv_round_trip() {
|
||||
let (tx, rx) = channel::<Item>(1024);
|
||||
for i in 0..5 {
|
||||
tx.send(Item { sz: 100, tag: i }).unwrap();
|
||||
}
|
||||
for i in 0..5 {
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got, Item { sz: 100, tag: i });
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn byte_accounting_decrements_on_recv() {
|
||||
// Internal book-keeping check via observable side-effect: after
|
||||
// sending K items totalling N bytes and receiving them all, a
|
||||
// subsequent send of an N-byte item must NOT block (no items
|
||||
// in flight, all capacity refunded).
|
||||
let (tx, rx) = channel::<Item>(1024);
|
||||
for _ in 0..4 {
|
||||
tx.send(Item { sz: 256, tag: 0 }).unwrap();
|
||||
}
|
||||
for _ in 0..4 {
|
||||
rx.recv().unwrap();
|
||||
}
|
||||
// Cap is now fully available again. Send a 1024-byte item; the
|
||||
// `used > 0` guard means it goes through alone (no wait).
|
||||
let start = Instant::now();
|
||||
tx.send(Item { sz: 1024, tag: 99 }).unwrap();
|
||||
assert!(start.elapsed() < Duration::from_millis(100));
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got.tag, 99);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_blocks_at_capacity_unblocks_on_recv() {
|
||||
// Cap = 200 bytes, item = 100 bytes. First two sends fit
|
||||
// exactly; the third must block until a recv frees capacity.
|
||||
let (tx, rx) = channel::<Item>(200);
|
||||
tx.send(Item { sz: 100, tag: 0 }).unwrap();
|
||||
tx.send(Item { sz: 100, tag: 1 }).unwrap();
|
||||
|
||||
let tx2 = tx.clone();
|
||||
let sent_at = Arc::new(Mutex::new(None::<Instant>));
|
||||
let sent_at2 = sent_at.clone();
|
||||
let h = thread::spawn(move || {
|
||||
tx2.send(Item { sz: 100, tag: 2 }).unwrap();
|
||||
*sent_at2.lock().unwrap() = Some(Instant::now());
|
||||
});
|
||||
|
||||
// Give the sender thread a head start; it should be parked in
|
||||
// `cv.wait` because used (200) + 100 > capacity (200).
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
assert!(
|
||||
sent_at.lock().unwrap().is_none(),
|
||||
"third send should be blocked at capacity"
|
||||
);
|
||||
|
||||
// Drain one. Sender wakes and completes.
|
||||
let recv_at = Instant::now();
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got.tag, 0);
|
||||
h.join().unwrap();
|
||||
|
||||
let sent_when = sent_at.lock().unwrap().unwrap();
|
||||
assert!(
|
||||
sent_when >= recv_at,
|
||||
"sender must complete AFTER receiver freed capacity"
|
||||
);
|
||||
|
||||
// Drain the remaining two.
|
||||
assert_eq!(rx.recv().unwrap().tag, 1);
|
||||
assert_eq!(rx.recv().unwrap().tag, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_larger_than_capacity_still_goes_through() {
|
||||
// Pathological case: a single item bigger than the capacity.
|
||||
// The guard `*used > 0` lets it through when the channel is
|
||||
// empty (otherwise the sender deadlocks forever). Matches
|
||||
// `mpsc::SyncSender` semantics for oversize messages.
|
||||
let (tx, rx) = channel::<Item>(100);
|
||||
tx.send(Item { sz: 1000, tag: 7 }).unwrap();
|
||||
let got = rx.recv().unwrap();
|
||||
assert_eq!(got, Item { sz: 1000, tag: 7 });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_send_recv_stress() {
|
||||
// 4 sender threads × 1k items each, 1 receiver. Verify byte
|
||||
// accounting stays sane (channel never deadlocks, every item
|
||||
// arrives exactly once) under contention.
|
||||
const SENDERS: u32 = 4;
|
||||
const PER_SENDER: u32 = 1000;
|
||||
const TOTAL: u32 = SENDERS * PER_SENDER;
|
||||
|
||||
let (tx, rx) = channel::<Item>(8 * 1024);
|
||||
let sent = Arc::new(AtomicUsize::new(0));
|
||||
let mut handles = Vec::new();
|
||||
for s in 0..SENDERS {
|
||||
let tx = tx.clone();
|
||||
let sent = sent.clone();
|
||||
handles.push(thread::spawn(move || {
|
||||
for i in 0..PER_SENDER {
|
||||
// Vary item size so accounting actually has to
|
||||
// multiplex differently-sized blockers. 1B → 256B.
|
||||
let sz = 1 + ((i as usize) % 256);
|
||||
tx.send(Item {
|
||||
sz,
|
||||
tag: s * PER_SENDER + i,
|
||||
})
|
||||
.unwrap();
|
||||
sent.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}));
|
||||
}
|
||||
// Drop our local sender so the receiver can eventually see
|
||||
// RecvError once all sender clones are done. Cloning the
|
||||
// sender into each producer means each clone Drop'd separately.
|
||||
drop(tx);
|
||||
|
||||
let mut received = 0u32;
|
||||
while let Ok(_item) = rx.recv() {
|
||||
received += 1;
|
||||
}
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
assert_eq!(received, TOTAL);
|
||||
assert_eq!(sent.load(Ordering::SeqCst) as u32, TOTAL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_after_recv_dropped_returns_err() {
|
||||
let (tx, rx) = channel::<Item>(1024);
|
||||
drop(rx);
|
||||
let r = tx.send(Item { sz: 10, tag: 0 });
|
||||
assert!(r.is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
//! Linux read-side platform hooks: sequential-access hint at open +
|
||||
//! periodic page-cache eviction during streaming reads.
|
||||
//!
|
||||
//! ## Why both
|
||||
//!
|
||||
//! `POSIX_FADV_SEQUENTIAL` at open widens the kernel's readahead window
|
||||
//! so each pread aggregates into fewer NFS round-trips. `DONTNEED` on
|
||||
//! the consumed window (called periodically by the caller) drops the
|
||||
//! already-read pages from the page cache so an 85 GB streaming ISO
|
||||
//! read doesn't fill memory and starve concurrent writes (the MKV
|
||||
//! output during mux). Together they mirror the write-side
|
||||
//! WritebackPipeline's policy.
|
||||
//!
|
||||
//! ## History
|
||||
//!
|
||||
//! Pre-Phase-1 (0.20.7 baseline) had both. Phase 1's introduction of
|
||||
//! `FileSectorSource` silently dropped the read-side DONTNEED, and
|
||||
//! 0.21.2's revert of `SEQUENTIAL` (mistakenly attributing a regression
|
||||
//! to it) removed the hint. Net effect: 85 GB of ISO reads pinned in
|
||||
//! the page cache + no readahead widening → mux throughput collapse
|
||||
//! from 18 MB/s historical to 2.7-8 MB/s on 0.21.x. Restored in 0.21.6.
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
pub(super) fn hint_sequential(file: &File, _len_bytes: u64) {
|
||||
// Best-effort: return value ignored. A fadvise failure has no
|
||||
// user-observable consequence.
|
||||
unsafe {
|
||||
libc::posix_fadvise(file.as_raw_fd(), 0, 0, libc::POSIX_FADV_SEQUENTIAL);
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop pages in the half-open byte range `[start, start+len)` from
|
||||
/// the page cache. Called periodically by `read_sectors` to bound the
|
||||
/// read-side page cache pressure.
|
||||
pub(super) fn drop_window(file: &File, start: u64, len: u64) {
|
||||
unsafe {
|
||||
libc::posix_fadvise(
|
||||
file.as_raw_fd(),
|
||||
start as i64,
|
||||
len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
//! macOS: hint the kernel to prefetch a generous chunk. macOS has no
|
||||
//! direct `POSIX_FADV_SEQUENTIAL` equivalent; the idiomatic hint is
|
||||
//! `fcntl(F_RDADVISE, &radvisory)` describing the byte range you
|
||||
//! intend to read soon. We point it at the whole file (clamped to a
|
||||
//! ceiling so a multi-TB ISO doesn't ask the kernel to prefetch
|
||||
//! everything at once).
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
/// `F_RDADVISE` opcode — not in libc's named constants on all SDKs.
|
||||
const F_RDADVISE: libc::c_int = 44;
|
||||
|
||||
/// Cap on the byte length we pass to `F_RDADVISE`. Asking for a
|
||||
/// multi-GB readahead window is counterproductive — the OS doesn't
|
||||
/// have that much cache to throw at one fd. 64 MiB is generous for
|
||||
/// our use case (sweep, mux) and matches the byte-channel cap so the
|
||||
/// kernel's prefetch ≥ our app-level pipeline depth.
|
||||
const RDADVISE_MAX_BYTES: i64 = 64 * 1024 * 1024;
|
||||
|
||||
/// `radvisory` per `<sys/fcntl.h>`. repr(C) layout is stable.
|
||||
#[repr(C)]
|
||||
struct RadAdvisory {
|
||||
ra_offset: libc::off_t,
|
||||
ra_count: libc::c_int,
|
||||
}
|
||||
|
||||
pub(super) fn hint_sequential(file: &File, len_bytes: u64) {
|
||||
let bytes = (len_bytes as i64).min(RDADVISE_MAX_BYTES);
|
||||
let mut ra = RadAdvisory {
|
||||
ra_offset: 0,
|
||||
ra_count: bytes as libc::c_int,
|
||||
};
|
||||
// Best-effort.
|
||||
unsafe {
|
||||
libc::fcntl(file.as_raw_fd(), F_RDADVISE, &mut ra);
|
||||
}
|
||||
}
|
||||
|
||||
/// macOS has no direct `POSIX_FADV_DONTNEED` equivalent for a byte
|
||||
/// range. `fcntl(F_NOCACHE)` would disable caching globally on the fd
|
||||
/// (too coarse — we want the unread region to still benefit). Best
|
||||
/// approximation: no-op. macOS's unified buffer cache is generally
|
||||
/// less prone to the pin-everything pathology that triggers the
|
||||
/// regression on Linux NFS clients.
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
@@ -0,0 +1,428 @@
|
||||
//! [`FileSectorSource`] — read 2048-byte sectors from an ISO file on
|
||||
//! disk, with an internal 32 MiB read-ahead buffer.
|
||||
//!
|
||||
//! ## Why the buffer
|
||||
//!
|
||||
//! On NFS-mounted ISOs, an unbuffered `pread(2048)` per sector pays an
|
||||
//! NFS round-trip for every sector. With `rsize=1 MiB` and a 100-150 ms
|
||||
//! NFS RTT, that's three orders of magnitude more round trips than
|
||||
//! necessary — the muxer goes read-bound on every read, even though
|
||||
//! the local NFS client could deliver MB/s on bigger requests.
|
||||
//!
|
||||
//! Internally this source keeps a [`READAHEAD_BUF_BYTES`] (32 MiB)
|
||||
//! window pre-read from the file. `read_sectors(lba, count)` slices
|
||||
//! into the window if `[lba, lba+count)` is contained in it; otherwise
|
||||
//! the window is refilled (full-size aligned to the requested LBA's
|
||||
//! buffer position).
|
||||
//!
|
||||
//! ## Access pattern assumption
|
||||
//!
|
||||
//! The buffer is sized for **forward-sequential** reads (sweep, mux).
|
||||
//! Reverse-mode patch is range-local, so a refill per range works out
|
||||
//! fine (the buffer covers the whole range for typical bad-range
|
||||
//! sizes). Random-access reads thrash the buffer — at which point the
|
||||
//! 32 MiB pre-read is wasted work. We accept that: the use case is
|
||||
//! mux + sweep, both forward-sequential.
|
||||
//!
|
||||
//! Backward seeks rebuffer from the new LBA; partial reads at EOF
|
||||
//! return only the bytes that exist (the underlying file is shorter
|
||||
//! than a full buffer slot).
|
||||
//!
|
||||
//! ## Platform open hints
|
||||
//!
|
||||
//! On `open()` each platform issues its "sequential access expected"
|
||||
//! hint to the kernel so OS-level readahead widens. The hint lives in
|
||||
//! a per-OS sibling module ([`linux::hint_sequential`] et al.) — no
|
||||
//! inline `#[cfg]` in this file.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
mod other;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux as platform;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos as platform;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
use other as platform;
|
||||
#[cfg(target_os = "windows")]
|
||||
use windows as platform;
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::Path;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorSource;
|
||||
|
||||
/// Internal read-ahead buffer size. 32 MiB amortises one NFS round
|
||||
/// trip across ~16 k sectors — three orders of magnitude fewer trips
|
||||
/// than per-sector pread, and large enough to coast through a typical
|
||||
/// NFS server commit blip.
|
||||
///
|
||||
/// 0.21.2: shrunk from 32 MiB → 4 MiB. On NFS-backed ISOs with
|
||||
/// concurrent NFS writes (the mux phase), a 32 MiB refill bursts the
|
||||
/// TCP connection hard enough to starve the writer thread, observed
|
||||
/// empirically as a ~3× drop in sustained mux throughput on the
|
||||
/// rip1/unraid-1 setup. 4 MiB matches `rsize=1 MiB` × 4 round-trips
|
||||
/// and interleaves cleanly with writes.
|
||||
///
|
||||
/// Tweakable. Named const, not a magic number.
|
||||
pub const READAHEAD_BUF_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
const SECTOR_SIZE: usize = 2048;
|
||||
/// Sectors per refill: [`READAHEAD_BUF_BYTES`] / [`SECTOR_SIZE`]. The
|
||||
/// buffer always tries to hold this many, except at the tail of the
|
||||
/// file where less data exists.
|
||||
const BUF_SECTORS: u32 = (READAHEAD_BUF_BYTES / SECTOR_SIZE) as u32;
|
||||
|
||||
/// SectorSource backed by a file (ISO image) with an internal
|
||||
/// `READAHEAD_BUF_BYTES`-sized read-ahead window.
|
||||
///
|
||||
/// `read_sectors` is satisfied from the buffer when possible; otherwise
|
||||
/// a full-buffer refill is issued at the requested LBA's position and
|
||||
/// the call is re-tried against the freshly populated window.
|
||||
/// Bytes-read threshold per `posix_fadvise(DONTNEED)` drop on the
|
||||
/// read side. Mirrors `WRITEBACK_CHUNK_BYTES` so the read-side page
|
||||
/// cache stays bounded the same way the write side does.
|
||||
///
|
||||
/// 0.21.6: re-added after empirical discovery that Phase 1 had silently
|
||||
/// dropped this from the pre-Phase-1 (0.20.7) hot path. Without it,
|
||||
/// 85 GB of streaming ISO reads pin the entire file in the kernel page
|
||||
/// cache, starving the MKV writeback and collapsing mux throughput
|
||||
/// (observed: 2.7 MB/s mux on 0.21.5 vs. 70 MB/s isolated NFS reads).
|
||||
const READ_DROP_CHUNK_BYTES: u64 = 32 * 1024 * 1024;
|
||||
|
||||
pub struct FileSectorSource {
|
||||
file: File,
|
||||
/// Total file size in sectors. Constant after construction;
|
||||
/// surfaced via [`SectorSource::capacity_sectors`].
|
||||
capacity: u32,
|
||||
/// 0.21.3+: the app-level buffer is no longer touched on the hot
|
||||
/// path (every `read_sectors` is a direct pread). The fields are
|
||||
/// retained so a future per-source-type policy (e.g. a local-disk
|
||||
/// source where batched reads ARE beneficial) can re-enable
|
||||
/// buffering cleanly without re-plumbing the struct.
|
||||
#[allow(dead_code)]
|
||||
buf: Box<[u8]>,
|
||||
#[allow(dead_code)]
|
||||
buf_start_lba: u32,
|
||||
buf_len_sectors: u32,
|
||||
/// 0.21.6: bytes read since the last DONTNEED drop. Drives the
|
||||
/// per-`READ_DROP_CHUNK_BYTES` page-cache eviction in read_sectors.
|
||||
bytes_read_since_drop: u64,
|
||||
/// 0.21.6: file offset at which the current drop window starts.
|
||||
/// The next DONTNEED drops from `drop_window_start` for
|
||||
/// `bytes_read_since_drop` bytes.
|
||||
drop_window_start: u64,
|
||||
}
|
||||
|
||||
impl FileSectorSource {
|
||||
/// Open an existing ISO file for reading. Capacity is derived
|
||||
/// from `metadata().len() / 2048`. Returns
|
||||
/// [`Error::IsoTooLarge`] if the file would exceed the 32-bit
|
||||
/// LBA address space (~8 TB).
|
||||
///
|
||||
/// Issues the platform's "sequential access expected" hint on the
|
||||
/// fd (Linux `posix_fadvise(SEQUENTIAL)`, macOS `fcntl(F_RDADVISE)`,
|
||||
/// Windows TODO stub) so the kernel's readahead widens.
|
||||
pub fn open(path: &Path) -> std::io::Result<Self> {
|
||||
let file = File::open(path)?;
|
||||
let len = file.metadata()?.len();
|
||||
let sectors = len / SECTOR_SIZE as u64;
|
||||
if sectors > u32::MAX as u64 {
|
||||
return Err(Error::IsoTooLarge {
|
||||
path: path.to_string_lossy().into_owned(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let capacity = sectors as u32;
|
||||
|
||||
// Best-effort sequential hint. Ignored on platforms without
|
||||
// an equivalent primitive (or where the API exists but the
|
||||
// FS doesn't honour it).
|
||||
platform::hint_sequential(&file, len);
|
||||
|
||||
// Pre-allocate the buffer once. `vec![0u8; N].into_boxed_slice()`
|
||||
// is the canonical way to fix the allocation size up-front;
|
||||
// `Vec::with_capacity` would leave `len == 0` and force callers
|
||||
// to do unsafe length manipulation to write into it.
|
||||
let buf = vec![0u8; READAHEAD_BUF_BYTES].into_boxed_slice();
|
||||
|
||||
Ok(Self {
|
||||
file,
|
||||
capacity,
|
||||
buf,
|
||||
buf_start_lba: 0,
|
||||
buf_len_sectors: 0,
|
||||
bytes_read_since_drop: 0,
|
||||
drop_window_start: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// True if `[lba, lba + count)` is wholly inside the current
|
||||
/// buffer window. `count == 0` is vacuously true.
|
||||
#[allow(dead_code)]
|
||||
fn buffer_covers(&self, lba: u32, count: u32) -> bool {
|
||||
if self.buf_len_sectors == 0 {
|
||||
return false;
|
||||
}
|
||||
let end = match lba.checked_add(count) {
|
||||
Some(e) => e,
|
||||
None => return false,
|
||||
};
|
||||
let buf_end = self.buf_start_lba.saturating_add(self.buf_len_sectors);
|
||||
lba >= self.buf_start_lba && end <= buf_end
|
||||
}
|
||||
|
||||
/// Refill the buffer so it starts at `lba`. Read as many sectors
|
||||
/// as we have buffer space AND file capacity for. Caller has
|
||||
/// already checked `lba < capacity`.
|
||||
#[allow(dead_code)]
|
||||
fn refill(&mut self, lba: u32) -> Result<()> {
|
||||
debug_assert!(lba < self.capacity, "refill past capacity");
|
||||
// Don't read past EOF — clamp the request to remaining
|
||||
// sectors. partial-buffer-at-EOF behaviour is intentional.
|
||||
let want = BUF_SECTORS.min(self.capacity - lba);
|
||||
let want_bytes = want as usize * SECTOR_SIZE;
|
||||
let offset = lba as u64 * SECTOR_SIZE as u64;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.read_exact(&mut self.buf[..want_bytes])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.buf_start_lba = lba;
|
||||
self.buf_len_sectors = want;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for FileSectorSource {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
out: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
let count = count as u32;
|
||||
let bytes = count as usize * SECTOR_SIZE;
|
||||
debug_assert!(
|
||||
out.len() >= bytes,
|
||||
"FileSectorSource::read_sectors: out len {} < requested {}",
|
||||
out.len(),
|
||||
bytes
|
||||
);
|
||||
if count == 0 {
|
||||
return Ok(0);
|
||||
}
|
||||
// 0.21.3: bypass the application-level buffer entirely.
|
||||
//
|
||||
// Empirically the 32 MiB readahead window (0.21.0–0.21.1) and the
|
||||
// 4 MiB shrink (0.21.2) both regressed mux throughput vs the
|
||||
// pre-Phase-1 0.20.7 baseline on NFS bidirectional workloads
|
||||
// (sweep ~25 MB/s OK; mux dropped from 18 → 7-8 → 5-6 MB/s).
|
||||
// Direct pread per call lets the kernel's own readahead policy
|
||||
// run, which interleaves naturally with concurrent NFS writes on
|
||||
// the same TCP connection.
|
||||
//
|
||||
// Buffer fields are retained (currently unused on this path) so
|
||||
// any future per-source policy can be reintroduced without
|
||||
// re-plumbing structure. `refill` / `buffer_covers` are kept too
|
||||
// (still exercised by the tests so the API contract is locked).
|
||||
let offset = lba as u64 * SECTOR_SIZE as u64;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.read_exact(&mut out[..bytes])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.buf_len_sectors = 0;
|
||||
|
||||
// 0.21.6: periodic page-cache eviction on the read side. Without
|
||||
// this, an 85 GB streaming ISO read pins the entire file in
|
||||
// kernel page cache, which starves concurrent NFS writes (the
|
||||
// MKV output) and collapses mux throughput. Mirrors the
|
||||
// write-side WritebackPipeline's DONTNEED policy.
|
||||
self.bytes_read_since_drop += bytes as u64;
|
||||
if self.bytes_read_since_drop >= READ_DROP_CHUNK_BYTES {
|
||||
let drop_start = self.drop_window_start;
|
||||
let drop_len = self.bytes_read_since_drop;
|
||||
platform::drop_window(&self.file, drop_start, drop_len);
|
||||
self.drop_window_start = drop_start + drop_len;
|
||||
self.bytes_read_since_drop = 0;
|
||||
}
|
||||
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
use tempfile::tempdir;
|
||||
|
||||
/// Build a deterministic ISO of `sectors` sectors where sector `n`
|
||||
/// is filled with the byte pattern `((n & 0xff) as u8)`. Lets us
|
||||
/// verify any sector by content alone.
|
||||
fn make_iso(path: &std::path::Path, sectors: u32) {
|
||||
let mut f = std::fs::File::create(path).unwrap();
|
||||
let mut chunk = vec![0u8; SECTOR_SIZE];
|
||||
for n in 0..sectors {
|
||||
let b = (n & 0xff) as u8;
|
||||
chunk.iter_mut().for_each(|c| *c = b);
|
||||
f.write_all(&chunk).unwrap();
|
||||
}
|
||||
f.flush().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sequential_reads_match_file() {
|
||||
// Two full buffer windows + a tail = exercise refill across
|
||||
// boundaries.
|
||||
let total = BUF_SECTORS * 2 + 17;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("seq.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), total);
|
||||
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
for lba in 0..total {
|
||||
src.read_sectors(lba, 1, &mut got, false).unwrap();
|
||||
let expected = (lba & 0xff) as u8;
|
||||
assert!(
|
||||
got.iter().all(|b| *b == expected),
|
||||
"sector {lba} content mismatch: expected 0x{expected:02x}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_sector_read_spanning_buffer_boundary() {
|
||||
// A read that lands exactly on the last sector of the buffer
|
||||
// plus the first sector of the next refill must rebuffer
|
||||
// mid-read. Bypass path triggers when count > BUF_SECTORS; we
|
||||
// want the in-window path, so count stays small but
|
||||
// straddles the boundary.
|
||||
let total = BUF_SECTORS * 2;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("span.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
|
||||
// Prime: read sector 0. (0.21.3+: app-level buffer is bypassed,
|
||||
// so we don't assert internal buf state here — just exercise
|
||||
// the read path.)
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
|
||||
// Now read 4 sectors crossing what used to be the buffer
|
||||
// boundary. Still a valid SectorSource-contract test.
|
||||
let span_lba = BUF_SECTORS - 2;
|
||||
let mut buf4 = vec![0u8; SECTOR_SIZE * 4];
|
||||
src.read_sectors(span_lba, 4, &mut buf4, false).unwrap();
|
||||
for i in 0..4 {
|
||||
let lba = span_lba + i as u32;
|
||||
let expected = (lba & 0xff) as u8;
|
||||
for b in &buf4[i * SECTOR_SIZE..(i + 1) * SECTOR_SIZE] {
|
||||
assert_eq!(*b, expected, "byte mismatch at sub-sector {i}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backward_seek_rebuffers() {
|
||||
// Read forward across two windows, then jump back to sector
|
||||
// 0. Buffer must refill from the start.
|
||||
let total = BUF_SECTORS * 2 + 5;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("back.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
|
||||
// Forward to the second window.
|
||||
src.read_sectors(BUF_SECTORS + 1, 1, &mut got, false)
|
||||
.unwrap();
|
||||
|
||||
// Backward to sector 0. (0.21.3+: app-level buffer is bypassed
|
||||
// so we only assert the byte-level contract, not internal
|
||||
// buffer state.)
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
assert!(got.iter().all(|b| *b == 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_buffer_at_eof() {
|
||||
// File is smaller than one buffer window. The buffer must
|
||||
// populate with only the available sectors and reads must
|
||||
// still succeed.
|
||||
let total: u32 = 100;
|
||||
assert!(total < BUF_SECTORS);
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("small.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
assert_eq!(src.capacity_sectors(), total);
|
||||
|
||||
let mut got = vec![0u8; SECTOR_SIZE];
|
||||
// First read at sector 0.
|
||||
src.read_sectors(0, 1, &mut got, false).unwrap();
|
||||
|
||||
// Read the very last sector. (0.21.3+: app-level buffer is
|
||||
// bypassed; the test still verifies that EOF-region reads
|
||||
// return correct bytes.)
|
||||
src.read_sectors(total - 1, 1, &mut got, false).unwrap();
|
||||
let expected = ((total - 1) & 0xff) as u8;
|
||||
assert!(got.iter().all(|b| *b == expected));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_read_bypasses_buffer() {
|
||||
// A request larger than the buffer must not deadlock the
|
||||
// refill (which only loads BUF_SECTORS at a time). Bypass
|
||||
// path handles it via direct pread.
|
||||
let total = BUF_SECTORS + 100;
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("over.iso");
|
||||
make_iso(&path, total);
|
||||
|
||||
let mut src = FileSectorSource::open(&path).unwrap();
|
||||
// Read more than BUF_SECTORS in one call. count is u16, so we
|
||||
// can't actually exceed BUF_SECTORS (16k) — but the path also
|
||||
// triggers via `out.len() / SECTOR_SIZE > BUF_SECTORS` check
|
||||
// implicitly because count > BUF_SECTORS. BUF_SECTORS for
|
||||
// 32 MiB is 16384, which does fit in u16 (max 65535). Cap
|
||||
// at BUF_SECTORS + 1 to exercise the bypass.
|
||||
let req = (BUF_SECTORS + 1) as u16;
|
||||
let req_bytes = req as usize * SECTOR_SIZE;
|
||||
let mut big = vec![0u8; req_bytes];
|
||||
src.read_sectors(0, req, &mut big, false).unwrap();
|
||||
// Spot-check sector 0 and the last requested sector.
|
||||
assert!(big[..SECTOR_SIZE].iter().all(|b| *b == 0));
|
||||
let last_lba = req as u32 - 1;
|
||||
let exp = (last_lba & 0xff) as u8;
|
||||
let last_off = (req as usize - 1) * SECTOR_SIZE;
|
||||
assert!(
|
||||
big[last_off..last_off + SECTOR_SIZE]
|
||||
.iter()
|
||||
.all(|b| *b == exp)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
//! Fallback for targets without a known sequential-readahead hint
|
||||
//! (BSDs, illumos, etc.). No-op — reads still work, they just don't
|
||||
//! get the OS-level prefetch widening.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn hint_sequential(_file: &File, _len_bytes: u64) {}
|
||||
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
@@ -0,0 +1,24 @@
|
||||
//! Windows: the canonical sequential-access hint is
|
||||
//! `FILE_FLAG_SEQUENTIAL_SCAN` passed to `CreateFile` at open time —
|
||||
//! it cannot be set after the fact via `SetFileInformationByHandle`.
|
||||
//! Routing the open call through this module would mean a custom
|
||||
//! `File::from_raw_handle` plumb for every `FileSectorSource::open`
|
||||
//! caller, which is more invasive than the Phase 1 scope.
|
||||
//!
|
||||
//! TODO: replumb `FileSectorSource::open` to take an
|
||||
//! `OpenOptions`-style builder so the Windows path can flip the flag
|
||||
//! at open time. For now this is a no-op stub.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn hint_sequential(_file: &File, _len_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"FileSectorSource hint_sequential: windows stub (TODO: FILE_FLAG_SEQUENTIAL_SCAN at open)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Windows page-cache eviction is not exposed via a posix_fadvise
|
||||
/// equivalent. The kernel does its own working-set management. No-op
|
||||
/// for now.
|
||||
pub(super) fn drop_window(_file: &File, _start: u64, _len: u64) {}
|
||||
@@ -10,10 +10,23 @@
|
||||
//! no-op pipeline — their default cache policies have not been shown
|
||||
//! to exhibit the same pathology for this access pattern.
|
||||
//!
|
||||
//! `FileSectorSource` is the read-side dual — it implements
|
||||
//! [`crate::sector::SectorSource`] for an ISO file with an internal
|
||||
//! 32 MiB read-ahead buffer that amortises NFS round-trip latency
|
||||
//! across thousands of sector reads.
|
||||
//!
|
||||
//! `Pipeline` + `Sink` (0.18) is the generic producer/consumer primitive
|
||||
//! used by sweep, patch, and mux to overlap reads with writes via a
|
||||
//! bounded channel + dedicated consumer thread.
|
||||
//!
|
||||
//! `byte_channel` is a byte-sized producer/consumer channel for the
|
||||
//! mux pipeline, sized to absorb worst-case input read stalls (see
|
||||
//! `(internal)/memory/project_buffering_architecture.md`).
|
||||
|
||||
pub(crate) mod bounded;
|
||||
pub mod byte_channel;
|
||||
pub mod file_sector_source;
|
||||
pub mod sink;
|
||||
mod writeback;
|
||||
mod writeback_file;
|
||||
|
||||
|
||||
+366
-4
@@ -34,10 +34,45 @@
|
||||
//! consumer lag detection). This is critical for diagnosing stalls.
|
||||
|
||||
use std::io;
|
||||
use std::sync::mpsc::{SyncSender, sync_channel};
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crossbeam_channel::{Sender, TrySendError, bounded};
|
||||
|
||||
use crate::error::Error;
|
||||
use crate::halt::Halt;
|
||||
|
||||
/// Deadline for [`Pipeline::finish_with_halt`]'s polling join. Chosen
|
||||
/// to be comfortably longer than the autorip hard watchdog
|
||||
/// (`HARD_WATCHDOG_STALL_SECS = 300s`) so the watchdog's `exit(1)`
|
||||
/// fires first when both are racing on the same wedged consumer.
|
||||
///
|
||||
/// 10 minutes is a backstop, not a normal timeout — the consumer is
|
||||
/// expected to drain in seconds. If we hit this, something is wedged
|
||||
/// inside a kernel call the consumer thread can't unwind from, and the
|
||||
/// caller has already lost the rip.
|
||||
pub const JOIN_TIMEOUT_SECS: u64 = 600;
|
||||
|
||||
/// Polling slice for the halt-aware send/finish loops. Mirrors the
|
||||
/// `bounded_syscall` cadence (250 ms) so halt observation feels equally
|
||||
/// responsive across both primitives.
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(250);
|
||||
|
||||
/// Halt-check cadence for the send loop. Producer blocks on
|
||||
/// [`crossbeam_channel::Sender::send_timeout`] for this slice — the
|
||||
/// kernel wakes it the instant the consumer drains a slot, so on the
|
||||
/// happy path there's no throughput cap from this primitive at all
|
||||
/// (the cap is whatever the underlying medium can sustain). When the
|
||||
/// consumer is genuinely wedged, the timeout fires every 250 ms and
|
||||
/// the producer checks the halt token; that's the latency a stop
|
||||
/// request will observe.
|
||||
///
|
||||
/// 0.21.7 replaced an old `std::sync::mpsc::sync_channel` + 50 ms
|
||||
/// `thread::sleep` polling loop that capped mux throughput at
|
||||
/// ~20 frames/sec ≈ 1 MB/s on saturated channels. See
|
||||
/// (internal)/memory/feedback_send_with_halt_poll_throttle.md
|
||||
/// for the multi-day diagnostic that surfaced it.
|
||||
const SEND_HALT_CHECK_INTERVAL: Duration = Duration::from_millis(250);
|
||||
|
||||
/// Check if verbose debug logging is enabled via FREEMKV_DEBUG env var.
|
||||
pub fn debug_enabled() -> bool {
|
||||
@@ -110,7 +145,7 @@ pub trait Sink<I>: Send + 'static {
|
||||
/// Bounded producer/consumer pipeline. Holds the producer-side
|
||||
/// channel and the consumer thread's join handle.
|
||||
pub struct Pipeline<I: Send + 'static, R: Send + 'static> {
|
||||
tx: SyncSender<I>,
|
||||
tx: Sender<I>,
|
||||
handle: JoinHandle<Result<R, Error>>,
|
||||
}
|
||||
|
||||
@@ -145,7 +180,7 @@ impl<I: Send + 'static, R: Send + 'static> Pipeline<I, R> {
|
||||
depth: usize,
|
||||
sink: S,
|
||||
) -> Result<Self, Error> {
|
||||
let (tx, rx) = sync_channel::<I>(depth);
|
||||
let (tx, rx) = bounded::<I>(depth);
|
||||
let handle = thread::Builder::new()
|
||||
.name(name.into())
|
||||
.spawn(move || -> Result<R, Error> {
|
||||
@@ -256,10 +291,84 @@ impl<I: Send + 'static, R: Send + 'static> Pipeline<I, R> {
|
||||
/// `Err`. Useful for best-effort signalling (e.g. sweep's
|
||||
/// throttled `StatsRequest`) where dropping the message is
|
||||
/// preferable to blocking the producer.
|
||||
pub fn try_send(&self, item: I) -> Result<(), std::sync::mpsc::TrySendError<I>> {
|
||||
pub fn try_send(&self, item: I) -> Result<(), TrySendError<I>> {
|
||||
self.tx.try_send(item)
|
||||
}
|
||||
|
||||
/// Halt-aware bounded variant of [`Pipeline::send`].
|
||||
///
|
||||
/// Uses [`crossbeam_channel::Sender::send_timeout`] so the producer
|
||||
/// thread BLOCKS on consumer drain (kernel-wakeup) rather than
|
||||
/// polling. The timeout slice is just the halt-observation cadence
|
||||
/// ([`SEND_HALT_CHECK_INTERVAL`]) — on the happy path the producer
|
||||
/// wakes the instant the consumer drains a slot, so there is no
|
||||
/// throughput cap from this primitive at any medium speed.
|
||||
///
|
||||
/// Returns:
|
||||
///
|
||||
/// - `Ok(())` once the item lands in the channel.
|
||||
/// - `Err(item)` if the consumer disconnected, the halt fired, or
|
||||
/// the deadline elapsed — the caller gets the item back so it
|
||||
/// can decide whether to drop it, route it elsewhere, or unwind.
|
||||
///
|
||||
/// Use this in producer threads that have a `Halt` token threaded
|
||||
/// through (mux, sweep, patch). Plain [`Pipeline::send`] is
|
||||
/// preserved for callers that don't (yet) plumb halt through.
|
||||
///
|
||||
/// Unlike [`Pipeline::send`], this never blocks the producer
|
||||
/// thread inside an unkillable `mpsc::send` — if the consumer is
|
||||
/// wedged inside an unkillable syscall, the producer can still
|
||||
/// observe `/api/stop` and unwind within
|
||||
/// [`SEND_HALT_CHECK_INTERVAL`].
|
||||
pub fn send_with_halt(&self, item: I, halt: &Halt, deadline: Duration) -> Result<(), I> {
|
||||
use crossbeam_channel::SendTimeoutError;
|
||||
let end = Instant::now() + deadline;
|
||||
let mut pending = item;
|
||||
loop {
|
||||
// Pre-check the cheap exit conditions before parking.
|
||||
if halt.is_cancelled() {
|
||||
if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline send_with_halt: halt observed, returning item={}",
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
return Err(pending);
|
||||
}
|
||||
let now = Instant::now();
|
||||
if now >= end {
|
||||
if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline send_with_halt: deadline elapsed, returning item={}",
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
return Err(pending);
|
||||
}
|
||||
// Wait for space-available or halt-check tick, whichever
|
||||
// is sooner. Crossbeam's send_timeout is kernel-wakeup
|
||||
// based: the consumer's recv on a saturated channel
|
||||
// signals this thread the moment a slot opens up.
|
||||
let slice = SEND_HALT_CHECK_INTERVAL.min(end.saturating_duration_since(now));
|
||||
match self.tx.send_timeout(pending, slice) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(SendTimeoutError::Timeout(returned)) => {
|
||||
pending = returned;
|
||||
// loop: re-check halt + deadline, then park again
|
||||
}
|
||||
Err(SendTimeoutError::Disconnected(returned)) => {
|
||||
if debug_enabled() {
|
||||
tracing::debug!(
|
||||
"Pipeline send_with_halt: consumer disconnected, item={}",
|
||||
std::any::type_name::<I>()
|
||||
);
|
||||
}
|
||||
return Err(returned);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the producer-side channel and wait for the consumer
|
||||
/// thread to finish. Returns whatever the consumer's `close()`
|
||||
/// produced, or the first `apply` error, or — on consumer panic —
|
||||
@@ -289,6 +398,69 @@ impl<I: Send + 'static, R: Send + 'static> Pipeline<I, R> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Halt-aware, deadline-bounded variant of [`Pipeline::finish`].
|
||||
///
|
||||
/// Drops the producer-side channel (same as `finish`) and then
|
||||
/// polls `JoinHandle::is_finished()` on a 250 ms cadence. Between
|
||||
/// slices, checks (1) the optional [`Halt`] token and (2) the
|
||||
/// [`JOIN_TIMEOUT_SECS`] deadline. Returns:
|
||||
///
|
||||
/// - `Ok(R)` on a clean consumer exit.
|
||||
/// - `Err(Error::IoError)` with one of three message prefixes for
|
||||
/// wedge cases:
|
||||
/// - `"pipeline join halted"` — halt fired while waiting.
|
||||
/// - `"pipeline join timed out"` — `JOIN_TIMEOUT_SECS` elapsed.
|
||||
/// - `"pipeline consumer panicked"` — same as `finish()`.
|
||||
///
|
||||
/// In the `halted` and `timed out` branches the consumer thread is
|
||||
/// intentionally leaked — exactly the same trade-off the
|
||||
/// `bounded_syscall` primitive makes. The wedged kernel call
|
||||
/// inside the consumer will unwind whenever it does, or at
|
||||
/// process exit. The caller is free to fall back to a degraded
|
||||
/// path (in autorip's case: `exit(1)` after the hard watchdog
|
||||
/// escalation, letting Docker restart the container).
|
||||
///
|
||||
/// Plain [`Pipeline::finish`] is preserved for callers without a
|
||||
/// halt-token plumbed through; that path still blocks indefinitely
|
||||
/// on `join()`, matching pre-0.20.8 behaviour.
|
||||
pub fn finish_with_halt(self, halt: Option<&Halt>) -> Result<R, Error> {
|
||||
let Pipeline { tx, handle } = self;
|
||||
drop(tx);
|
||||
let deadline = Instant::now() + Duration::from_secs(JOIN_TIMEOUT_SECS);
|
||||
loop {
|
||||
if handle.is_finished() {
|
||||
return match handle.join() {
|
||||
Ok(result) => result,
|
||||
Err(payload) => {
|
||||
let msg = payload
|
||||
.downcast_ref::<&'static str>()
|
||||
.copied()
|
||||
.or_else(|| payload.downcast_ref::<String>().map(|s| s.as_str()))
|
||||
.unwrap_or("(no message)");
|
||||
Err(Error::IoError {
|
||||
source: io::Error::other(format!("pipeline consumer panicked: {msg}")),
|
||||
})
|
||||
}
|
||||
};
|
||||
}
|
||||
if let Some(h) = halt {
|
||||
if h.is_cancelled() {
|
||||
// Consumer thread is intentionally leaked.
|
||||
return Err(Error::IoError {
|
||||
source: io::Error::other("pipeline join halted"),
|
||||
});
|
||||
}
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
// Consumer thread is intentionally leaked.
|
||||
return Err(Error::IoError {
|
||||
source: io::Error::other("pipeline join timed out"),
|
||||
});
|
||||
}
|
||||
thread::sleep(POLL_INTERVAL);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -553,4 +725,194 @@ mod tests {
|
||||
other => panic!("expected Err(IoError), got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Never-completing sink — `apply` blocks until cancelled. Signals
|
||||
/// `started` once it has consumed its first item so the test
|
||||
/// driver knows the consumer thread is wedged in `apply` (and
|
||||
/// will no longer drain the channel). Used to drive the
|
||||
/// halt/timeout paths of `send_with_halt` and `finish_with_halt`
|
||||
/// without depending on real I/O.
|
||||
struct NeverDrainsSink {
|
||||
cancel: Arc<std::sync::atomic::AtomicBool>,
|
||||
started: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
impl Sink<u64> for NeverDrainsSink {
|
||||
type Output = ();
|
||||
|
||||
fn apply(&mut self, _item: u64) -> Result<Flow, Error> {
|
||||
self.started.store(true, Ordering::SeqCst);
|
||||
while !self.cancel.load(Ordering::SeqCst) {
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
Ok(Flow::Continue)
|
||||
}
|
||||
|
||||
fn close(self) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Spin until `started` flips or `bail` elapses. Used by the
|
||||
/// send_with_halt tests to synchronise with the consumer thread
|
||||
/// before exercising the bounded-send timeout path.
|
||||
fn wait_for_started(started: &Arc<std::sync::atomic::AtomicBool>, bail: Duration) {
|
||||
let end = Instant::now() + bail;
|
||||
while !started.load(Ordering::SeqCst) {
|
||||
assert!(Instant::now() < end, "consumer never started apply()");
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_with_halt_returns_item_on_deadline() {
|
||||
// depth=1 + consumer wedged in apply on the first item, AND
|
||||
// the channel buffer already loaded with a second item, means
|
||||
// any further `try_send` sees Full; with a 200 ms deadline and
|
||||
// no halt fired, send_with_halt must return `Err(item)` within
|
||||
// roughly the deadline. Synchronising on `started` ensures the
|
||||
// consumer has actually started its wedged apply BEFORE we
|
||||
// load the channel-buffer slot — without that, the consumer
|
||||
// could still drain in a race window.
|
||||
let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipe = Pipeline::spawn(
|
||||
1,
|
||||
NeverDrainsSink {
|
||||
cancel: cancel.clone(),
|
||||
started: started.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
// First send: consumer recv()s it and wedges in apply.
|
||||
pipe.send(0u64).expect("first send hands off to consumer");
|
||||
wait_for_started(&started, Duration::from_secs(2));
|
||||
// Second send: lands in the depth=1 buffer slot, consumer
|
||||
// can't pick it up because it's wedged in apply. Channel now
|
||||
// full from the producer's perspective.
|
||||
pipe.send(1u64).expect("second send fills the buffer");
|
||||
|
||||
let halt = crate::halt::Halt::new();
|
||||
let start = Instant::now();
|
||||
let res = pipe.send_with_halt(99u64, &halt, Duration::from_millis(200));
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Release the leaked consumer so the test process winds down.
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
let _ = pipe.finish();
|
||||
|
||||
assert!(matches!(res, Err(99)), "expected item returned on deadline");
|
||||
assert!(
|
||||
elapsed >= Duration::from_millis(150),
|
||||
"deadline returned too early: {elapsed:?}"
|
||||
);
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"deadline blew past tolerance: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_with_halt_returns_item_on_halt() {
|
||||
// Same setup, but the halt fires before the deadline elapses.
|
||||
// The send loop must observe the halt within ~50 ms (the
|
||||
// SEND_POLL_INTERVAL) and return the item.
|
||||
let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipe = Pipeline::spawn(
|
||||
1,
|
||||
NeverDrainsSink {
|
||||
cancel: cancel.clone(),
|
||||
started: started.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
pipe.send(0u64).expect("first send hands off to consumer");
|
||||
wait_for_started(&started, Duration::from_secs(2));
|
||||
pipe.send(1u64).expect("second send fills the buffer");
|
||||
|
||||
let halt = crate::halt::Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
halt2.cancel();
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
let res = pipe.send_with_halt(7u64, &halt, Duration::from_secs(10));
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
let _ = pipe.finish();
|
||||
|
||||
assert!(matches!(res, Err(7)), "expected item returned on halt");
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"halt observation took too long: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_with_halt_returns_halted_when_consumer_wedged() {
|
||||
// Consumer wedges on the first apply; halt fires; finish
|
||||
// returns the documented "pipeline join halted" error rather
|
||||
// than blocking forever.
|
||||
let cancel = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let started = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let pipe = Pipeline::spawn(
|
||||
DEFAULT_PIPELINE_DEPTH,
|
||||
NeverDrainsSink {
|
||||
cancel: cancel.clone(),
|
||||
started: started.clone(),
|
||||
},
|
||||
)
|
||||
.expect("spawn should succeed");
|
||||
pipe.send(0u64).expect("seed item the consumer wedges on");
|
||||
wait_for_started(&started, Duration::from_secs(2));
|
||||
|
||||
let halt = crate::halt::Halt::new();
|
||||
let halt2 = halt.clone();
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(400));
|
||||
halt2.cancel();
|
||||
});
|
||||
|
||||
let start = Instant::now();
|
||||
let res = pipe.finish_with_halt(Some(&halt));
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Release the leaked consumer so the test process exits cleanly.
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
|
||||
match res {
|
||||
Err(Error::IoError { source }) => {
|
||||
assert!(
|
||||
source.to_string().contains("pipeline join halted"),
|
||||
"expected halt-prefix error, got: {source}"
|
||||
);
|
||||
}
|
||||
other => panic!("expected Err(IoError) halted, got {other:?}"),
|
||||
}
|
||||
// Bailed out within ~1 second of the halt firing (worst case
|
||||
// one POLL_INTERVAL = 250 ms of slack).
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"halt observation took too long: {elapsed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_with_halt_happy_path_returns_output() {
|
||||
// No halt token, sink completes normally — finish_with_halt
|
||||
// must return the same Output that `finish` would.
|
||||
let pipe = Pipeline::spawn(DEFAULT_PIPELINE_DEPTH, SumSink { total: 0 })
|
||||
.expect("spawn should succeed");
|
||||
for i in 0..10u64 {
|
||||
pipe.send(i).expect("send should succeed");
|
||||
}
|
||||
let total = pipe
|
||||
.finish_with_halt(None)
|
||||
.expect("happy-path finish_with_halt should succeed");
|
||||
assert_eq!(total, (0..10u64).sum::<u64>());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
//! `LocalFileSink` — `BufWriter<File>` for the common local-disk case.
|
||||
//!
|
||||
//! Buffering: 4 MiB internal `BufWriter`. Sized to coalesce the small
|
||||
//! per-PES writes that come out of the muxer into kernel-page-aligned
|
||||
//! flushes without making the buffer big enough to matter for memory
|
||||
//! pressure on a single concurrent rip.
|
||||
//!
|
||||
//! `Seek` flushes the underlying `BufWriter` first; otherwise a seek
|
||||
//! could leapfrog buffered data and silently corrupt the file. This is
|
||||
//! the same shape `BufWriter` itself uses when it impls `Seek` in
|
||||
//! stdlib, and is necessary for MKV's seek-back operations (cluster
|
||||
//! size patch, Cues index, segment header backpatch) to land on the
|
||||
//! right offset.
|
||||
//!
|
||||
//! `RandomAccessSink` is satisfied via the blanket impl in
|
||||
//! [`super::mod`]; no explicit impl needed here.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, BufWriter, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use super::preallocate;
|
||||
|
||||
const BUFFER_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Random-access write sink for local disks.
|
||||
///
|
||||
/// Wraps a `BufWriter<File>` with a 4 MiB internal buffer and forwards
|
||||
/// `Write`/`Seek` so any call site that previously held a `File` or
|
||||
/// `WritebackFile` can drop this in. `finish()` flushes the buffer and
|
||||
/// runs `sync_all` on the underlying file so the caller can drop it
|
||||
/// without losing data.
|
||||
///
|
||||
/// Construction always opens the file `create + truncate + read +
|
||||
/// write`. `read` is enabled so the same handle can be reused for a
|
||||
/// verification re-read after the mux (the existing
|
||||
/// `FileSectorSink::create` pattern). On Linux, [`with_size_hint`]
|
||||
/// additionally calls `fallocate(FALLOC_FL_KEEP_SIZE)` to pre-reserve
|
||||
/// extents.
|
||||
///
|
||||
/// [`with_size_hint`]: Self::with_size_hint
|
||||
pub struct LocalFileSink {
|
||||
inner: BufWriter<File>,
|
||||
}
|
||||
|
||||
impl LocalFileSink {
|
||||
/// Open `path` for writing, truncating any existing contents.
|
||||
pub fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
Ok(Self {
|
||||
inner: BufWriter::with_capacity(BUFFER_BYTES, file),
|
||||
})
|
||||
}
|
||||
|
||||
/// Like [`Self::create`] but additionally calls the per-OS
|
||||
/// preallocate path with `size_bytes`. On Linux this is
|
||||
/// `fallocate(FALLOC_FL_KEEP_SIZE)` so the on-disk extents are
|
||||
/// reserved up front (reducing fragmentation for big sequential
|
||||
/// muxer output); on other OSes it is a no-op today. Failures
|
||||
/// from the preallocate call are non-fatal — the file is still
|
||||
/// returned, just without the size reservation.
|
||||
pub fn with_size_hint(path: &Path, size_bytes: u64) -> io::Result<Self> {
|
||||
let file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(path)?;
|
||||
preallocate::preallocate(&file, size_bytes);
|
||||
Ok(Self {
|
||||
inner: BufWriter::with_capacity(BUFFER_BYTES, file),
|
||||
})
|
||||
}
|
||||
|
||||
/// Drain the internal buffer and `fsync` the underlying file.
|
||||
/// Idempotent with `Drop` (the `BufWriter` also flushes on drop;
|
||||
/// this call additionally surfaces fsync errors to the caller).
|
||||
#[allow(dead_code)] // exposed for parity with WritebackFile::sync_all
|
||||
pub fn sync_all(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()?;
|
||||
self.inner.get_ref().sync_all()
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for LocalFileSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.inner.write(buf)
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.inner.write_all(buf)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for LocalFileSink {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
// Flush before seeking so buffered bytes land at the offset
|
||||
// they were written for, not the new one.
|
||||
self.inner.flush()?;
|
||||
self.inner.get_mut().seek(from)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
|
||||
#[test]
|
||||
fn write_seek_roundtrip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("rt.bin");
|
||||
let mut s = LocalFileSink::create(&p).unwrap();
|
||||
s.write_all(b"AAAA").unwrap();
|
||||
s.write_all(b"BBBB").unwrap();
|
||||
// Seek back over the second word and overwrite.
|
||||
s.seek(SeekFrom::Start(4)).unwrap();
|
||||
s.write_all(b"CCCC").unwrap();
|
||||
s.sync_all().unwrap();
|
||||
drop(s);
|
||||
|
||||
let mut f = File::open(&p).unwrap();
|
||||
let mut got = Vec::new();
|
||||
f.read_to_end(&mut got).unwrap();
|
||||
assert_eq!(&got[..], b"AAAACCCC");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_flushes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("drop.bin");
|
||||
{
|
||||
let mut s = LocalFileSink::create(&p).unwrap();
|
||||
s.write_all(b"buffered").unwrap();
|
||||
// No explicit flush / sync_all — BufWriter drop runs the
|
||||
// flush and the file should land on disk.
|
||||
}
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..], b"buffered");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_size_hint_creates_writable_file() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("sz.bin");
|
||||
let mut s = LocalFileSink::with_size_hint(&p, 64 * 1024).unwrap();
|
||||
s.write_all(b"hint-ok").unwrap();
|
||||
s.sync_all().unwrap();
|
||||
drop(s);
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..], b"hint-ok");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//! Output-sink trait split for the buffering architecture.
|
||||
//!
|
||||
//! Two traits, one for each capability axis of an output destination:
|
||||
//!
|
||||
//! - [`SequentialSink`] — anything you can `Write` to in order. Sockets,
|
||||
//! pipes, append-only stores, plain files. Containers that don't need
|
||||
//! seek (M2TS, fMP4, HEVC elementary) target this.
|
||||
//! - [`RandomAccessSink`] — everything `SequentialSink` plus a working
|
||||
//! `Seek`. Local files, NFS files, anything with random-write
|
||||
//! semantics. Containers that need backpatch (MKV cluster sizes, Cues
|
||||
//! index, MP4 moov-at-end) target this.
|
||||
//!
|
||||
//! `RandomAccessSink: SequentialSink` — every random-access sink is
|
||||
//! also a valid sequential sink. The muxer is generic over which it
|
||||
//! requires (`MkvMux<S: RandomAccessSink>`, `M2tsMux<S: SequentialSink>`)
|
||||
//! so an attempt to mux MKV to a network socket is a compile error.
|
||||
//!
|
||||
//! Buffering policy belongs to the concrete sink, not to a wrapper at
|
||||
//! the call site. `LocalFileSink` wraps a `BufWriter<File>` with a
|
||||
//! 4 MiB buffer for the common local-disk case; `WritebackFile`
|
||||
//! (separate module) wraps a `File` with the adaptive-chunk
|
||||
//! `sync_file_range` machinery for the Linux+NFS case.
|
||||
//!
|
||||
//! See `(internal)/memory/project_buffering_architecture.md` for
|
||||
//! the full design and the source/sink matrix.
|
||||
|
||||
use std::io::{Seek, Write};
|
||||
|
||||
mod local_file;
|
||||
mod preallocate;
|
||||
mod socket;
|
||||
|
||||
pub use local_file::LocalFileSink;
|
||||
pub use socket::{SocketSink, UdpSocketSink};
|
||||
|
||||
/// Sequential-only write destination. Sockets, pipes, append-only
|
||||
/// stores. No seek. Implementations own their write buffering — the
|
||||
/// trait does not impose or hide any buffering of its own.
|
||||
///
|
||||
/// `finish` drains any internal buffering and signals end-of-stream to
|
||||
/// the underlying transport (close-write on a socket, flush on a
|
||||
/// buffered writer, etc.). The default impl is a no-op; concrete
|
||||
/// implementations that need explicit shutdown can override it but the
|
||||
/// blanket impl below keeps it optional for adapter types like
|
||||
/// `&mut File`.
|
||||
pub trait SequentialSink: Write + Send {
|
||||
fn finish(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Random-access write destination. Local files, NFS files, anything
|
||||
/// with a working `Seek`. Inherits the `SequentialSink` contract — a
|
||||
/// random-access sink is always usable as a sequential sink.
|
||||
pub trait RandomAccessSink: SequentialSink + Seek {}
|
||||
|
||||
// Blanket impls so any `Write + Send` type acts as a `SequentialSink`
|
||||
// (with default `finish`), and any sink that also impls `Seek` is
|
||||
// automatically a `RandomAccessSink`. Keeps call-site ergonomics simple
|
||||
// — `&mut File`, `LocalFileSink`, `WritebackFile`, `BufWriter<File>`,
|
||||
// and `Cursor<Vec<u8>>` all satisfy the right trait without per-type
|
||||
// boilerplate.
|
||||
impl<T: Write + Send + ?Sized> SequentialSink for T {}
|
||||
impl<T: SequentialSink + Seek + ?Sized> RandomAccessSink for T {}
|
||||
|
||||
/// Pick the right `RandomAccessSink` impl for `dest` based on its
|
||||
/// filesystem type.
|
||||
///
|
||||
/// - Linux + NFS path → `WritebackFile` with its adaptive-chunk
|
||||
/// sync_file_range machinery and (when supported) `fallocate` size
|
||||
/// hint.
|
||||
/// - everything else → [`LocalFileSink`] over `BufWriter<File>`. On
|
||||
/// non-Linux there is no `WritebackFile` machinery to opt into, and
|
||||
/// on local Linux the kernel's default writeback policy is already
|
||||
/// fine.
|
||||
///
|
||||
/// `size_hint`, when present, is forwarded to the per-OS preallocate
|
||||
/// path (`fallocate(KEEP_SIZE)` on Linux, `F_PREALLOCATE` on macOS when
|
||||
/// implemented, no-op elsewhere).
|
||||
///
|
||||
/// Returns a boxed trait object so the call site (mux construction)
|
||||
/// stays agnostic of which concrete sink got picked.
|
||||
#[allow(dead_code)] // wiring to mux::resolve is a follow-up commit
|
||||
pub fn open_for_mkv(
|
||||
dest: &std::path::Path,
|
||||
size_hint: Option<u64>,
|
||||
) -> std::io::Result<Box<dyn RandomAccessSink>> {
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
use crate::platform::fs_type::detect;
|
||||
#[cfg(target_os = "linux")]
|
||||
use crate::platform::fs_type::{FsType, detect};
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
if detect(dest) == FsType::Nfs {
|
||||
let wf = match size_hint {
|
||||
Some(n) => crate::io::WritebackFile::create_with_size_hint(dest, n)?,
|
||||
None => crate::io::WritebackFile::create(dest)?,
|
||||
};
|
||||
return Ok(Box::new(wf));
|
||||
}
|
||||
}
|
||||
// Silence the unused-binding warning on non-Linux where the only
|
||||
// branch above is cfg-gated out.
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
let _ = detect(dest);
|
||||
}
|
||||
|
||||
let sink = match size_hint {
|
||||
Some(n) => LocalFileSink::with_size_hint(dest, n)?,
|
||||
None => LocalFileSink::create(dest)?,
|
||||
};
|
||||
Ok(Box::new(sink))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs::File;
|
||||
|
||||
// Type-level assertion: the blanket impls cover the shapes we care
|
||||
// about. These functions never run; they just have to type-check.
|
||||
fn _assert_file_is_sequential(_: &mut dyn SequentialSink) {}
|
||||
fn _assert_file_is_random_access(_: &mut dyn RandomAccessSink) {}
|
||||
|
||||
#[test]
|
||||
fn blanket_impls_cover_file_and_localfilesink() {
|
||||
// `File` directly via blanket impls.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut f = File::create(dir.path().join("a.bin")).unwrap();
|
||||
_assert_file_is_sequential(&mut f);
|
||||
_assert_file_is_random_access(&mut f);
|
||||
|
||||
// `LocalFileSink` ditto.
|
||||
let mut s = LocalFileSink::create(&dir.path().join("b.bin")).unwrap();
|
||||
_assert_file_is_sequential(&mut s);
|
||||
_assert_file_is_random_access(&mut s);
|
||||
|
||||
// `WritebackFile` — confirms the Phase 1 type still satisfies
|
||||
// the trait via the blanket impl without needing an explicit
|
||||
// `impl RandomAccessSink for WritebackFile {}`.
|
||||
let mut wf = crate::io::WritebackFile::create(&dir.path().join("c.bin")).unwrap();
|
||||
_assert_file_is_sequential(&mut wf);
|
||||
_assert_file_is_random_access(&mut wf);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_for_mkv_returns_a_random_access_sink() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("c.bin");
|
||||
let mut sink = open_for_mkv(&p, Some(64 * 1024)).unwrap();
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
sink.write_all(b"hello").unwrap();
|
||||
sink.seek(SeekFrom::Start(0)).unwrap();
|
||||
sink.finish().unwrap();
|
||||
drop(sink);
|
||||
let bytes = std::fs::read(&p).unwrap();
|
||||
assert_eq!(&bytes[..5], b"hello");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
//! Linux `fallocate(FALLOC_FL_KEEP_SIZE)` preallocation.
|
||||
//!
|
||||
//! `KEEP_SIZE` reserves extents without changing the apparent file
|
||||
//! length, which matches the muxer's expectation that writes still grow
|
||||
//! the file naturally.
|
||||
|
||||
use std::fs::File;
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
pub(super) fn preallocate_impl(file: &File, size_bytes: u64) {
|
||||
let fd = file.as_raw_fd();
|
||||
// FALLOC_FL_KEEP_SIZE = 0x01.
|
||||
let rc = unsafe { libc::fallocate(fd, libc::FALLOC_FL_KEEP_SIZE, 0, size_bytes as i64) };
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink fallocate size_hint={size_bytes} rc={rc} ok={}",
|
||||
rc == 0
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
//! macOS `F_PREALLOCATE` extent reservation.
|
||||
//!
|
||||
//! `fcntl(F_PREALLOCATE)` with `F_ALLOCATECONTIG` first (try for a
|
||||
//! contiguous run) and fall back to `F_ALLOCATEALL` (non-contig OK).
|
||||
//! Reported file size is unchanged — the muxer's writes still grow it.
|
||||
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
|
||||
// Mirror the Darwin `fstore_t` struct from `<sys/fcntl.h>`. libc on
|
||||
// some Rust toolchains/versions doesn't ship this binding, so define
|
||||
// it locally with the layout the kernel ABI requires.
|
||||
#[repr(C)]
|
||||
struct Fstore {
|
||||
fst_flags: libc::c_uint,
|
||||
fst_posmode: libc::c_int,
|
||||
fst_offset: libc::off_t,
|
||||
fst_length: libc::off_t,
|
||||
fst_bytesalloc: libc::off_t,
|
||||
}
|
||||
|
||||
// Constants from <sys/fcntl.h>.
|
||||
const F_PREALLOCATE: libc::c_int = 42;
|
||||
const F_ALLOCATECONTIG: libc::c_uint = 0x0000_0002;
|
||||
const F_ALLOCATEALL: libc::c_uint = 0x0000_0004;
|
||||
const F_PEOFPOSMODE: libc::c_int = 3;
|
||||
|
||||
pub(super) fn preallocate_impl(file: &File, size_bytes: u64) {
|
||||
let fd = file.as_raw_fd();
|
||||
let mut store = Fstore {
|
||||
fst_flags: F_ALLOCATECONTIG,
|
||||
fst_posmode: F_PEOFPOSMODE,
|
||||
fst_offset: 0,
|
||||
fst_length: size_bytes as libc::off_t,
|
||||
fst_bytesalloc: 0,
|
||||
};
|
||||
let mut rc = unsafe { libc::fcntl(fd, F_PREALLOCATE, &mut store as *mut Fstore) };
|
||||
if rc == -1 {
|
||||
// Fall back to non-contiguous.
|
||||
store.fst_flags = F_ALLOCATEALL;
|
||||
rc = unsafe { libc::fcntl(fd, F_PREALLOCATE, &mut store as *mut Fstore) };
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink F_PREALLOCATE size_hint={size_bytes} rc={rc} bytesalloc={}",
|
||||
store.fst_bytesalloc
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! Per-OS extent preallocation. Best-effort; failures are logged at
|
||||
//! debug and otherwise swallowed because the file is still usable
|
||||
//! without the size reservation — only large-file fragmentation gets
|
||||
//! marginally worse.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
mod other;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux::preallocate_impl;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos::preallocate_impl;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
use other::preallocate_impl;
|
||||
|
||||
/// Reserve `size_bytes` of disk space for `file`'s on-disk extents.
|
||||
/// Reported file size is unchanged — writes still grow the file
|
||||
/// naturally; only the allocator's extent map is primed.
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
preallocate_impl(file, size_bytes);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
//! Fallback preallocate impl. No-op.
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
pub(super) fn preallocate_impl(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"LocalFileSink preallocate size_hint={size_bytes} skipped (no platform impl)"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
//! TCP / UDP socket sinks (sequential-only).
|
||||
//!
|
||||
//! [`SocketSink`] wraps a `TcpStream` in a 1 MiB `BufWriter`. Constructor
|
||||
//! tunes `SO_SNDBUF` to a caller hint when provided. `finish()` flushes
|
||||
//! the buffer then `shutdown(Write)`s the socket so the peer sees clean
|
||||
//! end-of-stream.
|
||||
//!
|
||||
//! [`UdpSocketSink`] wraps a connected `UdpSocket`. Each `write` call
|
||||
//! emits exactly one datagram — the caller is responsible for packetizing
|
||||
//! to a reasonable MTU (188 × 7 = 1316 bytes for MPEG-TS-over-UDP is the
|
||||
//! conventional choice). `finish()` is a no-op; UDP has no end-of-stream
|
||||
//! marker.
|
||||
//!
|
||||
//! Both types satisfy [`SequentialSink`] via the blanket impl in
|
||||
//! `super::mod`. Neither implements `Seek`, so neither satisfies
|
||||
//! [`RandomAccessSink`] — using one with `MkvMux` is a compile error,
|
||||
//! which is the design intent.
|
||||
//!
|
||||
//! [`SequentialSink`]: super::SequentialSink
|
||||
//! [`RandomAccessSink`]: super::RandomAccessSink
|
||||
|
||||
use std::io::{self, BufWriter, Write};
|
||||
use std::net::{Shutdown, TcpStream, ToSocketAddrs, UdpSocket};
|
||||
|
||||
/// `BufWriter` capacity for [`SocketSink`]. 1 MiB matches the typical
|
||||
/// kernel send-buffer ceiling and keeps small-write amplification from
|
||||
/// containers (TS = 188-byte packets, fMP4 fragment headers = ~100 bytes)
|
||||
/// from translating into syscall storms.
|
||||
const TCP_BUF_CAPACITY: usize = 1024 * 1024;
|
||||
|
||||
/// Sequential-only sink over a TCP connection.
|
||||
///
|
||||
/// Wraps a `BufWriter<TcpStream>`; the inner `TcpStream` is kept as a
|
||||
/// clone so [`finish`](Self::finish) can call `shutdown(Write)` after
|
||||
/// flushing the buffer (the buffered writer doesn't expose the socket
|
||||
/// directly).
|
||||
pub struct SocketSink {
|
||||
/// Buffered write half. All payload bytes go through this.
|
||||
buf: BufWriter<TcpStream>,
|
||||
/// Shutdown handle — clone of the socket inside `buf`. Used only by
|
||||
/// `finish()` for `shutdown(Write)`; never read or written through.
|
||||
shutdown_handle: TcpStream,
|
||||
}
|
||||
|
||||
impl SocketSink {
|
||||
/// Open a TCP connection to `addr` and wrap it for sequential
|
||||
/// writing. `sndbuf_bytes`, when present, is forwarded to
|
||||
/// `setsockopt(SO_SNDBUF)` as a kernel hint — the OS may clamp it.
|
||||
///
|
||||
/// `addr` accepts anything `ToSocketAddrs` does: `"10.0.0.1:1234"`,
|
||||
/// `("host", 1234)`, a `SocketAddr`, etc.
|
||||
pub fn connect<A: ToSocketAddrs>(addr: A, sndbuf_bytes: Option<usize>) -> io::Result<Self> {
|
||||
let stream = TcpStream::connect(addr)?;
|
||||
// `set_nodelay(true)` keeps small writes (TS packet trains, fMP4
|
||||
// moof headers) from sitting in Nagle's algorithm until the buffer
|
||||
// fills. The BufWriter already absorbs syscall overhead; Nagle
|
||||
// would just add latency without coalescing more.
|
||||
stream.set_nodelay(true)?;
|
||||
if let Some(n) = sndbuf_bytes {
|
||||
set_send_buffer(&stream, n)?;
|
||||
}
|
||||
let shutdown_handle = stream.try_clone()?;
|
||||
Ok(Self {
|
||||
buf: BufWriter::with_capacity(TCP_BUF_CAPACITY, stream),
|
||||
shutdown_handle,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for SocketSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.buf.write(buf)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.buf.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl SocketSink {
|
||||
/// Drain the BufWriter and `shutdown(Write)` the underlying socket
|
||||
/// so the peer sees a clean EOF.
|
||||
///
|
||||
/// Note: [`SequentialSink::finish`](super::SequentialSink::finish)'s
|
||||
/// blanket-impl default is a no-op. Trait-object call sites that
|
||||
/// need socket shutdown should call this inherent method directly
|
||||
/// before dropping the sink, or hold the concrete `SocketSink` type
|
||||
/// (typical pattern: each muxer's `finish()` calls the appropriate
|
||||
/// inherent close method on its captured concrete sink).
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
self.buf.flush()?;
|
||||
// `shutdown(Write)` signals clean EOF to the peer. Errors here
|
||||
// are non-fatal — the connection may have already been torn down
|
||||
// by the peer — but we surface them so callers can log.
|
||||
self.shutdown_handle.shutdown(Shutdown::Write)
|
||||
}
|
||||
}
|
||||
|
||||
/// Sequential-only sink over a connected UDP socket.
|
||||
///
|
||||
/// Each [`write`](Write::write) call sends exactly one datagram. The
|
||||
/// caller is responsible for splitting payload at packet boundaries —
|
||||
/// for MPEG-TS this means 7 × 188 = 1316 bytes per datagram, the
|
||||
/// industry standard for MPEG-TS-over-UDP. No buffering happens here;
|
||||
/// adding it would silently merge datagrams.
|
||||
///
|
||||
/// `finish()` is a no-op: UDP has no end-of-stream marker. Closing the
|
||||
/// socket happens on drop.
|
||||
pub struct UdpSocketSink {
|
||||
socket: UdpSocket,
|
||||
}
|
||||
|
||||
impl UdpSocketSink {
|
||||
/// Bind a local UDP socket to an ephemeral port and `connect` it to
|
||||
/// `peer`. `connect` doesn't open a connection — it just fixes the
|
||||
/// peer address so subsequent `send` calls don't need to repeat it,
|
||||
/// and so receive-side filtering rejects packets from other sources.
|
||||
///
|
||||
/// `sndbuf_bytes`, when present, is a hint to `SO_SNDBUF`.
|
||||
pub fn connect<A: ToSocketAddrs>(peer: A, sndbuf_bytes: Option<usize>) -> io::Result<Self> {
|
||||
// Bind to all-zeros / any port. The kernel picks an ephemeral
|
||||
// source port and the source IP at first send.
|
||||
let socket = UdpSocket::bind("0.0.0.0:0")?;
|
||||
socket.connect(peer)?;
|
||||
if let Some(n) = sndbuf_bytes {
|
||||
set_udp_send_buffer(&socket, n)?;
|
||||
}
|
||||
Ok(Self { socket })
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for UdpSocketSink {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
// `send` writes the entire datagram or fails — no partial sends
|
||||
// for UDP. Match `Write::write`'s contract by reporting bytes
|
||||
// accepted.
|
||||
self.socket.send(buf)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl UdpSocketSink {
|
||||
/// No-op — UDP has no end-of-stream marker. Provided for parity
|
||||
/// with [`SocketSink::finish`] so call sites can treat them uniformly.
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ── Platform `SO_SNDBUF` tuning ────────────────────────────────────────────
|
||||
//
|
||||
// std's `TcpStream` / `UdpSocket` don't expose `SO_SNDBUF`. We drop to
|
||||
// libc on Linux + macOS (the libc-dep targets in Cargo.toml). On other
|
||||
// targets the hint is silently ignored — the socket still works, the
|
||||
// kernel just picks its own send-buffer size.
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn set_send_buffer(stream: &TcpStream, bytes: usize) -> io::Result<()> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
setsockopt_sndbuf(stream.as_raw_fd(), bytes)
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn set_udp_send_buffer(socket: &UdpSocket, bytes: usize) -> io::Result<()> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
setsockopt_sndbuf(socket.as_raw_fd(), bytes)
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
fn set_send_buffer(_stream: &TcpStream, _bytes: usize) -> io::Result<()> {
|
||||
// Non-Linux-non-macOS targets aren't in Cargo.toml's libc dep list;
|
||||
// silently ignore the hint rather than failing the connect. Callers
|
||||
// can detect via the lack of an explicit "sndbuf applied" signal
|
||||
// (not provided, intentionally — this is a hint, not a guarantee).
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
fn set_udp_send_buffer(_socket: &UdpSocket, _bytes: usize) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
fn setsockopt_sndbuf(fd: std::os::unix::io::RawFd, bytes: usize) -> io::Result<()> {
|
||||
// Clamp into c_int range; SO_SNDBUF takes an `int` argument.
|
||||
let want: libc::c_int = bytes.try_into().unwrap_or(libc::c_int::MAX);
|
||||
let ret = unsafe {
|
||||
libc::setsockopt(
|
||||
fd,
|
||||
libc::SOL_SOCKET,
|
||||
libc::SO_SNDBUF,
|
||||
&want as *const _ as *const libc::c_void,
|
||||
std::mem::size_of::<libc::c_int>() as libc::socklen_t,
|
||||
)
|
||||
};
|
||||
if ret != 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
use std::net::{TcpListener, UdpSocket};
|
||||
use std::thread;
|
||||
|
||||
/// Bind a listener, accept on a thread, return (listener_addr,
|
||||
/// accepted-bytes future via JoinHandle).
|
||||
#[test]
|
||||
fn socket_sink_round_trips_bytes() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let accept = thread::spawn(move || {
|
||||
let (mut sock, _) = listener.accept().unwrap();
|
||||
let mut buf = Vec::new();
|
||||
sock.read_to_end(&mut buf).unwrap();
|
||||
buf
|
||||
});
|
||||
|
||||
let mut sink = SocketSink::connect(addr, Some(256 * 1024)).unwrap();
|
||||
// Write enough to overflow the BufWriter at least once, then a
|
||||
// tail that lives in the buffer until `finish` flushes.
|
||||
let big: Vec<u8> = (0..(2 * TCP_BUF_CAPACITY))
|
||||
.map(|i| (i & 0xff) as u8)
|
||||
.collect();
|
||||
sink.write_all(&big).unwrap();
|
||||
sink.write_all(b"tail\n").unwrap();
|
||||
sink.finish().unwrap();
|
||||
drop(sink);
|
||||
|
||||
let received = accept.join().unwrap();
|
||||
assert_eq!(received.len(), big.len() + 5);
|
||||
assert_eq!(&received[..big.len()], &big[..]);
|
||||
assert_eq!(&received[big.len()..], b"tail\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn socket_sink_is_sequential_only() {
|
||||
// Compile-time assertion via dyn — if this ever started
|
||||
// satisfying `RandomAccessSink`, the trait split would be broken.
|
||||
fn _assert_seq(_: &mut dyn super::super::SequentialSink) {}
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let _accept = thread::spawn(move || {
|
||||
let _ = listener.accept();
|
||||
});
|
||||
let mut sink = SocketSink::connect(addr, None).unwrap();
|
||||
_assert_seq(&mut sink);
|
||||
// The negative is harder to assert directly (no `is_not<T>`),
|
||||
// but `SocketSink` does not impl `Seek`, so it can't unify with
|
||||
// `RandomAccessSink`'s super-bound. The Phase 2 blanket impl
|
||||
// `impl<T: SequentialSink + Seek> RandomAccessSink for T {}` thus
|
||||
// excludes it by construction.
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn udp_socket_sink_delivers_datagrams() {
|
||||
let receiver = UdpSocket::bind("127.0.0.1:0").unwrap();
|
||||
receiver
|
||||
.set_read_timeout(Some(std::time::Duration::from_secs(2)))
|
||||
.unwrap();
|
||||
let addr = receiver.local_addr().unwrap();
|
||||
let mut sink = UdpSocketSink::connect(addr, Some(128 * 1024)).unwrap();
|
||||
|
||||
sink.write_all(&[1, 2, 3, 4, 5]).unwrap();
|
||||
sink.write_all(&[9, 9, 9]).unwrap();
|
||||
sink.finish().unwrap();
|
||||
|
||||
let mut buf = [0u8; 64];
|
||||
let n1 = receiver.recv(&mut buf).unwrap();
|
||||
assert_eq!(&buf[..n1], &[1, 2, 3, 4, 5]);
|
||||
let n2 = receiver.recv(&mut buf).unwrap();
|
||||
assert_eq!(&buf[..n2], &[9, 9, 9]);
|
||||
}
|
||||
}
|
||||
+316
-31
@@ -14,9 +14,77 @@
|
||||
//! a full chunk's worth of work to flush — the wait is near-instant.
|
||||
//! Dirty cache stays bounded at ~2 × `chunk_bytes` and writes drain
|
||||
//! continuously instead of in bursts.
|
||||
//!
|
||||
//! The chunk size is adaptive: we measure the elapsed time of the
|
||||
//! `WAIT_AFTER` call over a rolling window of the last 16 chunks and
|
||||
//! resize the chunk based on the p95. Slow storage (NFS, network
|
||||
//! shares, HDD) sees larger chunks to amortise per-chunk overhead;
|
||||
//! fast storage (NVMe) sees smaller chunks to keep cache pressure
|
||||
//! tight. Bounds: [4 MiB, 256 MiB].
|
||||
//!
|
||||
//! ## Medium-agnostic by construction
|
||||
//!
|
||||
//! `sync_file_range(WAIT_AFTER)` runs on every medium (local FS, NFS,
|
||||
//! etc.). It bounds the dirty-page set at ~2 × `chunk_bytes` by
|
||||
//! waiting for each chunk's kernel writeback to complete before
|
||||
//! advancing. There is no `if is_nfs` branch in the hot path. The
|
||||
//! safety net described below ([`WAIT_AFTER_TIMEOUT`]) handles wedged
|
||||
//! filesystems generically: if a particular FS proves unable to ack
|
||||
//! `WAIT_AFTER` within the deadline, the pipeline flips to `degraded`
|
||||
//! and skips for the rest of its life. That covers the original
|
||||
//! NFS-hang concern (network partition, server stuck on commit)
|
||||
//! without baking the failure mode into a per-FS branch.
|
||||
//!
|
||||
//! **No `posix_fadvise(DONTNEED)` after the wait.** Earlier revisions
|
||||
//! evicted the just-WAIT_AFTER'd range from the page cache on the
|
||||
//! theory that bounding *dirty* pages required also bounding *clean*
|
||||
//! pages. That was unnecessary and counterproductive: clean cached
|
||||
//! pages cost nothing (the kernel reclaims them under LRU when memory
|
||||
//! is needed) and DONTNEED forces a read-modify-write on any
|
||||
//! subsequent in-window write — exactly the pattern matroska
|
||||
//! cluster-size backpatches produce, costing ~40-50% of measured NFS
|
||||
//! write bandwidth in empirical tests 2026-05-15/16.
|
||||
//!
|
||||
//! Medium detection happens *exactly once*, at construction, and
|
||||
//! produces *exactly one* value: the initial `chunk_bytes` seed for
|
||||
//! the autotuner. The detector's job is to give the autotuner a
|
||||
//! decent starting point on cold start; from there p95 of WAIT_AFTER
|
||||
//! latency drives all subsequent decisions, identically on every
|
||||
//! medium.
|
||||
//!
|
||||
//! ## Defence in depth: WAIT_AFTER timeout
|
||||
//!
|
||||
//! A degraded disk, wedged NFS server, or odd FS driver could in
|
||||
//! principle hang inside WAIT_AFTER. Each call runs on a worker
|
||||
//! thread with a 30s recv_timeout on its result channel
|
||||
//! ([`crate::io::bounded::bounded_syscall`]). On timeout the pipeline
|
||||
//! flips to `degraded`, logs a loud error, and skips WAIT_AFTER for
|
||||
//! the rest of its life. The worker thread is intentionally leaked —
|
||||
//! it unwinds whenever the syscall eventually returns or the process
|
||||
//! exits. The mux continues; the original dirty-burst pathology
|
||||
//! re-emerges only on the specific (medium, server, FS) combination
|
||||
//! that actually wedged.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::fs::File;
|
||||
use std::os::unix::io::{AsRawFd, RawFd};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const ADAPTIVE_WINDOW: usize = 16;
|
||||
const CHUNK_BYTES_MIN: u64 = 4 * 1024 * 1024;
|
||||
const CHUNK_BYTES_MAX: u64 = 256 * 1024 * 1024;
|
||||
const ADAPTIVE_GROW_MS: u64 = 200;
|
||||
const ADAPTIVE_SHRINK_MS: u64 = 20;
|
||||
/// Every N chunks, emit a `debug!` snapshot of the current chunk
|
||||
/// size so operators tailing the log can see where the autoscaler
|
||||
/// settled.
|
||||
const SIZE_LOG_INTERVAL: u64 = 32;
|
||||
/// Hard upper bound on a single `sync_file_range(WAIT_AFTER)` call.
|
||||
/// Beyond this we declare the pipeline degraded and stop calling
|
||||
/// WAIT_AFTER for the rest of its life.
|
||||
const WAIT_AFTER_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
pub(crate) struct WritebackPipeline {
|
||||
/// Aliases the wrapping `WritebackFile::file`. Only valid for the
|
||||
@@ -28,6 +96,23 @@ pub(crate) struct WritebackPipeline {
|
||||
chunk_bytes: u64,
|
||||
last_flush_pos: u64,
|
||||
pending: Option<(u64, u64)>,
|
||||
/// Rolling window of recent `WAIT_AFTER` elapsed_ms measurements.
|
||||
wait_after_window: VecDeque<u64>,
|
||||
/// Count of chunks emitted (used to space out periodic
|
||||
/// `debug!` size snapshots).
|
||||
chunk_count: u64,
|
||||
/// What `detect_storage_class` saw at construction. Carried for
|
||||
/// diagnostic logging only — *not* keyed off in the hot path.
|
||||
storage_class: StorageClass,
|
||||
/// Set the first time WAIT_AFTER exceeds [`WAIT_AFTER_TIMEOUT`].
|
||||
/// Once set, behaviour matches the NFS path for the rest of the
|
||||
/// pipeline's life. Wrapped in `Arc` only because both this
|
||||
/// struct and the spawned worker thread (which itself doesn't
|
||||
/// touch the flag) share-via-fd patterns might one day need it;
|
||||
/// today it's effectively a single-owner cell — the `Arc` shape
|
||||
/// keeps the door open for moving the read side into a worker
|
||||
/// without re-plumbing types.
|
||||
degraded: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl WritebackPipeline {
|
||||
@@ -35,15 +120,47 @@ impl WritebackPipeline {
|
||||
/// returned `WritebackPipeline` MUST be dropped before `file`
|
||||
/// itself, or kept inside the same struct that owns `file` — the
|
||||
/// alias is unchecked.
|
||||
pub(crate) fn new(file: &File, start_pos: u64, chunk_bytes: u64) -> Self {
|
||||
///
|
||||
/// `chunk_bytes_hint` is the caller's preferred starting chunk
|
||||
/// size; the constructor overrides it with a medium-specific
|
||||
/// initial seed if `detect_storage_class` recognises the fd's
|
||||
/// filesystem. The autotuner then drives the value from there
|
||||
/// based on measured `WAIT_AFTER` latency, identically on every
|
||||
/// medium.
|
||||
pub(crate) fn new(file: &File, start_pos: u64, chunk_bytes_hint: u64) -> Self {
|
||||
let fd = file.as_raw_fd();
|
||||
let (class, seed) = detect_storage_class(fd);
|
||||
// Medium-aware initial seed: bias toward bigger chunks on
|
||||
// slow-commit media so the autotuner doesn't waste a minute
|
||||
// climbing from a too-small starting value. A returned seed
|
||||
// of 0 means "the detector has no opinion; use the caller's
|
||||
// hint". All subsequent tuning is medium-agnostic.
|
||||
let chunk_bytes = if seed > 0 { seed } else { chunk_bytes_hint };
|
||||
tracing::info!(
|
||||
target: "mux",
|
||||
"WritebackPipeline fd={fd} storage_class={class:?} chunk_bytes={chunk_bytes}"
|
||||
);
|
||||
Self {
|
||||
fd: file.as_raw_fd(),
|
||||
fd,
|
||||
chunk_bytes,
|
||||
last_flush_pos: start_pos,
|
||||
pending: None,
|
||||
wait_after_window: VecDeque::with_capacity(ADAPTIVE_WINDOW),
|
||||
chunk_count: 0,
|
||||
storage_class: class,
|
||||
degraded: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
/// True if we should bypass the WAIT_AFTER finalisation step. The
|
||||
/// pipeline only skips after `WAIT_AFTER` has *empirically* hung
|
||||
/// past [`WAIT_AFTER_TIMEOUT`] on this particular fd. No per-FS
|
||||
/// branch — the failure is detected, not predicted.
|
||||
#[inline]
|
||||
fn skip_wait(&self) -> bool {
|
||||
self.degraded.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Caller advanced the file position to `pos`. If a chunk boundary
|
||||
/// was crossed, kick async writeback for the just-completed chunk
|
||||
/// and finalise the previous one.
|
||||
@@ -51,27 +168,105 @@ impl WritebackPipeline {
|
||||
if pos < self.last_flush_pos.saturating_add(self.chunk_bytes) {
|
||||
return;
|
||||
}
|
||||
let chunk_off = self.last_flush_pos as i64;
|
||||
let chunk_len = (pos - self.last_flush_pos) as i64;
|
||||
let mut wait_ms: u64 = 0;
|
||||
let mut fadvise_ms: u64 = 0;
|
||||
// Async kickoff for the just-completed chunk runs on every
|
||||
// path (NFS, degraded, normal) — it's nominally non-blocking
|
||||
// by spec and gives the kernel an early hint that this range
|
||||
// is ready to flush.
|
||||
unsafe {
|
||||
let chunk_off = self.last_flush_pos as i64;
|
||||
let chunk_len = (pos - self.last_flush_pos) as i64;
|
||||
libc::sync_file_range(self.fd, chunk_off, chunk_len, libc::SYNC_FILE_RANGE_WRITE);
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
libc::sync_file_range(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::SYNC_FILE_RANGE_WAIT_AFTER,
|
||||
);
|
||||
libc::posix_fadvise(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
);
|
||||
}
|
||||
self.pending = Some((chunk_off as u64, chunk_len as u64));
|
||||
}
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
if self.skip_wait() {
|
||||
// Degraded path (set only after an empirical
|
||||
// WAIT_AFTER timeout on this fd). We still advance
|
||||
// `pending` so the next call has a stable cycle.
|
||||
} else {
|
||||
match wait_after_with_timeout(self.fd, prev_off, prev_len) {
|
||||
Some(ms) => {
|
||||
wait_ms = ms;
|
||||
// Deliberately NO `posix_fadvise(DONTNEED)`
|
||||
// here. See module-level docs: evicting clean
|
||||
// pages costs us read-modify-write on
|
||||
// matroska cluster backpatches and gains
|
||||
// nothing — the kernel will reclaim clean
|
||||
// pages under LRU when memory is needed.
|
||||
self.record_wait(wait_ms);
|
||||
}
|
||||
None => {
|
||||
// Timeout branch: this fd's FS or server is
|
||||
// unable to ack WAIT_AFTER. Flip to skip mode
|
||||
// for the rest of the pipeline's life.
|
||||
self.degraded.store(true, Ordering::Relaxed);
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackPipeline WAIT_AFTER timed out after {}s on chunk off={} len={}, marking writeback degraded (subsequent chunks will skip WAIT_AFTER)",
|
||||
WAIT_AFTER_TIMEOUT.as_secs(),
|
||||
prev_off,
|
||||
prev_len
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.pending = Some((chunk_off as u64, chunk_len as u64));
|
||||
self.last_flush_pos = pos;
|
||||
self.chunk_count += 1;
|
||||
let _ = fadvise_ms;
|
||||
tracing::trace!(
|
||||
target: "mux",
|
||||
"WritebackPipeline chunk off={} len={} sync_file_range_ms={wait_ms} chunk_bytes={} skip_wait={}",
|
||||
chunk_off,
|
||||
chunk_len,
|
||||
self.chunk_bytes,
|
||||
self.skip_wait(),
|
||||
);
|
||||
if self.chunk_count % SIZE_LOG_INTERVAL == 0 {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackPipeline chunk_bytes={} after {} chunks storage_class={:?} degraded={}",
|
||||
self.chunk_bytes,
|
||||
self.chunk_count,
|
||||
self.storage_class,
|
||||
self.degraded.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Push a new `WAIT_AFTER` measurement into the rolling window
|
||||
/// and, if the window is full, adapt `chunk_bytes` based on p95.
|
||||
fn record_wait(&mut self, wait_ms: u64) {
|
||||
if self.wait_after_window.len() == ADAPTIVE_WINDOW {
|
||||
self.wait_after_window.pop_front();
|
||||
}
|
||||
self.wait_after_window.push_back(wait_ms);
|
||||
if self.wait_after_window.len() < ADAPTIVE_WINDOW {
|
||||
return;
|
||||
}
|
||||
// p95 of 16 samples ≈ sorted[14] (5 % of 16 = 0.8 ≈ 1 above).
|
||||
let mut sorted: Vec<u64> = self.wait_after_window.iter().copied().collect();
|
||||
sorted.sort_unstable();
|
||||
let p95 = sorted[14];
|
||||
let old = self.chunk_bytes;
|
||||
let new = if p95 > ADAPTIVE_GROW_MS && self.chunk_bytes < CHUNK_BYTES_MAX {
|
||||
(self.chunk_bytes * 2).min(CHUNK_BYTES_MAX)
|
||||
} else if p95 < ADAPTIVE_SHRINK_MS && self.chunk_bytes > CHUNK_BYTES_MIN {
|
||||
(self.chunk_bytes / 2).max(CHUNK_BYTES_MIN)
|
||||
} else {
|
||||
self.chunk_bytes
|
||||
};
|
||||
if new != old {
|
||||
self.chunk_bytes = new;
|
||||
tracing::info!(
|
||||
target: "mux",
|
||||
"WritebackPipeline adaptive chunk_bytes {} -> {} p95_ms={p95}",
|
||||
old,
|
||||
new
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Caller is about to seek away from the current write region.
|
||||
@@ -85,20 +280,110 @@ impl WritebackPipeline {
|
||||
/// or when discarding the pipeline.
|
||||
pub(crate) fn finalize(&mut self) {
|
||||
if let Some((prev_off, prev_len)) = self.pending.take() {
|
||||
unsafe {
|
||||
libc::sync_file_range(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::SYNC_FILE_RANGE_WAIT_AFTER,
|
||||
);
|
||||
libc::posix_fadvise(
|
||||
self.fd,
|
||||
prev_off as i64,
|
||||
prev_len as i64,
|
||||
libc::POSIX_FADV_DONTNEED,
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackPipeline finalize chunk off={prev_off} len={prev_len} skip_wait={} storage_class={:?} degraded={}",
|
||||
self.skip_wait(),
|
||||
self.storage_class,
|
||||
self.degraded.load(Ordering::Relaxed),
|
||||
);
|
||||
if self.skip_wait() {
|
||||
return;
|
||||
}
|
||||
if wait_after_with_timeout(self.fd, prev_off, prev_len).is_none() {
|
||||
self.degraded.store(true, Ordering::Relaxed);
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackPipeline finalize WAIT_AFTER timed out after {}s on chunk off={prev_off} len={prev_len}, marking writeback degraded",
|
||||
WAIT_AFTER_TIMEOUT.as_secs(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Coarse classification of the medium an open file is on. Used only
|
||||
/// at construction to seed the autotuner with a sensible initial
|
||||
/// `chunk_bytes`. Never branched on in the hot path.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum StorageClass {
|
||||
/// `fstatfs.f_type == NFS_SUPER_MAGIC`. Slow-commit; seed a
|
||||
/// generous initial chunk so the autotuner doesn't waste cycles
|
||||
/// climbing from a too-small value.
|
||||
Nfs,
|
||||
/// Anything else we successfully stat'd. Local FS, tmpfs, ZFS,
|
||||
/// network FS we don't have a constant for, etc. Use the caller's
|
||||
/// hint as the seed.
|
||||
Other,
|
||||
/// `fstatfs` failed. Caller's hint wins.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Probe the FS class via `fstatfs` and return both the class label
|
||||
/// and a recommended `chunk_bytes` seed. `seed == 0` means "no
|
||||
/// medium-specific recommendation; use the caller's hint." Failure
|
||||
/// modes default to `Unknown` + seed 0 so the caller's hint applies.
|
||||
///
|
||||
/// This is the single medium-detection point in the whole writeback
|
||||
/// pipeline. Everything else is medium-agnostic.
|
||||
fn detect_storage_class(fd: RawFd) -> (StorageClass, u64) {
|
||||
// `libc::statfs` is repr(C) with a fixed layout; zeroing is the
|
||||
// documented init pattern for the kernel uapi struct.
|
||||
let mut buf: libc::statfs = unsafe { std::mem::zeroed() };
|
||||
let rc = unsafe { libc::fstatfs(fd, &mut buf) };
|
||||
if rc != 0 {
|
||||
let errno = std::io::Error::last_os_error();
|
||||
tracing::warn!(
|
||||
target: "mux",
|
||||
"WritebackPipeline fstatfs(fd={fd}) failed: {errno} — defaulting to Unknown",
|
||||
);
|
||||
return (StorageClass::Unknown, 0);
|
||||
}
|
||||
// `f_type` is signed (`__fsword_t`) on glibc and unsigned
|
||||
// (`c_ulong`) on musl. Cast both sides to i64 for a portable
|
||||
// comparison. On glibc x86_64 both already are i64 — clippy flags
|
||||
// the cast as unnecessary on that target only, but we need it for
|
||||
// musl, so silence the lint.
|
||||
#[allow(clippy::unnecessary_cast)]
|
||||
let f_type = buf.f_type as i64;
|
||||
#[allow(clippy::unnecessary_cast)]
|
||||
let nfs_magic = libc::NFS_SUPER_MAGIC as i64;
|
||||
if f_type == nfs_magic {
|
||||
// NFS commit ack typically ~10-30 ms RTT. The autotuner grows
|
||||
// by doubling when p95 > 200 ms — so starting at 32 MiB it
|
||||
// would never grow on a healthy NFS. Seed at 64 MiB so the
|
||||
// commit cadence amortizes properly from the first chunk.
|
||||
(StorageClass::Nfs, 64 * 1024 * 1024)
|
||||
} else {
|
||||
// Local FS / unknown remote FS. Caller's hint (typically 32
|
||||
// MiB) is a fine starting point; autotuner adjusts from
|
||||
// measured latency.
|
||||
(StorageClass::Other, 0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Run `sync_file_range(WAIT_AFTER)` on a worker thread and wait up
|
||||
/// to [`WAIT_AFTER_TIMEOUT`] for it to return. `Some(elapsed_ms)` on
|
||||
/// success; `None` on timeout. On timeout the worker thread is
|
||||
/// 0.20.6 generalizes the worker-thread + recv_timeout pattern into
|
||||
/// [`crate::io::bounded::bounded_syscall`]; this helper now just adapts
|
||||
/// the generic primitive to the WAIT_AFTER call shape (returns elapsed_ms
|
||||
/// instead of the syscall's `()` return, treats `WorkerLost` as a benign
|
||||
/// no-op to match the original semantics).
|
||||
fn wait_after_with_timeout(fd: RawFd, off: u64, len: u64) -> Option<u64> {
|
||||
let started = Instant::now();
|
||||
match crate::io::bounded::bounded_syscall(None, WAIT_AFTER_TIMEOUT, move || unsafe {
|
||||
libc::sync_file_range(fd, off as i64, len as i64, libc::SYNC_FILE_RANGE_WAIT_AFTER);
|
||||
}) {
|
||||
Ok(()) => Some(started.elapsed().as_millis() as u64),
|
||||
Err(crate::io::bounded::BoundedError::Timeout)
|
||||
| Err(crate::io::bounded::BoundedError::Halted) => None,
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => {
|
||||
// Worker thread spawn failed or panicked before sending.
|
||||
// Treat as a benign success (no syscall ran) rather than
|
||||
// a degrade trigger — falling through with elapsed_ms=0
|
||||
// matches the no-op behaviour.
|
||||
Some(0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
//! `WritebackFile` — a `File` wrapper whose reason for existing is the
|
||||
//! bounded-cache writeback pipeline.
|
||||
//!
|
||||
//! Why: large sequential writes (sweep, patch, mux on UHD-scale output)
|
||||
//! left to the kernel's default writeback policy accumulate hundreds of
|
||||
//! megabytes of dirty pages and then burst-flush, stalling subsequent
|
||||
//! writes for seconds at a time. `WritebackFile` drives a continuous
|
||||
//! [`super::writeback::WritebackPipeline`] that on Linux issues
|
||||
//! incremental `sync_file_range` + `posix_fadvise(DONTNEED)` calls at
|
||||
//! 32 MB granularity so dirty pages drain at the same rate they're
|
||||
//! produced. macOS and Windows fall through to a no-op pipeline — their
|
||||
//! default cache policies have not been shown to exhibit the same
|
||||
//! pathology for this access pattern.
|
||||
//!
|
||||
//! It implements `Write` and `Seek` so any call site that wrote to a
|
||||
//! plain `File` through those traits (sweep, patch, mux) can swap in
|
||||
//! `WritebackFile` without touching the body of the loop. The wrapper
|
||||
//! also tracks the current file position to feed the pipeline with
|
||||
//! progress + seek boundaries.
|
||||
//!
|
||||
//! See `super::writeback::linux` for the underlying pathology and the
|
||||
//! strategy.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use super::writeback::WritebackPipeline;
|
||||
|
||||
const WRITEBACK_CHUNK_BYTES: u64 = 32 * 1024 * 1024;
|
||||
|
||||
pub(crate) struct WritebackFile {
|
||||
file: File,
|
||||
pipeline: WritebackPipeline,
|
||||
pos: u64,
|
||||
}
|
||||
|
||||
impl WritebackFile {
|
||||
/// Wrap an open `File`. The current OS file position is queried
|
||||
/// once so the pipeline starts tracking from wherever the file
|
||||
/// already is (typically 0 for fresh files; non-zero for resumed
|
||||
/// or appended files).
|
||||
pub(crate) fn new(mut file: File) -> io::Result<Self> {
|
||||
let pos = file.stream_position()?;
|
||||
let pipeline = WritebackPipeline::new(&file, pos, WRITEBACK_CHUNK_BYTES);
|
||||
Ok(Self {
|
||||
file,
|
||||
pipeline,
|
||||
pos,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new file at `path` (truncating any existing contents)
|
||||
/// and wrap it. Convenience for the common
|
||||
/// `File::create(path)` + `WritebackFile::new(file)` pair so callers
|
||||
/// don't have to assemble a `File` first.
|
||||
pub(crate) fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = File::create(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Open an existing file at `path` for writing (no truncation) and
|
||||
/// wrap it. Mirrors `File::open` semantics for the writable case
|
||||
/// — used by patch / resume paths that mutate an existing ISO in
|
||||
/// place.
|
||||
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new().write(true).open(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Drain in-flight writeback then issue a full fsync. Use this in
|
||||
/// place of `File::sync_all`.
|
||||
pub(crate) fn sync_all(&mut self) -> io::Result<()> {
|
||||
self.pipeline.finalize();
|
||||
self.file.sync_all()
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for WritebackFile {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let n = self.file.write(buf)?;
|
||||
self.pos += n as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.file.write_all(buf)?;
|
||||
self.pos += buf.len() as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for WritebackFile {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
let p = self.file.seek(from)?;
|
||||
// Only treat seeks that actually move the position as
|
||||
// boundaries — sweep does a redundant `seek(Current(pos))`
|
||||
// before every write, and we don't want that to drain the
|
||||
// pipeline on every iteration.
|
||||
if p != self.pos {
|
||||
self.pipeline.handle_seek(p);
|
||||
self.pos = p;
|
||||
}
|
||||
Ok(p)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WritebackFile {
|
||||
fn drop(&mut self) {
|
||||
// Run the pipeline's tail finalize so the last in-flight chunk
|
||||
// gets its `WAIT_AFTER` + `posix_fadvise(DONTNEED)`. Without
|
||||
// this, callers that drop a `WritebackFile` without calling
|
||||
// `sync_all` (panic, early-return, idiomatic `let _ = w;`)
|
||||
// leave the trailing chunk in cache; the kernel still flushes
|
||||
// on close, but the bounded-cache invariant fails at the tail.
|
||||
// We deliberately do *not* call `self.file.sync_all()` here —
|
||||
// close already triggers a flush, and an `fsync` from `Drop`
|
||||
// would silently swallow its `io::Error` anyway. `finalize` is
|
||||
// idempotent so an explicit `sync_all` followed by drop is
|
||||
// still safe.
|
||||
self.pipeline.finalize();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//! Linux platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! - `preallocate`: `fallocate(FALLOC_FL_KEEP_SIZE)` — reserve extents
|
||||
//! without growing the reported file size. Reduces extent
|
||||
//! fragmentation on large sequential writes (mux output on NFS in
|
||||
//! particular).
|
||||
//! - `durable_sync`: `fsync` wrapped in
|
||||
//! [`crate::io::bounded::bounded_syscall`] with a 60 s deadline so a
|
||||
//! wedged NFS server can't trap the calling thread indefinitely.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Pre-reserve extents for `size_bytes` of upcoming sequential writes.
|
||||
/// Best-effort: a non-zero rc is logged but not propagated, since the
|
||||
/// caller would just continue with the unreserved file anyway.
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
// FALLOC_FL_KEEP_SIZE = 0x01 — keep the reported file size at 0
|
||||
// (writes grow it normally) while still pre-reserving the extents.
|
||||
let rc = unsafe {
|
||||
libc::fallocate(
|
||||
file.as_raw_fd(),
|
||||
libc::FALLOC_FL_KEEP_SIZE,
|
||||
0,
|
||||
size_bytes as i64,
|
||||
)
|
||||
};
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile fallocate size_hint={size_bytes} rc={rc} ok={}",
|
||||
rc == 0
|
||||
);
|
||||
}
|
||||
|
||||
/// Run `fsync` on `file` with a 60 s deadline. On timeout we log loudly
|
||||
/// and return `Ok(())` — the kernel will still flush on close, so the
|
||||
/// data is best-effort durable; the alternative (trap the thread for
|
||||
/// the rest of the rip) defeats `/api/stop`.
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
let fd = file.as_raw_fd();
|
||||
match crate::io::bounded::bounded_syscall(
|
||||
None,
|
||||
Duration::from_secs(60),
|
||||
move || -> io::Result<()> {
|
||||
let rc = unsafe { libc::fsync(fd) };
|
||||
if rc == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::last_os_error())
|
||||
}
|
||||
},
|
||||
) {
|
||||
Ok(inner) => inner,
|
||||
Err(crate::io::bounded::BoundedError::Timeout) => {
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all fsync timed out after 60s; kernel will flush on close (best-effort)"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(crate::io::bounded::BoundedError::Halted) => Ok(()),
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => Ok(()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
//! macOS platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! - `preallocate`: `fcntl(F_PREALLOCATE)` — macOS's fallocate-equiv.
|
||||
//! Reserves a contiguous extent when possible, falling back to a
|
||||
//! non-contiguous reservation if the FS can't satisfy it. Reported
|
||||
//! file size is unchanged (`F_ALLOCATEALL` is not set, so allocation
|
||||
//! is "best effort up to length"; growth happens via writes).
|
||||
//! - `durable_sync`: `fcntl(F_FULLFSYNC)` wrapped in
|
||||
//! [`crate::io::bounded::bounded_syscall`] with a 60 s deadline.
|
||||
//! F_FULLFSYNC is HFS+/APFS's true-fsync (flushes the disk's own
|
||||
//! write cache) — what `fsync` should have been on macOS. Falls back
|
||||
//! to plain `fsync` if F_FULLFSYNC returns ENOTSUP.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::time::Duration;
|
||||
|
||||
/// libc `F_PREALLOCATE` — not exposed by the `libc` crate on all macOS
|
||||
/// SDK versions, so define it here.
|
||||
const F_PREALLOCATE: libc::c_int = 42;
|
||||
/// Allocate from current EOF.
|
||||
const F_PEOFPOSMODE: libc::c_int = 3;
|
||||
/// Hint: contiguous extent preferred.
|
||||
const F_ALLOCATECONTIG: libc::c_uint = 0x00000002;
|
||||
/// Allocate all the requested bytes (fall back to non-contig if needed).
|
||||
const F_ALLOCATEALL: libc::c_uint = 0x00000004;
|
||||
|
||||
/// `fcntl(F_FULLFSYNC)` opcode. Documented in `man 2 fcntl` on macOS;
|
||||
/// not in the `libc` crate as a named constant.
|
||||
const F_FULLFSYNC: libc::c_int = 51;
|
||||
|
||||
/// `fstore_t` layout matches `sys/fcntl.h`. Repr is C-stable so we can
|
||||
/// build it manually.
|
||||
#[repr(C)]
|
||||
struct Fstore {
|
||||
fst_flags: libc::c_uint,
|
||||
fst_posmode: libc::c_int,
|
||||
fst_offset: libc::off_t,
|
||||
fst_length: libc::off_t,
|
||||
fst_bytesalloc: libc::off_t,
|
||||
}
|
||||
|
||||
pub(super) fn preallocate(file: &File, size_bytes: u64) {
|
||||
let mut fst = Fstore {
|
||||
fst_flags: F_ALLOCATECONTIG | F_ALLOCATEALL,
|
||||
fst_posmode: F_PEOFPOSMODE,
|
||||
fst_offset: 0,
|
||||
fst_length: size_bytes as libc::off_t,
|
||||
fst_bytesalloc: 0,
|
||||
};
|
||||
// First attempt: contiguous.
|
||||
let mut rc = unsafe { libc::fcntl(file.as_raw_fd(), F_PREALLOCATE, &mut fst) };
|
||||
if rc == -1 {
|
||||
// Fall back: drop the contiguous hint, allow scattered extents.
|
||||
fst.fst_flags = F_ALLOCATEALL;
|
||||
rc = unsafe { libc::fcntl(file.as_raw_fd(), F_PREALLOCATE, &mut fst) };
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile F_PREALLOCATE size_hint={size_bytes} rc={rc} bytes_allocated={} ok={}",
|
||||
fst.fst_bytesalloc,
|
||||
rc != -1
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
let fd = file.as_raw_fd();
|
||||
match crate::io::bounded::bounded_syscall(
|
||||
None,
|
||||
Duration::from_secs(60),
|
||||
move || -> io::Result<()> {
|
||||
// Try F_FULLFSYNC first. If it isn't supported on this
|
||||
// filesystem (older HFS, some network mounts) fall back to
|
||||
// plain fsync — better than nothing.
|
||||
let rc = unsafe { libc::fcntl(fd, F_FULLFSYNC, 0) };
|
||||
if rc == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let err = io::Error::last_os_error();
|
||||
if err.raw_os_error() == Some(libc::ENOTSUP) {
|
||||
let rc = unsafe { libc::fsync(fd) };
|
||||
if rc == 0 {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::last_os_error())
|
||||
}
|
||||
} else {
|
||||
Err(err)
|
||||
}
|
||||
},
|
||||
) {
|
||||
Ok(inner) => inner,
|
||||
Err(crate::io::bounded::BoundedError::Timeout) => {
|
||||
tracing::error!(
|
||||
target: "mux",
|
||||
"WritebackFile::sync_all F_FULLFSYNC timed out after 60s; kernel will flush on close (best-effort)"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(crate::io::bounded::BoundedError::Halted) => Ok(()),
|
||||
Err(crate::io::bounded::BoundedError::WorkerLost) => Ok(()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
//! `WritebackFile` — a `File` wrapper whose reason for existing is the
|
||||
//! bounded-cache writeback pipeline.
|
||||
//!
|
||||
//! Why: large sequential writes (sweep, patch, mux on UHD-scale output)
|
||||
//! left to the kernel's default writeback policy accumulate hundreds of
|
||||
//! megabytes of dirty pages and then burst-flush, stalling subsequent
|
||||
//! writes for seconds at a time. `WritebackFile` drives a continuous
|
||||
//! [`super::writeback::WritebackPipeline`] that on Linux issues
|
||||
//! incremental `sync_file_range` + `posix_fadvise(DONTNEED)` calls at
|
||||
//! 32 MB granularity so dirty pages drain at the same rate they're
|
||||
//! produced. macOS and Windows fall through to a no-op pipeline — their
|
||||
//! default cache policies have not been shown to exhibit the same
|
||||
//! pathology for this access pattern.
|
||||
//!
|
||||
//! It implements `Write` and `Seek` so any call site that wrote to a
|
||||
//! plain `File` through those traits (sweep, patch, mux) can swap in
|
||||
//! `WritebackFile` without touching the body of the loop. The wrapper
|
||||
//! also tracks the current file position to feed the pipeline with
|
||||
//! progress + seek boundaries.
|
||||
//!
|
||||
//! See `super::writeback::linux` for the underlying pathology and the
|
||||
//! strategy.
|
||||
//!
|
||||
//! ## Platform split
|
||||
//!
|
||||
//! The platform-specific pieces of this wrapper — extent preallocation
|
||||
//! (Linux `fallocate(KEEP_SIZE)`, macOS `F_PREALLOCATE`, Windows
|
||||
//! `SetFileValidData`) and the durable-flush primitive (Linux/macOS
|
||||
//! `fsync`/`F_FULLFSYNC` wrapped in a bounded syscall, Windows
|
||||
//! `FlushFileBuffers`) — live in per-OS sibling modules. The dispatch
|
||||
//! happens once at the bottom of this file via cfg-gated `mod` decls.
|
||||
//! No inline `#[cfg(target_os = "...")]` in the business-logic above.
|
||||
//!
|
||||
//! ## Write path
|
||||
//!
|
||||
//! Writes are direct passthrough to the underlying `File` (no writer
|
||||
//! thread, no ring, no batching). Empirically the Phase-2.5
|
||||
//! writer-thread architecture introduced a ~60% mux throughput
|
||||
//! regression on NFS bidirectional workloads; reverting the write path
|
||||
//! to direct passthrough restores the 0.20.7 baseline. The writeback
|
||||
//! pipeline still runs (it's called inline from `write` / `write_all` /
|
||||
//! `seek`) so the bounded-cache invariant on Linux is preserved.
|
||||
//!
|
||||
//! ## Halt-safety
|
||||
//!
|
||||
//! `sync_all` runs the per-OS durable-flush primitive, which on
|
||||
//! Linux/macOS is wrapped in [`crate::io::bounded::bounded_syscall`]
|
||||
//! with a 60 s deadline. A wedged NFS server cannot trap the muxer
|
||||
//! indefinitely on the final fsync.
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
mod other;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux as platform;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos as platform;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
use other as platform;
|
||||
#[cfg(target_os = "windows")]
|
||||
use windows as platform;
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use super::writeback::WritebackPipeline;
|
||||
|
||||
/// Granularity at which the Linux writeback pipeline issues
|
||||
/// `sync_file_range` / `posix_fadvise(DONTNEED)` pairs. 32 MiB is the
|
||||
/// historical default — bounded-cache pressure stays at ~2 × this size.
|
||||
const WRITEBACK_CHUNK_BYTES: u64 = 32 * 1024 * 1024;
|
||||
|
||||
pub(crate) struct WritebackFile {
|
||||
file: File,
|
||||
pipeline: WritebackPipeline,
|
||||
pos: u64,
|
||||
}
|
||||
|
||||
impl WritebackFile {
|
||||
/// Wrap an open `File`. The current OS file position is queried
|
||||
/// once so the pipeline starts tracking from wherever the file
|
||||
/// already is (typically 0 for fresh files; non-zero for resumed
|
||||
/// or appended files).
|
||||
pub(crate) fn new(mut file: File) -> io::Result<Self> {
|
||||
let pos = file.stream_position()?;
|
||||
let pipeline = WritebackPipeline::new(&file, pos, WRITEBACK_CHUNK_BYTES);
|
||||
Ok(Self {
|
||||
file,
|
||||
pipeline,
|
||||
pos,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new file at `path` (truncating any existing contents)
|
||||
/// and wrap it. Convenience for the common
|
||||
/// `File::create(path)` + `WritebackFile::new(file)` pair so callers
|
||||
/// don't have to assemble a `File` first.
|
||||
///
|
||||
/// Callers that know the target output size should prefer
|
||||
/// [`Self::create_with_size_hint`] so the kernel can pre-reserve
|
||||
/// extents.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn create(path: &Path) -> io::Result<Self> {
|
||||
let file = File::create(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Like [`Self::create`] but pre-reserves `size_bytes` of disk
|
||||
/// space via the platform's extent-preallocation primitive (Linux
|
||||
/// `fallocate(KEEP_SIZE)`, macOS `F_PREALLOCATE`, Windows
|
||||
/// `SetFileValidData` stub). The reported file size is unchanged
|
||||
/// (writes still grow the file naturally) — only the on-disk extent
|
||||
/// allocation is preallocated, which reduces extent fragmentation
|
||||
/// on large sequential writes (mux output, especially on slow
|
||||
/// storage / NFS).
|
||||
///
|
||||
/// On platforms without an extent-preallocation primitive this is
|
||||
/// equivalent to `create` — the size hint is dropped after a debug
|
||||
/// log.
|
||||
pub(crate) fn create_with_size_hint(path: &Path, size_bytes: u64) -> io::Result<Self> {
|
||||
let file = File::create(path)?;
|
||||
platform::preallocate(&file, size_bytes);
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Open an existing file at `path` for writing (no truncation) and
|
||||
/// wrap it. Mirrors `File::open` semantics for the writable case
|
||||
/// — used by patch / resume paths that mutate an existing ISO in
|
||||
/// place.
|
||||
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
||||
let file = OpenOptions::new().write(true).open(path)?;
|
||||
Self::new(file)
|
||||
}
|
||||
|
||||
/// Drain in-flight writeback then issue a full fsync. Use this in
|
||||
/// place of `File::sync_all`.
|
||||
///
|
||||
/// The final durable flush is wrapped in
|
||||
/// [`crate::io::bounded::bounded_syscall`] (per the per-OS module)
|
||||
/// with a 60 s deadline on Linux/macOS — a wedged NFS server cannot
|
||||
/// trap the calling thread indefinitely. On timeout the page cache
|
||||
/// is left to the kernel's normal flush-on-close path — best
|
||||
/// effort, but bounded.
|
||||
pub(crate) fn sync_all(&mut self) -> io::Result<()> {
|
||||
self.pipeline.finalize();
|
||||
platform::durable_sync(&self.file)
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for WritebackFile {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let n = self.file.write(buf)?;
|
||||
self.pos += n as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.file.write_all(buf)?;
|
||||
self.pos += buf.len() as u64;
|
||||
self.pipeline.note_progress(self.pos);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl Seek for WritebackFile {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
let p = self.file.seek(from)?;
|
||||
// Only treat seeks that actually move the position as
|
||||
// boundaries — sweep does a redundant `seek(Current(pos))`
|
||||
// before every write, and we don't want that to drain the
|
||||
// pipeline on every iteration.
|
||||
if p != self.pos {
|
||||
// Diagnostic for the NFS mux hang: the MKV format requires
|
||||
// the muxer to seek back occasionally (cluster size
|
||||
// patching, Cues index write, Segment header backpatch).
|
||||
// Each such seek invalidates the writeback chunk tracking
|
||||
// and forces a finalize → WAIT_AFTER on the in-flight
|
||||
// chunk. Logging the seek delta lets us correlate hang
|
||||
// offsets with specific muxer operations.
|
||||
let from_pos = self.pos;
|
||||
let to_pos = p;
|
||||
let delta: i64 = (to_pos as i64).wrapping_sub(from_pos as i64);
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile seek from={from_pos} to={to_pos} delta={delta}"
|
||||
);
|
||||
self.pipeline.handle_seek(p);
|
||||
self.pos = p;
|
||||
}
|
||||
Ok(p)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WritebackFile {
|
||||
fn drop(&mut self) {
|
||||
// Run the pipeline's tail finalize so the last in-flight chunk
|
||||
// gets its `WAIT_AFTER` + `posix_fadvise(DONTNEED)`. Without
|
||||
// this, callers that drop a `WritebackFile` without calling
|
||||
// `sync_all` (panic, early-return, idiomatic `let _ = w;`)
|
||||
// leave the trailing chunk in cache; the kernel still flushes
|
||||
// on close, but the bounded-cache invariant fails at the tail.
|
||||
// We deliberately do *not* call `self.file.sync_all()` here —
|
||||
// close already triggers a flush, and an `fsync` from `Drop`
|
||||
// would silently swallow its `io::Error` anyway. `finalize` is
|
||||
// idempotent so an explicit `sync_all` followed by drop is
|
||||
// still safe.
|
||||
self.pipeline.finalize();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
|
||||
fn read_back(path: &Path) -> Vec<u8> {
|
||||
let mut f = File::open(path).unwrap();
|
||||
let mut v = Vec::new();
|
||||
f.read_to_end(&mut v).unwrap();
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_then_drop_persists_bytes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("a.bin");
|
||||
{
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"hello world").unwrap();
|
||||
// Drop drains the pipeline tail.
|
||||
}
|
||||
assert_eq!(read_back(&p), b"hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sync_all_drains_and_flushes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("b.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
for _ in 0..32 {
|
||||
w.write_all(&[0x5au8; 1024]).unwrap();
|
||||
}
|
||||
// After sync_all, the bytes MUST be visible to a separate
|
||||
// reader. The pipeline has been finalised and durable-sync has
|
||||
// run.
|
||||
w.sync_all().unwrap();
|
||||
let bytes = read_back(&p);
|
||||
assert_eq!(bytes.len(), 32 * 1024);
|
||||
assert!(bytes.iter().all(|&b| b == 0x5a));
|
||||
drop(w);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seek_then_patch_roundtrip() {
|
||||
// Write A; seek back; patch with B; read back; the patch lands
|
||||
// at the right offset.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("c.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
let big = vec![b'A'; 4096];
|
||||
w.write_all(&big).unwrap();
|
||||
// Seek back to offset 1000 and overwrite 8 bytes.
|
||||
w.seek(SeekFrom::Start(1000)).unwrap();
|
||||
w.write_all(b"PATCHED!").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
let bytes = read_back(&p);
|
||||
assert_eq!(bytes.len(), 4096);
|
||||
assert_eq!(&bytes[1000..1008], b"PATCHED!");
|
||||
// Bytes outside the patch are still 'A'.
|
||||
assert_eq!(bytes[999], b'A');
|
||||
assert_eq!(bytes[1008], b'A');
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flush_is_observed_in_order() {
|
||||
// `Write::flush` should not panic or reorder; verify the bytes
|
||||
// land in order through interleaved flushes.
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("f.bin");
|
||||
let mut w = WritebackFile::create(&p).unwrap();
|
||||
w.write_all(b"one").unwrap();
|
||||
w.flush().unwrap();
|
||||
w.write_all(b"two").unwrap();
|
||||
w.flush().unwrap();
|
||||
w.write_all(b"three").unwrap();
|
||||
w.sync_all().unwrap();
|
||||
drop(w);
|
||||
assert_eq!(read_back(&p), b"onetwothree");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//! Fallback platform impl for [`super::WritebackFile`] on targets
|
||||
//! without a dedicated implementation (BSDs, illumos, etc.).
|
||||
//!
|
||||
//! - `preallocate` is a logged no-op.
|
||||
//! - `durable_sync` calls `File::sync_all` directly (no bounded-syscall
|
||||
//! wrapper — the wrapper depends on Linux/macOS unix idioms that
|
||||
//! aren't universally portable). If a future BSD impl needs the
|
||||
//! 60-s deadline, it should land in its own per-OS file rather than
|
||||
//! bloat this fallback.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
|
||||
pub(super) fn preallocate(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile preallocate size_hint={size_bytes} skipped (no impl on this target)"
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
file.sync_all()
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//! Windows platform impl for [`super::WritebackFile`].
|
||||
//!
|
||||
//! TODO: this stub matches the design's "validate without a Windows
|
||||
//! build env, leave a stub" carve-out. The real impl should use:
|
||||
//!
|
||||
//! - `SetEndOfFile` + `SetFileValidData` for extent preallocation
|
||||
//! (caller needs `SE_MANAGE_VOLUME_NAME` privilege; if unavailable
|
||||
//! fall back to a write-zero path or just skip).
|
||||
//! - `FlushFileBuffers` for fsync-equivalent durable flush.
|
||||
//!
|
||||
//! Until then: preallocate is a debug-logged no-op; durable_sync calls
|
||||
//! the std `File::sync_all` (which on Windows maps to
|
||||
//! `FlushFileBuffers` internally).
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
|
||||
pub(super) fn preallocate(_file: &File, size_bytes: u64) {
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"WritebackFile preallocate size_hint={size_bytes} skipped (windows stub; TODO: SetFileValidData)"
|
||||
);
|
||||
}
|
||||
|
||||
pub(super) fn durable_sync(file: &File) -> io::Result<()> {
|
||||
// `File::sync_all` on Windows is `FlushFileBuffers`. Acceptable
|
||||
// for now; the bounded-syscall wrapper is not used here because
|
||||
// the stub also skips the worker-thread + leak machinery (the
|
||||
// wrapper would need an `unsafe impl Send` for `RawHandle`, and
|
||||
// designing that without a Windows test env is asking for it).
|
||||
file.sync_all()
|
||||
}
|
||||
+2
-2
@@ -26,7 +26,7 @@
|
||||
// The module wiring (registry hook + public re-export) is added
|
||||
// separately. Until then the parse/detect entry points have no
|
||||
use super::xml;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
@@ -60,7 +60,7 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
/// Read every `bdmt_<lang>.xml` under `/BDMV/META/DL/` and return the
|
||||
/// aggregated [`DiscMetadata`]. Returns `None` if no titles could be
|
||||
/// extracted from any file.
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<DiscMetadata> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<DiscMetadata> {
|
||||
let dir = udf.find_dir("/BDMV/META/DL")?;
|
||||
let mut out = DiscMetadata::default();
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
//! report. Surfaced via the labels-analyze tool — not part of the
|
||||
//! `analyze()` pipeline (no impact on the label output).
|
||||
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
@@ -116,7 +116,7 @@ impl ClpiVsMplsAudit {
|
||||
/// dedup-by-PID table of (CLPI fields, MPLS fields), return the
|
||||
/// merged view. Missing files (read errors, parse failures) are
|
||||
/// silently skipped — this is diagnostic, not correctness-critical.
|
||||
pub fn audit(reader: &mut dyn SectorReader, udf: &UdfFs) -> ClpiVsMplsAudit {
|
||||
pub fn audit(reader: &mut dyn SectorSource, udf: &UdfFs) -> ClpiVsMplsAudit {
|
||||
// Aggregate by PID across all CLPI files. If a PID appears in
|
||||
// multiple clips (typical — main movie clip + trailers reference
|
||||
// the same audio stream PIDs), first encountered wins (they should
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
//! stream number mapping via playbackconfig.
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, xml};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -12,7 +12,7 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "streamproperties.xml")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<ParseResult> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let sp_data = super::read_jar_file(reader, udf, "streamproperties.xml")?;
|
||||
let sp_text = std::str::from_utf8(&sp_data).ok()?;
|
||||
|
||||
|
||||
+5
-5
@@ -5,7 +5,7 @@
|
||||
//! menu_base.prop provides stream number → button name mapping.
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, vocab};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -14,7 +14,7 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
|| super::jar_file_exists(udf, "language_streams.txt")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<ParseResult> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
// Try language_streams.txt first (richer structured data)
|
||||
let ls_labels = parse_language_streams(reader, udf);
|
||||
|
||||
@@ -55,7 +55,7 @@ fn merge(ls: Vec<StreamLabel>, mb: Vec<StreamLabel>) -> Vec<StreamLabel> {
|
||||
|
||||
// ── language_streams.txt parser ────────────────────────────────────────────
|
||||
|
||||
fn parse_language_streams(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
fn parse_language_streams(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
let data = super::read_jar_file(reader, udf, "language_streams.txt")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
@@ -184,7 +184,7 @@ mod tests {
|
||||
|
||||
/// Build a minimal menu_base.prop text and run `parse_menu_base`'s
|
||||
/// inner logic via a temporary closure. This isolates the prop
|
||||
/// parsing without needing a SectorReader.
|
||||
/// parsing without needing a SectorSource.
|
||||
fn parse_props(text: &str) -> Vec<StreamLabel> {
|
||||
// Mirror the inner loop of parse_menu_base exactly. Kept
|
||||
// separate so the test doesn't need disc fixtures.
|
||||
@@ -338,7 +338,7 @@ mod tests {
|
||||
|
||||
// ── menu_base.prop parser ──────────────────────────────────────────────────
|
||||
|
||||
fn parse_menu_base(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
fn parse_menu_base(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<Vec<StreamLabel>> {
|
||||
let data = super::read_jar_file(reader, udf, "menu_base.prop")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
|
||||
+3
-3
@@ -35,11 +35,11 @@
|
||||
|
||||
use super::class_reader::CpInfo;
|
||||
use super::{ParseResult, StreamLabel, StreamLabelType, jar, vocab};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// dbp detect can't peek inside a jar without a SectorReader (the
|
||||
/// dbp detect can't peek inside a jar without a SectorSource (the
|
||||
/// trait function only takes `&UdfFs`), so we trigger on the cheap
|
||||
/// signal "any top-level .jar in /BDMV/JAR/." That fires on every
|
||||
/// BD-J disc, but parse() does the real `com/dbp/` check and
|
||||
@@ -49,7 +49,7 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
jar::has_any_top_level_jar(udf)
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<ParseResult> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
jar::for_each_jar(reader, udf, |_entry_name, archive| {
|
||||
if !jar::has_path_prefix(archive, "com/dbp/") {
|
||||
return None;
|
||||
|
||||
@@ -84,18 +84,18 @@ use super::class_reader::{
|
||||
ICONST_2, ICONST_3, ICONST_4, ICONST_5, ICONST_M1, INVOKESPECIAL, LDC, LDC_W, NEW, SIPUSH,
|
||||
};
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, jar, vocab};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
// Cheap pre-check at the dir level; the real signal is
|
||||
// `com/bydeluxe/` inside any top-level jar's central directory,
|
||||
// which `parse()` confirms when given a `SectorReader`.
|
||||
// which `parse()` confirms when given a `SectorSource`.
|
||||
jar::has_any_top_level_jar(udf)
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<ParseResult> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
jar::for_each_jar(reader, udf, |entry_name, archive| {
|
||||
if !jar::has_path_prefix(archive, "com/bydeluxe/") {
|
||||
return None;
|
||||
|
||||
+3
-3
@@ -13,7 +13,7 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use super::class_reader::ClassFile;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::io::Cursor;
|
||||
use zip::ZipArchive;
|
||||
@@ -26,7 +26,7 @@ pub type Jar = ZipArchive<Cursor<Vec<u8>>>;
|
||||
/// True if `/BDMV/JAR/` contains at least one top-level `.jar` file
|
||||
/// (not under a subdir). Used by `detect()` in parsers whose real
|
||||
/// signal lives inside a jar — they can't open the jar without a
|
||||
/// `SectorReader`, so they use this cheap pre-check and do the real
|
||||
/// `SectorSource`, so they use this cheap pre-check and do the real
|
||||
/// `com/<vendor>/` discriminator in `parse()`.
|
||||
pub fn has_any_top_level_jar(udf: &UdfFs) -> bool {
|
||||
let Some(jar_dir) = udf.find_dir("/BDMV/JAR") else {
|
||||
@@ -49,7 +49,7 @@ pub fn has_any_top_level_jar(udf: &UdfFs) -> bool {
|
||||
///
|
||||
/// Entries that fail to read from UDF or that aren't valid zips are
|
||||
/// silently skipped — same defensive shape as the existing dbp parser.
|
||||
pub fn for_each_jar<R, F>(reader: &mut dyn SectorReader, udf: &UdfFs, mut f: F) -> Option<R>
|
||||
pub fn for_each_jar<R, F>(reader: &mut dyn SectorSource, udf: &UdfFs, mut f: F) -> Option<R>
|
||||
where
|
||||
F: FnMut(&str, &mut Jar) -> Option<R>,
|
||||
{
|
||||
|
||||
+11
-12
@@ -4,7 +4,7 @@
|
||||
//! To add a new format:
|
||||
//! 1. Create `src/labels/myformat.rs`
|
||||
//! 2. Implement `pub fn detect(udf: &UdfFs) -> bool`
|
||||
//! 3. Implement `pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<Vec<StreamLabel>>`
|
||||
//! 3. Implement `pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<Vec<StreamLabel>>`
|
||||
//! 4. Add `mod myformat;` below and one line to `PARSERS` array
|
||||
|
||||
mod bdmt;
|
||||
@@ -18,13 +18,12 @@ pub(crate) mod jar;
|
||||
mod mpls_universal;
|
||||
mod paramount;
|
||||
mod pixelogic;
|
||||
mod png_filenames;
|
||||
pub(crate) mod text;
|
||||
pub mod vocab;
|
||||
pub(crate) mod xml;
|
||||
|
||||
use crate::disc::{DiscTitle, Stream};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
// Re-export bdmt's public type so callers can construct/inspect
|
||||
@@ -88,7 +87,7 @@ pub enum LabelQualifier {
|
||||
// to array order on confidence ties.
|
||||
|
||||
type DetectFn = fn(&UdfFs) -> bool;
|
||||
type ParseFn = fn(&mut dyn SectorReader, &UdfFs) -> Option<ParseResult>;
|
||||
type ParseFn = fn(&mut dyn SectorSource, &UdfFs) -> Option<ParseResult>;
|
||||
|
||||
/// Per-parser claim of how reliable its output is. Used by the
|
||||
/// registry to pick between parsers when more than one matches (e.g.
|
||||
@@ -176,7 +175,7 @@ const PARSERS: &[(&str, DetectFn, ParseFn)] = &[
|
||||
|
||||
/// Search disc for config files, extract labels, apply to streams.
|
||||
/// This is 100% optional — if anything fails, streams are untouched.
|
||||
pub fn apply(reader: &mut dyn SectorReader, udf: &UdfFs, titles: &mut [DiscTitle]) {
|
||||
pub fn apply(reader: &mut dyn SectorSource, udf: &UdfFs, titles: &mut [DiscTitle]) {
|
||||
let labels = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| extract(reader, udf)))
|
||||
.unwrap_or_default();
|
||||
if labels.is_empty() {
|
||||
@@ -192,7 +191,7 @@ pub fn apply(reader: &mut dyn SectorReader, udf: &UdfFs, titles: &mut [DiscTitle
|
||||
/// `forced` flag.
|
||||
///
|
||||
/// Extracted from `apply()` so the matching logic is unit-testable
|
||||
/// without needing a SectorReader / UdfFs.
|
||||
/// without needing a SectorSource / UdfFs.
|
||||
pub(crate) fn apply_labels(labels: &[StreamLabel], titles: &mut [DiscTitle]) {
|
||||
for title in titles.iter_mut() {
|
||||
let mut audio_idx: u16 = 0;
|
||||
@@ -370,7 +369,7 @@ fn generate_audio_label(
|
||||
}
|
||||
}
|
||||
|
||||
fn extract(reader: &mut dyn SectorReader, udf: &UdfFs) -> Vec<StreamLabel> {
|
||||
fn extract(reader: &mut dyn SectorSource, udf: &UdfFs) -> Vec<StreamLabel> {
|
||||
let mut best: Option<(&'static str, ParseResult)> = None;
|
||||
for (name, detect, parse) in PARSERS {
|
||||
if !detect(udf) {
|
||||
@@ -483,7 +482,7 @@ fn type_tag(t: StreamLabelType) -> u8 {
|
||||
/// authoring tool left out of the published playlist.
|
||||
fn append_clpi_orphans(
|
||||
labels: &mut Vec<StreamLabel>,
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf: &UdfFs,
|
||||
) -> usize {
|
||||
// Index existing labels by PID — but StreamLabel doesn't carry
|
||||
@@ -608,7 +607,7 @@ fn append_clpi_orphans(
|
||||
/// the return shape is richer (includes confidence, all detected
|
||||
/// parsers, and any parsers that produced empty results).
|
||||
#[doc(hidden)]
|
||||
pub fn analyze(reader: &mut dyn SectorReader, udf: &UdfFs) -> LabelAnalysis {
|
||||
pub fn analyze(reader: &mut dyn SectorSource, udf: &UdfFs) -> LabelAnalysis {
|
||||
let inventory = jar_inventory(udf);
|
||||
let mut parsers_detected: Vec<&'static str> = Vec::new();
|
||||
let mut all_results: Vec<(&'static str, ParseResult)> = Vec::new();
|
||||
@@ -699,7 +698,7 @@ pub fn analyze(reader: &mut dyn SectorReader, udf: &UdfFs) -> LabelAnalysis {
|
||||
/// playlist filename. Skipped entries (read error, parse error, no
|
||||
/// marks) silently dropped — this is a diagnostic field, not a
|
||||
/// correctness-critical one.
|
||||
fn collect_chapter_summary(reader: &mut dyn SectorReader, udf: &UdfFs) -> Vec<ChapterSummary> {
|
||||
fn collect_chapter_summary(reader: &mut dyn SectorSource, udf: &UdfFs) -> Vec<ChapterSummary> {
|
||||
let Some(playlist_dir) = udf.find_dir("/BDMV/PLAYLIST") else {
|
||||
return Vec::new();
|
||||
};
|
||||
@@ -842,7 +841,7 @@ pub(crate) fn find_jar_file(udf: &UdfFs, filename: &str) -> Option<String> {
|
||||
|
||||
/// Read a file from any BDMV/JAR subdirectory by filename.
|
||||
pub(crate) fn read_jar_file(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
udf: &UdfFs,
|
||||
filename: &str,
|
||||
) -> Option<Vec<u8>> {
|
||||
@@ -1073,7 +1072,7 @@ mod gap_fill_tests {
|
||||
// ── apply() integration tests ──────────────────────────────────────────────
|
||||
//
|
||||
// End-to-end coverage for the apply_labels + fill_defaults pipeline
|
||||
// without needing a SectorReader / UdfFs. Synthetic DiscTitle +
|
||||
// without needing a SectorSource / UdfFs. Synthetic DiscTitle +
|
||||
// StreamLabel inputs, assert on the resulting Stream field values.
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -24,7 +24,7 @@ use super::{
|
||||
LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType,
|
||||
vocab::{self, LangInfo},
|
||||
};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
/// True iff `/BDMV/PLAYLIST/` exists and contains at least one
|
||||
@@ -45,7 +45,7 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
/// Returns `None` if no labels could be produced (e.g. no .mpls files
|
||||
/// parsed successfully, or every parsed stream was a type we skip
|
||||
/// like IG / DV EL).
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<ParseResult> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let playlist_dir = udf.find_dir("/BDMV/PLAYLIST")?;
|
||||
|
||||
// Collect mpls filenames first so we don't hold a borrow on udf
|
||||
|
||||
@@ -13,14 +13,14 @@
|
||||
//! ```
|
||||
|
||||
use super::{LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, xml};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "playlists.xml")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<ParseResult> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let data = super::read_jar_file(reader, udf, "playlists.xml")?;
|
||||
let text = std::str::from_utf8(&data).ok()?;
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ use super::{
|
||||
Confidence, LabelPurpose, LabelQualifier, ParseResult, StreamLabel, StreamLabelType, text,
|
||||
vocab,
|
||||
};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use crate::udf::UdfFs;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
@@ -24,7 +24,7 @@ pub fn detect(udf: &UdfFs) -> bool {
|
||||
super::jar_file_exists(udf, "bluray_project.bin")
|
||||
}
|
||||
|
||||
pub fn parse(reader: &mut dyn SectorReader, udf: &UdfFs) -> Option<ParseResult> {
|
||||
pub fn parse(reader: &mut dyn SectorSource, udf: &UdfFs) -> Option<ParseResult> {
|
||||
let data = super::read_jar_file(reader, udf, "bluray_project.bin")?;
|
||||
// min_len=4 matches the prior local extract_strings impl. The token
|
||||
// grammar is `{lang3}_{codec?}_{purpose?}_{region?}_` so the
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
//! PNG-filename language token parser — stubbed (noop) pending need.
|
||||
//!
|
||||
//! ## What this would do
|
||||
//!
|
||||
//! Some discs encode per-language menu localization as pre-rendered PNG
|
||||
//! menu buttons, one per language, with the language token embedded in
|
||||
//! the filename. Examples observed in the 2026-05-10 corpus:
|
||||
//!
|
||||
//! - **disc-01 (The Amateur)** — `<region>_<lang>_<context>_<format>.png`
|
||||
//! Region prefix: `USA` / `UK` / `JPN` / etc.
|
||||
//! Lang tokens (3-char, uppercase): `ENG`, `FRC`, `FRP`, `DEU`, `DUT`,
|
||||
//! `ITA`, `JPN`, `LAS`, `CSP`, `POL`, `CZE` (11 languages)
|
||||
//!
|
||||
//! - **disc-09 (Dune orig)** — `<title>_<variant>_<lang>_Composite<N>.png`
|
||||
//! Lang tokens (3-char, mixed-case): `Eng`, `Ger` (2 languages)
|
||||
//!
|
||||
//! ## Why stubbed
|
||||
//!
|
||||
//! MPLS already gives per-stream `language` + `coding_type` + stream-type
|
||||
//! (audio vs subtitle) on every disc. For the 2 unknown-framework discs
|
||||
//! that PNG filenames would close (disc-01, disc-09), MPLS will produce
|
||||
//! a strict superset of what filenames could give us, because MPLS knows
|
||||
//! per-stream attribution while filenames only know "the disc offers
|
||||
//! these N language buttons."
|
||||
//!
|
||||
//! The **only** thing PNG filenames give us that MPLS doesn't is **studio
|
||||
//! variant disambiguation**:
|
||||
//! - `FRC` (French Canadian) vs `FRP` (French Parisian) — MPLS just says `fra`
|
||||
//! - `LAS` (Latin American Spanish) vs `CSP` (Castilian Spanish) — MPLS just says `spa`
|
||||
//!
|
||||
//! That's niche enough that it doesn't justify implementing right now.
|
||||
//! Reactivate this parser only when:
|
||||
//! 1. We hit a disc where MPLS is malformed/empty AND PNG filenames are
|
||||
//! the only language hint, OR
|
||||
//! 2. A downstream consumer needs the studio variant suffix for output
|
||||
//! naming (e.g. `Title (French Canadian).mkv` vs `Title (French).mkv`).
|
||||
//!
|
||||
//! ## When reactivating
|
||||
//!
|
||||
//! Implement `parse` to:
|
||||
//! 1. Iterate top-level PNG paths in `/BDMV/JAR/` (and `<id>/` subdirs).
|
||||
//! 2. Tokenize each filename on `_` / `-` / `.`
|
||||
//! 3. Match each token against an alias table:
|
||||
//! - ISO 639-1 / 639-2 standard codes
|
||||
//! - Studio variants: `FRC`/`FRP` → `fra-CA`/`fra-FR`,
|
||||
//! `LAS`/`CSP` → `spa-419`/`spa-ES`,
|
||||
//! mixed-case shortforms `Eng`/`Ger`/`Fra`/`Spa`/`Jpn` → ISO 639-2
|
||||
//! - Country prefix filter: drop `USA`/`UK`/`JPN`/`AUS`/`GER`/`FR` when
|
||||
//! they appear in position 0 (those are region markers, not langs).
|
||||
//! 4. Deduplicate. Confidence stays `Low` because we still don't know
|
||||
//! per-stream codec or audio/subtitle attribution.
|
||||
//!
|
||||
//! Wire as ENRICHMENT after MPLS in `mod.rs::analyze`, not as a primary
|
||||
//! parser: PNG filenames upgrade `lang=fra` to `lang=fra-CA` when both
|
||||
//! sources agree on the disc; they should never overwrite MPLS data.
|
||||
|
||||
use super::ParseResult;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::udf::UdfFs;
|
||||
|
||||
/// Stub: returns false so the dispatcher never calls `parse`. Reactivate
|
||||
/// by checking for the patterns described in the module docs.
|
||||
#[allow(dead_code)] // module-level noop, not wired into PARSERS until needed
|
||||
pub fn detect(_udf: &UdfFs) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Stub: returns None. See module docs for the implementation sketch.
|
||||
#[allow(dead_code)] // module-level noop, not wired into PARSERS until needed
|
||||
pub fn parse(_reader: &mut dyn SectorReader, _udf: &UdfFs) -> Option<ParseResult> {
|
||||
None
|
||||
}
|
||||
+10
-17
@@ -165,8 +165,8 @@ pub use decrypt::{DecryptKeys, decrypt_sectors};
|
||||
// structured representation; never compare against display strings.
|
||||
// Note: `disc::Stream` here is the codec enum (audio / video / sub kind)
|
||||
// — not the `pes::Stream` trait re-exported below as `PesStream`. Two
|
||||
// different concepts, the same short name; both stay because both are
|
||||
// load-bearing in their respective domains.
|
||||
// different concepts, the same short name; the trait gets the `Pes`
|
||||
// prefix at the crate root to keep both addressable.
|
||||
pub use disc::{
|
||||
AacsState, AudioChannels, AudioStream, Clip, Codec, ColorSpace, ContentFormat, DamageSeverity,
|
||||
Disc, DiscFormat, DiscId, DiscTitle, Extent, FrameRate, HdrFormat, KeySource, LabelPurpose,
|
||||
@@ -179,7 +179,7 @@ pub use disc::{
|
||||
// All stream types implement `pes::Stream` — read PES frames from a source,
|
||||
// write PES frames to a sink. Pick the right type at construction:
|
||||
//
|
||||
// - `DiscStream` — physical drive or ISO (any `SectorReader`). Always read.
|
||||
// - `DiscStream` — physical drive or ISO (any `SectorSource`). Read-only.
|
||||
// - `MkvStream` — Matroska container. Read on `open()`, write on `create()`.
|
||||
// - `M2tsStream` — Blu-ray Transport Stream. Read on `open()`, write on `create()`.
|
||||
// - `NetworkStream` — TCP. Read on `listen()`, write on `connect()`.
|
||||
@@ -189,14 +189,11 @@ pub use disc::{
|
||||
// Most consumers use the URL resolvers (`input()` / `output()`) which pick
|
||||
// the right type from a scheme:// URL. Direct construction is for callers
|
||||
// that need to wire custom readers (e.g. autorip's drive-session reuse).
|
||||
// 0.18 trait split: `FrameSource` (read-only) and `FrameSink` (write-only)
|
||||
// supersede the unified `pes::Stream`. The old `Stream` re-export below
|
||||
// stays available for the deprecation window — re-exported as
|
||||
// `PesStream` to disambiguate from `disc::Stream` (the codec-kind enum
|
||||
// re-exported above), which would otherwise collide at the crate root.
|
||||
#[allow(deprecated)]
|
||||
// The trait is re-exported as `PesStream` here to disambiguate from
|
||||
// `disc::Stream` (the codec-kind enum re-exported above), which would
|
||||
// otherwise collide at the crate root.
|
||||
pub use pes::PesFrame;
|
||||
pub use pes::Stream as PesStream;
|
||||
pub use pes::{FrameSink, FrameSource, PesFrame};
|
||||
|
||||
pub use mux::DiscStream;
|
||||
pub use mux::M2tsStream;
|
||||
@@ -209,18 +206,14 @@ pub use mux::{InputOptions, StreamUrl, input, output, parse_url};
|
||||
// ─── Lower-level surfaces ───────────────────────────────────────────────────
|
||||
//
|
||||
// `ScsiTransport` is the platform-abstraction trait Drive uses; expose for
|
||||
// out-of-tree platform backends. `SectorSource` / `SectorSink` are the 0.18
|
||||
// out-of-tree platform backends. `SectorSource` / `SectorSink` are the
|
||||
// direction-typed read/write traits; `FileSectorSource` and `FileSectorSink`
|
||||
// are the ISO-on-disk implementations. [`DecryptingSectorSource`] is the
|
||||
// single decrypt-on-read decorator (AACS / CSS / none) — wrap any
|
||||
// `SectorSource` to get plaintext sectors out. The legacy `SectorReader` /
|
||||
// `FileSectorReader` names stay re-exported through the 0.18 migration
|
||||
// window so existing call sites compile unchanged; a blanket impl makes
|
||||
// every `SectorReader` automatically usable as a `SectorSource`.
|
||||
// `SectorSource` to get plaintext sectors out.
|
||||
pub use scsi::{DriveInfo, ScsiSense, ScsiTransport, drive_has_disc, list_drives};
|
||||
pub use sector::{
|
||||
DecryptingSectorSource, FileSectorReader, FileSectorSink, FileSectorSource, SectorReader,
|
||||
SectorSink, SectorSource,
|
||||
DecryptingSectorSource, FileSectorSink, FileSectorSource, SectorSink, SectorSource,
|
||||
};
|
||||
pub use speed::DriveSpeed;
|
||||
pub use udf::{UdfFs, read_filesystem};
|
||||
|
||||
+40
-32
@@ -1,6 +1,6 @@
|
||||
//! DiscStream — read any disc (physical drive or ISO file) → PES frames.
|
||||
//!
|
||||
//! One stream type for all disc sources. The source is a SectorReader —
|
||||
//! One stream type for all disc sources. The source is a SectorSource —
|
||||
//! Drive (hardware) or IsoSectorReader (file). DiscStream doesn't care.
|
||||
//!
|
||||
//! Read-only. For disc→ISO (raw sector copy), use `Disc::copy()`.
|
||||
@@ -9,7 +9,7 @@ use crate::disc::{Disc, DiscTitle, Extent};
|
||||
use crate::drive::extract_scsi_context;
|
||||
use crate::event::{BatchSizeReason, Event, EventKind};
|
||||
use crate::halt::Halt;
|
||||
use crate::sector::{DecryptingSectorSource, SectorReader, SectorSource};
|
||||
use crate::sector::{DecryptingSectorSource, SectorSource};
|
||||
use std::io;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
@@ -97,7 +97,7 @@ impl AdaptiveBatch {
|
||||
|
||||
/// Disc stream. Reads sectors from any source → PES frames.
|
||||
///
|
||||
/// Sources: physical drive, ISO file, or any SectorReader.
|
||||
/// Sources: physical drive, ISO file, or any SectorSource.
|
||||
/// Decrypt, demux, and codec parsing happen internally.
|
||||
pub struct DiscStream {
|
||||
/// Underlying sector source wrapped in the 0.18
|
||||
@@ -105,7 +105,7 @@ pub struct DiscStream {
|
||||
/// call yields plaintext, so `fill_extents` no longer needs an
|
||||
/// inline `decrypt::decrypt_sectors` step. `DecryptKeys::None`
|
||||
/// (raw / unencrypted disc) makes the decorator a pass-through.
|
||||
reader: DecryptingSectorSource<Box<dyn SectorReader>>,
|
||||
reader: DecryptingSectorSource<Box<dyn SectorSource>>,
|
||||
title: DiscTitle,
|
||||
disc: Option<Disc>,
|
||||
/// Mirror of the keys handed in at construction. The decorator
|
||||
@@ -160,11 +160,11 @@ pub struct DiscStream {
|
||||
impl DiscStream {
|
||||
/// Create a disc stream from any sector reader.
|
||||
///
|
||||
/// Works with physical drives and ISO files — both implement SectorReader.
|
||||
/// Works with physical drives and ISO files — both implement SectorSource.
|
||||
/// The caller opens the source, scans for titles/keys, and passes them in.
|
||||
/// The stream handles demuxing, decryption, and codec parsing internally.
|
||||
pub fn new(
|
||||
reader: Box<dyn SectorReader>,
|
||||
reader: Box<dyn SectorSource>,
|
||||
title: DiscTitle,
|
||||
decrypt_keys: crate::decrypt::DecryptKeys,
|
||||
batch_sectors: u16,
|
||||
@@ -173,6 +173,13 @@ impl DiscStream {
|
||||
let extents = title.extents.clone();
|
||||
let bytes_total_extents: u64 = extents.iter().map(|e| e.sector_count as u64 * 2048).sum();
|
||||
|
||||
// Debug log reader type at construction — critical for diagnosing mux reading from drive instead of ISO
|
||||
tracing::debug!(
|
||||
target: "mux",
|
||||
"DiscStream constructed with reader type: {}",
|
||||
std::any::type_name::<dyn SectorSource>()
|
||||
);
|
||||
|
||||
let mut pids = Vec::new();
|
||||
let mut parsers = Vec::new();
|
||||
let mut pid_to_track = Vec::new();
|
||||
@@ -315,10 +322,19 @@ impl DiscStream {
|
||||
// this loop can spend minutes shrinking and skipping sectors; without
|
||||
// the check, Stop wouldn't take effect until the outer PES read() loop
|
||||
// finally emits a frame, which may never happen.
|
||||
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
loop {
|
||||
if self.is_halted() {
|
||||
return Err(crate::error::Error::Halted.into());
|
||||
}
|
||||
|
||||
// Debug: log slow reads during mux — helps diagnose stalls
|
||||
if cfg!(debug_assertions) && start_time.elapsed().as_secs() > 5 {
|
||||
tracing::debug!(target: "mux", "fill_extents waiting at LBA {} ({}s elapsed, sectors={})", lba, start_time.elapsed().as_secs(), remaining);
|
||||
}
|
||||
|
||||
let mut sectors = remaining.min(self.adaptive.current() as u32) as u16;
|
||||
// Align to 3-sector AACS units when possible. Partial units at
|
||||
// extent boundaries are safely handled by decrypt_sectors().
|
||||
@@ -387,7 +403,6 @@ impl DiscStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource/FrameSink in follow-up commit.
|
||||
impl crate::pes::Stream for DiscStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
if let Some(frame) = self.pending_frames.pop_front() {
|
||||
@@ -568,31 +583,26 @@ impl crate::pes::Stream for DiscStream {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
//! `DiscStream` is the only meaningful `FrameSource` impl in tree (every
|
||||
//! other concrete `pes::Stream` impl in `mux/*` is a sink). The 0.18
|
||||
//! round-1 blanket `impl<T: pes::Stream + Send> pes::FrameSource for T`
|
||||
//! covers `DiscStream` for free as long as it is `Send`. These tests
|
||||
//! lock that down: a static `Send` assertion plus a `Box<dyn FrameSource>`
|
||||
//! round trip exercising every `FrameSource` method through the trait
|
||||
//! object, so future Send-breaking edits to `DiscStream`'s interior
|
||||
//! types fail at compile time and the trait-bridge dispatch is verified
|
||||
//! at runtime.
|
||||
#![allow(deprecated)] // exercising the 0.18 deprecation-window blanket bridge.
|
||||
//! `DiscStream` is the only read-only `Stream` impl in tree (every
|
||||
//! other concrete impl in `mux/*` is bidirectional or write-only).
|
||||
//! These tests lock down a static `Send` assertion plus a
|
||||
//! `Box<dyn Stream>` round trip exercising every method through the
|
||||
//! trait object, so future Send-breaking edits to `DiscStream`'s
|
||||
//! interior types fail at compile time.
|
||||
use super::*;
|
||||
use crate::disc::{ContentFormat, DiscTitle};
|
||||
use crate::pes::FrameSource;
|
||||
use crate::pes::Stream;
|
||||
|
||||
/// Static-assert `DiscStream: Send`. The blanket
|
||||
/// `impl<T: pes::Stream + Send> pes::FrameSource for T` only fires for
|
||||
/// `Send` types — if a future field on `DiscStream` is non-`Send` (e.g.
|
||||
/// a `Box<dyn Read>` instead of `Box<dyn SectorReader>`), this fails
|
||||
/// Static-assert `DiscStream: Send`. The `Stream` trait has `Send` as a
|
||||
/// supertrait — if a future field on `DiscStream` is non-`Send` (e.g.
|
||||
/// a `Box<dyn Read>` instead of `Box<dyn SectorSource>`), this fails
|
||||
/// at compile time, before the runtime trait-object test below.
|
||||
fn _assert_disc_stream_is_send() {
|
||||
fn requires_send<T: Send>() {}
|
||||
requires_send::<DiscStream>();
|
||||
}
|
||||
|
||||
/// Trivial `SectorReader` that yields zeroed sectors. Empty title means
|
||||
/// Trivial `SectorSource` that yields zeroed sectors. Empty title means
|
||||
/// the demuxer produces no PES frames, so `read()` walks the extents to
|
||||
/// EOF and returns `Ok(None)`. That's enough to exercise the trait-object
|
||||
/// dispatch — the goal here is the bridge, not the demuxer.
|
||||
@@ -600,7 +610,7 @@ mod tests {
|
||||
capacity: u32,
|
||||
}
|
||||
|
||||
impl crate::sector::SectorReader for ZeroReader {
|
||||
impl crate::sector::SectorSource for ZeroReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
_lba: u32,
|
||||
@@ -613,7 +623,7 @@ mod tests {
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn capacity(&self) -> u32 {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
@@ -628,13 +638,11 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Smallest credible witness that `DiscStream` flows through the
|
||||
/// `FrameSource` blanket impl: build a `Box<dyn FrameSource>`, drive
|
||||
/// `read()` to EOF, exercise `info()` / `headers_ready()` /
|
||||
/// `codec_private()` through the trait object. The trait-bridge
|
||||
/// correctness is what's being verified — not demuxer behaviour.
|
||||
/// Smallest credible witness that `DiscStream` flows through `dyn Stream`:
|
||||
/// build a `Box<dyn Stream>`, drive `read()` to EOF, exercise `info()` /
|
||||
/// `headers_ready()` / `codec_private()` through the trait object.
|
||||
#[test]
|
||||
fn frame_source_via_dyn_object() {
|
||||
fn stream_via_dyn_object() {
|
||||
let reader = ZeroReader { capacity: 8 };
|
||||
let title = synthetic_title(8);
|
||||
let stream = DiscStream::new(
|
||||
@@ -645,7 +653,7 @@ mod tests {
|
||||
ContentFormat::BdTs,
|
||||
);
|
||||
|
||||
let mut src: Box<dyn FrameSource> = Box::new(stream);
|
||||
let mut src: Box<dyn Stream> = Box::new(stream);
|
||||
|
||||
// Empty-title fixture has no streams configured, so headers are
|
||||
// trivially ready and codec_private() yields nothing on track 0.
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
//! Fragmented MP4 muxer — **stub** for Phase 3.
|
||||
//!
|
||||
//! Goal: ISO/IEC 14496-12 fragmented MP4 (`ftyp` + `moov` init segment,
|
||||
//! then a sequence of `moof+mdat` media fragments) targeting a
|
||||
//! [`SequentialSink`](crate::io::sink::SequentialSink). DASH-friendly,
|
||||
//! no Cues backpatch.
|
||||
//!
|
||||
//! Status (v0.21.0 Phase 3): **STUB**. We ship the init segment
|
||||
//! (`ftyp` + a minimal HEVC `moov` skeleton with one video track) so
|
||||
//! the muxer's shape and call site are validated, but media fragments
|
||||
//! are NOT yet emitted — calls to [`Fmp4Mux::write_video`] currently
|
||||
//! accumulate frames into an internal buffer and discard them on
|
||||
//! [`Fmp4Mux::finish`].
|
||||
//!
|
||||
//! ## What's TODO (tracked in Phase 4 / v0.22.0 scope)
|
||||
//!
|
||||
//! - `moof` box: `mfhd` (sequence_number) + `traf` (`tfhd` + `tfdt`
|
||||
//! + `trun` with sample sizes, durations, flags, composition offsets).
|
||||
//! - `mdat` box: concatenated sample data.
|
||||
//! - Fragment cadence: one fragment per GOP or every N seconds,
|
||||
//! whichever comes first.
|
||||
//! - HEVC `hvcC` box inside `moov.trak.mdia.minf.stbl.stsd` so the
|
||||
//! init segment is self-describing.
|
||||
//! - Sample-flags computation (sync vs. delta, depends_on, etc.).
|
||||
//! - Edit lists / fragment_duration for accurate seeking.
|
||||
//!
|
||||
//! Reference: ISO/IEC 14496-12 §8 (Movie Fragments).
|
||||
|
||||
use std::io::{self, Write};
|
||||
|
||||
// Box type literals — four-character codes per ISO/IEC 14496-12 §4.2.
|
||||
|
||||
const FTYP: [u8; 4] = *b"ftyp";
|
||||
const MOOV: [u8; 4] = *b"moov";
|
||||
const MVHD: [u8; 4] = *b"mvhd";
|
||||
const TRAK: [u8; 4] = *b"trak";
|
||||
const TKHD: [u8; 4] = *b"tkhd";
|
||||
const MDIA: [u8; 4] = *b"mdia";
|
||||
const MDHD: [u8; 4] = *b"mdhd";
|
||||
const HDLR: [u8; 4] = *b"hdlr";
|
||||
const MINF: [u8; 4] = *b"minf";
|
||||
const VMHD: [u8; 4] = *b"vmhd";
|
||||
const DINF: [u8; 4] = *b"dinf";
|
||||
const DREF: [u8; 4] = *b"dref";
|
||||
const URL_: [u8; 4] = *b"url ";
|
||||
const STBL: [u8; 4] = *b"stbl";
|
||||
const STSD: [u8; 4] = *b"stsd";
|
||||
const STTS: [u8; 4] = *b"stts";
|
||||
const STSC: [u8; 4] = *b"stsc";
|
||||
const STSZ: [u8; 4] = *b"stsz";
|
||||
const STCO: [u8; 4] = *b"stco";
|
||||
const MVEX: [u8; 4] = *b"mvex";
|
||||
const TREX: [u8; 4] = *b"trex";
|
||||
|
||||
/// Default movie timescale — 90 kHz lines up with MPEG-TS PTS and the
|
||||
/// HEVC SPS `vui_time_scale` for film content, simplifying the math
|
||||
/// when fragment emission lands.
|
||||
const MOVIE_TIMESCALE: u32 = 90_000;
|
||||
/// Video track ID. fMP4 init segments conventionally use track_ID=1
|
||||
/// for the primary video track; a single-track DASH representation has
|
||||
/// no reason to deviate.
|
||||
const VIDEO_TRACK_ID: u32 = 1;
|
||||
|
||||
/// Fragmented MP4 muxer — stub.
|
||||
///
|
||||
/// See the module-level doc comment for what is and isn't shipped in
|
||||
/// this stub.
|
||||
pub struct Fmp4Mux<W: Write> {
|
||||
writer: W,
|
||||
header_written: bool,
|
||||
/// Pending frames — held for the future fragment-emit path. The
|
||||
/// stub drops these on `finish` but keeping them around lets the
|
||||
/// post-stub work re-attach without changing the public API.
|
||||
pending: Vec<PendingSample>,
|
||||
/// hvcC bytes, if provided. Embedded in the `moov.…stsd.hvc1.hvcC`
|
||||
/// box once that path lands.
|
||||
#[allow(dead_code)]
|
||||
codec_private: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
struct PendingSample {
|
||||
#[allow(dead_code)]
|
||||
pts_ns: i64,
|
||||
#[allow(dead_code)]
|
||||
keyframe: bool,
|
||||
#[allow(dead_code)]
|
||||
data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl<W: Write> Fmp4Mux<W> {
|
||||
pub fn new(writer: W) -> Self {
|
||||
Self {
|
||||
writer,
|
||||
header_written: false,
|
||||
pending: Vec::new(),
|
||||
codec_private: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Provide the `HEVCDecoderConfigurationRecord` for the video track.
|
||||
/// The stub stores it but doesn't yet embed it in `moov` — that's
|
||||
/// part of the post-stub work.
|
||||
pub fn set_video_codec_private(&mut self, hvcc: Vec<u8>) {
|
||||
self.codec_private = Some(hvcc);
|
||||
}
|
||||
|
||||
/// Write one video PES frame.
|
||||
///
|
||||
/// **Stub behaviour:** the first call emits the init segment
|
||||
/// (`ftyp` + `moov`) so any consumer that just wants the shape can
|
||||
/// receive it. Subsequent calls accumulate frames in memory for
|
||||
/// the future fragmenting path; **no media bytes are written yet**.
|
||||
pub fn write_video(&mut self, pts_ns: i64, keyframe: bool, data: &[u8]) -> io::Result<()> {
|
||||
if !self.header_written {
|
||||
self.write_init_segment()?;
|
||||
self.header_written = true;
|
||||
}
|
||||
// TODO(0.22.0): emit one `moof+mdat` per GOP. For now stash the
|
||||
// frame so the future patch can hot-wire emission without API
|
||||
// churn.
|
||||
self.pending.push(PendingSample {
|
||||
pts_ns,
|
||||
keyframe,
|
||||
data: data.to_vec(),
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush. The stub additionally drops accumulated `pending` frames.
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
// TODO(0.22.0): emit final fragment from pending; today the
|
||||
// stub just clears the buffer to release memory.
|
||||
self.pending.clear();
|
||||
self.writer.flush()
|
||||
}
|
||||
|
||||
fn write_init_segment(&mut self) -> io::Result<()> {
|
||||
let ftyp = build_ftyp();
|
||||
let moov = build_moov();
|
||||
self.writer.write_all(&ftyp)?;
|
||||
self.writer.write_all(&moov)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the `ftyp` box. `major_brand = "iso6"`, `minor_version = 1`,
|
||||
/// compatible brands `iso6 dash msdh hvc1` — the same conservative set
|
||||
/// shaka-packager uses for HEVC-in-fMP4 outputs.
|
||||
fn build_ftyp() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(b"iso6");
|
||||
body.extend_from_slice(&1u32.to_be_bytes());
|
||||
body.extend_from_slice(b"iso6");
|
||||
body.extend_from_slice(b"dash");
|
||||
body.extend_from_slice(b"msdh");
|
||||
body.extend_from_slice(b"hvc1");
|
||||
wrap_box(&FTYP, &body)
|
||||
}
|
||||
|
||||
/// Build the `moov` box — minimal skeleton. Single video trak, no
|
||||
/// hvcC inside stsd yet (TODO: full hvc1 sample entry).
|
||||
fn build_moov() -> Vec<u8> {
|
||||
let mvhd = build_mvhd();
|
||||
let trak = build_video_trak();
|
||||
let mvex = build_mvex();
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&mvhd);
|
||||
body.extend_from_slice(&trak);
|
||||
body.extend_from_slice(&mvex);
|
||||
wrap_box(&MOOV, &body)
|
||||
}
|
||||
|
||||
fn build_mvhd() -> Vec<u8> {
|
||||
// Version 0, 100 bytes total body. Fields per ISO/IEC 14496-12 §8.2.2.
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0, 0, 0, 0]); // version + flags
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // creation_time
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // modification_time
|
||||
body.extend_from_slice(&MOVIE_TIMESCALE.to_be_bytes());
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // duration = 0 (fragmented)
|
||||
body.extend_from_slice(&0x0001_0000u32.to_be_bytes()); // rate 1.0
|
||||
body.extend_from_slice(&0x0100u16.to_be_bytes()); // volume 1.0
|
||||
body.extend_from_slice(&[0u8; 2]); // reserved
|
||||
body.extend_from_slice(&[0u8; 8]); // reserved
|
||||
// 3x3 identity transformation matrix in 16.16 fixed point.
|
||||
for v in [0x1_0000u32, 0, 0, 0, 0x1_0000, 0, 0, 0, 0x4000_0000] {
|
||||
body.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
body.extend_from_slice(&[0u8; 24]); // pre_defined[6]
|
||||
body.extend_from_slice(&2u32.to_be_bytes()); // next_track_ID (1 reserved for video)
|
||||
wrap_box(&MVHD, &body)
|
||||
}
|
||||
|
||||
fn build_video_trak() -> Vec<u8> {
|
||||
let tkhd = build_tkhd();
|
||||
let mdia = build_mdia();
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&tkhd);
|
||||
body.extend_from_slice(&mdia);
|
||||
wrap_box(&TRAK, &body)
|
||||
}
|
||||
|
||||
fn build_tkhd() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
// version=0 | flags=0x000007 (track_enabled | in_movie | in_preview)
|
||||
body.extend_from_slice(&[0, 0, 0, 7]);
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // creation_time
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // modification_time
|
||||
body.extend_from_slice(&VIDEO_TRACK_ID.to_be_bytes());
|
||||
body.extend_from_slice(&[0u8; 4]); // reserved
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // duration
|
||||
body.extend_from_slice(&[0u8; 8]); // reserved
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // layer
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // alternate_group
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // volume (video=0)
|
||||
body.extend_from_slice(&[0u8; 2]); // reserved
|
||||
// 3x3 identity matrix.
|
||||
for v in [0x1_0000u32, 0, 0, 0, 0x1_0000, 0, 0, 0, 0x4000_0000] {
|
||||
body.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
// width / height in 16.16 fixed point — placeholder 1920x1080.
|
||||
body.extend_from_slice(&(1920u32 << 16).to_be_bytes());
|
||||
body.extend_from_slice(&(1080u32 << 16).to_be_bytes());
|
||||
wrap_box(&TKHD, &body)
|
||||
}
|
||||
|
||||
fn build_mdia() -> Vec<u8> {
|
||||
let mdhd = build_mdhd();
|
||||
let hdlr = build_hdlr_vide();
|
||||
let minf = build_minf();
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&mdhd);
|
||||
body.extend_from_slice(&hdlr);
|
||||
body.extend_from_slice(&minf);
|
||||
wrap_box(&MDIA, &body)
|
||||
}
|
||||
|
||||
fn build_mdhd() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0, 0, 0, 0]); // version + flags
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // creation_time
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // modification_time
|
||||
body.extend_from_slice(&MOVIE_TIMESCALE.to_be_bytes());
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // duration
|
||||
// language: 'und' in 5-bit-per-char ISO 639-2 packed (bit 15 = 0).
|
||||
body.extend_from_slice(&[0x55, 0xC4]);
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // pre_defined
|
||||
wrap_box(&MDHD, &body)
|
||||
}
|
||||
|
||||
fn build_hdlr_vide() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0, 0, 0, 0]); // version + flags
|
||||
body.extend_from_slice(&0u32.to_be_bytes()); // pre_defined
|
||||
body.extend_from_slice(b"vide");
|
||||
body.extend_from_slice(&[0u8; 12]); // reserved
|
||||
body.extend_from_slice(b"VideoHandler\0");
|
||||
wrap_box(&HDLR, &body)
|
||||
}
|
||||
|
||||
fn build_minf() -> Vec<u8> {
|
||||
let vmhd = build_vmhd();
|
||||
let dinf = build_dinf();
|
||||
let stbl = build_stbl();
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&vmhd);
|
||||
body.extend_from_slice(&dinf);
|
||||
body.extend_from_slice(&stbl);
|
||||
wrap_box(&MINF, &body)
|
||||
}
|
||||
|
||||
fn build_vmhd() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0, 0, 0, 1]); // version + flags=1
|
||||
body.extend_from_slice(&0u16.to_be_bytes()); // graphicsmode
|
||||
body.extend_from_slice(&[0u8; 6]); // opcolor
|
||||
wrap_box(&VMHD, &body)
|
||||
}
|
||||
|
||||
fn build_dinf() -> Vec<u8> {
|
||||
let mut dref_body = Vec::new();
|
||||
dref_body.extend_from_slice(&[0, 0, 0, 0]);
|
||||
dref_body.extend_from_slice(&1u32.to_be_bytes()); // entry_count
|
||||
// url with flags=1 (self-contained) and zero name.
|
||||
let url_body = [0u8, 0, 0, 1];
|
||||
dref_body.extend_from_slice(&wrap_box(&URL_, &url_body));
|
||||
let dref = wrap_box(&DREF, &dref_body);
|
||||
wrap_box(&DINF, &dref)
|
||||
}
|
||||
|
||||
fn build_stbl() -> Vec<u8> {
|
||||
// Stub stsd: empty sample description (zero entries). Replace with
|
||||
// hvc1+hvcC once the fragmenting path lands so the init segment is
|
||||
// actually decodable.
|
||||
let mut stsd_body = Vec::new();
|
||||
stsd_body.extend_from_slice(&[0, 0, 0, 0]);
|
||||
stsd_body.extend_from_slice(&0u32.to_be_bytes()); // entry_count
|
||||
let stsd = wrap_box(&STSD, &stsd_body);
|
||||
|
||||
// Empty stts/stsc/stsz/stco — fragmented init has no samples here.
|
||||
let stts = wrap_box(&STTS, &[0, 0, 0, 0, 0, 0, 0, 0]); // version+flags, count=0
|
||||
let stsc = wrap_box(&STSC, &[0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
let stsz = wrap_box(
|
||||
&STSZ,
|
||||
&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], // version+flags, sample_size=0, count=0
|
||||
);
|
||||
let stco = wrap_box(&STCO, &[0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&stsd);
|
||||
body.extend_from_slice(&stts);
|
||||
body.extend_from_slice(&stsc);
|
||||
body.extend_from_slice(&stsz);
|
||||
body.extend_from_slice(&stco);
|
||||
wrap_box(&STBL, &body)
|
||||
}
|
||||
|
||||
fn build_mvex() -> Vec<u8> {
|
||||
// trex: track_ID=1, default_sample_description_index=1, others=0.
|
||||
let mut trex_body = Vec::new();
|
||||
trex_body.extend_from_slice(&[0, 0, 0, 0]); // version + flags
|
||||
trex_body.extend_from_slice(&VIDEO_TRACK_ID.to_be_bytes());
|
||||
trex_body.extend_from_slice(&1u32.to_be_bytes()); // default_sample_description_index
|
||||
trex_body.extend_from_slice(&0u32.to_be_bytes()); // default_sample_duration
|
||||
trex_body.extend_from_slice(&0u32.to_be_bytes()); // default_sample_size
|
||||
trex_body.extend_from_slice(&0u32.to_be_bytes()); // default_sample_flags
|
||||
let trex = wrap_box(&TREX, &trex_body);
|
||||
wrap_box(&MVEX, &trex)
|
||||
}
|
||||
|
||||
/// Wrap a box body in `[size:u32-BE][type:4]`. Suitable for any body
|
||||
/// that fits in u32; oversized boxes (size > 4 GiB) need the 64-bit
|
||||
/// large-size extension which we don't generate in the stub.
|
||||
fn wrap_box(box_type: &[u8; 4], body: &[u8]) -> Vec<u8> {
|
||||
let size = (body.len() + 8) as u32;
|
||||
let mut out = Vec::with_capacity(body.len() + 8);
|
||||
out.extend_from_slice(&size.to_be_bytes());
|
||||
out.extend_from_slice(box_type);
|
||||
out.extend_from_slice(body);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Decode the first box's size + type from `buf`.
|
||||
fn read_box_header(buf: &[u8]) -> (u32, [u8; 4]) {
|
||||
let size = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]);
|
||||
let bt = [buf[4], buf[5], buf[6], buf[7]];
|
||||
(size, bt)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn init_segment_starts_with_ftyp_then_moov() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = Fmp4Mux::new(&mut sink);
|
||||
// Trigger init emission via a single (stubbed) write.
|
||||
mux.write_video(0, true, &[0x00, 0x00, 0x00, 0x01, 0x40])
|
||||
.unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
let (ftyp_size, ftyp_type) = read_box_header(&sink);
|
||||
assert_eq!(&ftyp_type, b"ftyp");
|
||||
assert!(ftyp_size >= 24, "ftyp too small: {ftyp_size}");
|
||||
|
||||
let (moov_size, moov_type) = read_box_header(&sink[ftyp_size as usize..]);
|
||||
assert_eq!(&moov_type, b"moov");
|
||||
assert!(moov_size > 100, "moov skeleton too small: {moov_size}");
|
||||
|
||||
// Stub guarantee: no media bytes after the init segment.
|
||||
let total = ftyp_size as usize + moov_size as usize;
|
||||
assert_eq!(sink.len(), total, "stub leaked media bytes past moov");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn moov_contains_trak_mvex() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = Fmp4Mux::new(&mut sink);
|
||||
mux.write_video(0, true, &[]).unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(sink);
|
||||
|
||||
// Re-emit into a fresh buffer for parsing.
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
let mut mux2 = Fmp4Mux::new(&mut buf);
|
||||
mux2.write_video(0, true, &[]).unwrap();
|
||||
mux2.finish().unwrap();
|
||||
drop(mux2);
|
||||
|
||||
// Find moov payload start.
|
||||
let (ftyp_size, _) = read_box_header(&buf);
|
||||
let moov_start = ftyp_size as usize;
|
||||
let (moov_size, _) = read_box_header(&buf[moov_start..]);
|
||||
let moov_payload = &buf[moov_start + 8..moov_start + moov_size as usize];
|
||||
|
||||
// Scan for the trak and mvex four-CC anywhere in the moov payload.
|
||||
let has_trak = moov_payload.windows(4).any(|w| w == b"trak");
|
||||
let has_mvex = moov_payload.windows(4).any(|w| w == b"mvex");
|
||||
assert!(has_trak, "moov missing trak");
|
||||
assert!(has_mvex, "moov missing mvex");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
//! HEVC (H.265) elementary stream muxer — Annex B byte stream.
|
||||
//!
|
||||
//! Consumes [`PesFrame`](crate::pes::PesFrame)s for a single video track
|
||||
//! and writes them as a raw `.hevc` / `.h265` Annex B byte stream:
|
||||
//! `00 00 00 01 | NAL_unit | 00 00 00 01 | NAL_unit | …` with no
|
||||
//! container framing.
|
||||
//!
|
||||
//! On the first frame the muxer emits the codec_private's VPS, SPS, PPS
|
||||
//! (parsed from a `HEVCDecoderConfigurationRecord` in
|
||||
//! `length-prefixed-in-hvcC` form), then converts each PES frame's
|
||||
//! length-prefixed NAL units to Annex B and writes them.
|
||||
//!
|
||||
//! Sequential-only — no Cues, no backpatch. Target sink is any
|
||||
//! [`SequentialSink`](crate::io::sink::SequentialSink): file, socket,
|
||||
//! pipe, anything `Write + Send`.
|
||||
|
||||
use std::io::{self, Write};
|
||||
|
||||
/// Annex B 4-byte start code.
|
||||
pub(crate) const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01];
|
||||
|
||||
/// HEVC NAL unit type bits live in `(byte0 >> 1) & 0x3F` in Annex B.
|
||||
/// We don't filter NAL types here — the muxer is format-only — but we
|
||||
/// keep the constant as documentation of the field layout.
|
||||
#[allow(dead_code)]
|
||||
const HEVC_NAL_TYPE_MASK: u8 = 0x3F;
|
||||
|
||||
/// Streaming HEVC Annex B muxer.
|
||||
///
|
||||
/// One instance per output stream. Tracks whether parameter sets have
|
||||
/// already been emitted so they're written exactly once at the head of
|
||||
/// the stream, mirroring the convention used by `ffmpeg -c:v copy -f
|
||||
/// hevc`.
|
||||
pub struct HevcMux<W: Write> {
|
||||
writer: W,
|
||||
/// `HEVCDecoderConfigurationRecord` payload (hvcC). Parsed lazily
|
||||
/// on the first `write_frame` so callers can set it after
|
||||
/// construction but before the first frame.
|
||||
codec_private: Option<Vec<u8>>,
|
||||
/// Set once VPS/SPS/PPS have been written to the stream. Subsequent
|
||||
/// frames write only their own NAL units.
|
||||
params_written: bool,
|
||||
}
|
||||
|
||||
impl<W: Write> HevcMux<W> {
|
||||
/// Construct over `writer`. The muxer does not impose any extra
|
||||
/// buffering of its own — the sink owns its write buffering policy
|
||||
/// (see [`LocalFileSink`](crate::io::sink::LocalFileSink) and
|
||||
/// [`SocketSink`](crate::io::sink::SocketSink)).
|
||||
pub fn new(writer: W) -> Self {
|
||||
Self {
|
||||
writer,
|
||||
codec_private: None,
|
||||
params_written: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Provide the `HEVCDecoderConfigurationRecord` (hvcC) so the muxer
|
||||
/// can prepend VPS/SPS/PPS Annex B NALs at stream start. Optional —
|
||||
/// if the PES frames already carry inline parameter sets (some
|
||||
/// upstream demuxers do this), skipping this call is fine.
|
||||
pub fn set_codec_private(&mut self, data: Vec<u8>) {
|
||||
self.codec_private = Some(data);
|
||||
}
|
||||
|
||||
/// Write one PES frame (= one access unit) as Annex B NAL units.
|
||||
///
|
||||
/// Input may be either:
|
||||
/// - Length-prefixed: `[u32-BE len][NAL bytes]` repeated. This is
|
||||
/// the form emitted by libfreemkv's HEVC parser (the MKV-native
|
||||
/// layout). Converted to Annex B.
|
||||
/// - Already Annex B: bytes containing `00 00 00 01` start codes
|
||||
/// anywhere in the buffer. Passed through unchanged.
|
||||
///
|
||||
/// `_pts_ns` is accepted for symmetry with other muxers but ignored
|
||||
/// — Annex B has no timing layer.
|
||||
pub fn write_frame(&mut self, _pts_ns: i64, data: &[u8]) -> io::Result<()> {
|
||||
if !self.params_written {
|
||||
if let Some(cp) = &self.codec_private {
|
||||
if let Some(params) = hvcc_to_annex_b(cp) {
|
||||
self.writer.write_all(¶ms)?;
|
||||
}
|
||||
}
|
||||
self.params_written = true;
|
||||
}
|
||||
let annex_b = length_prefixed_to_annex_b(data);
|
||||
self.writer.write_all(&annex_b)
|
||||
}
|
||||
|
||||
/// Flush the underlying writer. No trailer NAL is needed — an Annex
|
||||
/// B stream ends whenever the file/socket ends.
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
self.writer.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a `HEVCDecoderConfigurationRecord` (hvcC) into Annex B NAL
|
||||
/// units. Returns `Some(bytes)` if at least one NAL was extracted, else
|
||||
/// `None`.
|
||||
///
|
||||
/// Layout (per ISO/IEC 14496-15 §8.3.3.1.2):
|
||||
/// - 22-byte fixed header
|
||||
/// - byte 22 = `numOfArrays`
|
||||
/// - each array: `array_completeness:1 | reserved:1 | NAL_unit_type:6`,
|
||||
/// `numNalus:u16-BE`, then `numNalus` × `(nalUnitLength:u16-BE +
|
||||
/// NAL bytes)`.
|
||||
///
|
||||
/// We don't filter on NAL type — VPS (32), SPS (33), PPS (34), and any
|
||||
/// SEI arrays included in hvcC all get the same Annex B treatment.
|
||||
fn hvcc_to_annex_b(hvcc: &[u8]) -> Option<Vec<u8>> {
|
||||
if hvcc.len() < 23 {
|
||||
return None;
|
||||
}
|
||||
let num_arrays = hvcc[22] as usize;
|
||||
let mut out = Vec::new();
|
||||
let mut offset = 23;
|
||||
for _ in 0..num_arrays {
|
||||
if offset + 3 > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
offset += 1; // array_completeness + nal_type byte
|
||||
let num_nalus = u16::from_be_bytes([hvcc[offset], hvcc[offset + 1]]) as usize;
|
||||
offset += 2;
|
||||
for _ in 0..num_nalus {
|
||||
if offset + 2 > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
let nal_len = u16::from_be_bytes([hvcc[offset], hvcc[offset + 1]]) as usize;
|
||||
offset += 2;
|
||||
if offset + nal_len > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
out.extend_from_slice(&START_CODE);
|
||||
out.extend_from_slice(&hvcc[offset..offset + nal_len]);
|
||||
offset += nal_len;
|
||||
}
|
||||
}
|
||||
if out.is_empty() { None } else { Some(out) }
|
||||
}
|
||||
|
||||
/// Convert length-prefixed NAL units (`[u32-BE len][NAL]` repeated) to
|
||||
/// Annex B (`00 00 00 01 [NAL]` repeated).
|
||||
///
|
||||
/// If the input doesn't parse as length-prefixed (no valid lengths
|
||||
/// extracted), it's returned unchanged on the assumption that it's
|
||||
/// already Annex B — some upstream paths (raw HEVC ES from disc) pass
|
||||
/// Annex B straight through the PES layer.
|
||||
pub(crate) fn length_prefixed_to_annex_b(data: &[u8]) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(data.len() + (data.len() / 32));
|
||||
let mut offset = 0;
|
||||
while offset + 4 <= data.len() {
|
||||
let len = u32::from_be_bytes([
|
||||
data[offset],
|
||||
data[offset + 1],
|
||||
data[offset + 2],
|
||||
data[offset + 3],
|
||||
]) as usize;
|
||||
offset += 4;
|
||||
if offset + len > data.len() {
|
||||
// Mid-NAL truncation — fall through to the pass-through path
|
||||
// rather than emitting a half-NAL.
|
||||
return data.to_vec();
|
||||
}
|
||||
out.extend_from_slice(&START_CODE);
|
||||
out.extend_from_slice(&data[offset..offset + len]);
|
||||
offset += len;
|
||||
}
|
||||
if out.is_empty() && !data.is_empty() {
|
||||
// No length prefixes found — input is likely already Annex B.
|
||||
return data.to_vec();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn length_prefixed_converts_to_annex_b() {
|
||||
// Two NALs: [3-byte payload AA BB CC] and [2-byte payload DD EE].
|
||||
let mut buf = Vec::new();
|
||||
buf.extend_from_slice(&3u32.to_be_bytes());
|
||||
buf.extend_from_slice(&[0xAA, 0xBB, 0xCC]);
|
||||
buf.extend_from_slice(&2u32.to_be_bytes());
|
||||
buf.extend_from_slice(&[0xDD, 0xEE]);
|
||||
|
||||
let got = length_prefixed_to_annex_b(&buf);
|
||||
let want = [
|
||||
0x00, 0x00, 0x00, 0x01, 0xAA, 0xBB, 0xCC, // first NAL
|
||||
0x00, 0x00, 0x00, 0x01, 0xDD, 0xEE, // second NAL
|
||||
];
|
||||
assert_eq!(&got[..], &want[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn already_annex_b_passes_through_when_no_lengths_match() {
|
||||
// A buffer < 4 bytes can't parse a length prefix at all →
|
||||
// pass-through path triggers.
|
||||
let raw = [0xAA, 0xBB, 0xCC];
|
||||
let got = length_prefixed_to_annex_b(&raw);
|
||||
assert_eq!(&got[..], &raw[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mid_nal_truncation_returns_original() {
|
||||
// `[u32-BE 100][only 3 bytes]` — length prefix claims 100 bytes
|
||||
// but the input only has 3 after the prefix. We treat that as
|
||||
// malformed and pass the original buffer through so receivers
|
||||
// can attempt their own recovery.
|
||||
let mut raw = Vec::new();
|
||||
raw.extend_from_slice(&100u32.to_be_bytes());
|
||||
raw.extend_from_slice(&[0xAA, 0xBB, 0xCC]);
|
||||
let got = length_prefixed_to_annex_b(&raw);
|
||||
assert_eq!(&got[..], &raw[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hvcc_extracts_vps_sps_pps() {
|
||||
// Build a minimal-but-valid hvcC: 22-byte header, then 3 arrays
|
||||
// (VPS / SPS / PPS), each with 1 NAL of a 4-byte payload that
|
||||
// we can spot in the output.
|
||||
let mut hvcc = vec![0u8; 22];
|
||||
hvcc.push(3); // numOfArrays
|
||||
for (nal_type, payload) in [
|
||||
(32u8, [0x40, 0x01, 0x0C, 0x01]),
|
||||
(33, [0x42, 0x01, 0x01, 0x01]),
|
||||
(34, [0x44, 0x01, 0xC1, 0x72]),
|
||||
] {
|
||||
hvcc.push(nal_type & 0x3F);
|
||||
hvcc.extend_from_slice(&1u16.to_be_bytes()); // numNalus
|
||||
hvcc.extend_from_slice(&(payload.len() as u16).to_be_bytes());
|
||||
hvcc.extend_from_slice(&payload);
|
||||
}
|
||||
|
||||
let annex_b = hvcc_to_annex_b(&hvcc).expect("at least one NAL");
|
||||
// Three NALs × (4-byte start + 4-byte payload) = 24 bytes.
|
||||
assert_eq!(annex_b.len(), 24);
|
||||
assert_eq!(&annex_b[..4], &START_CODE);
|
||||
assert_eq!(&annex_b[8..12], &START_CODE);
|
||||
assert_eq!(&annex_b[16..20], &START_CODE);
|
||||
assert_eq!(annex_b[4], 0x40); // VPS first byte
|
||||
assert_eq!(annex_b[12], 0x42); // SPS first byte
|
||||
assert_eq!(annex_b[20], 0x44); // PPS first byte
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mux_writes_params_then_frames() {
|
||||
// Build hvcC with one SPS to verify params-once semantics.
|
||||
let mut hvcc = vec![0u8; 22];
|
||||
hvcc.push(1);
|
||||
hvcc.push(33);
|
||||
hvcc.extend_from_slice(&1u16.to_be_bytes());
|
||||
hvcc.extend_from_slice(&3u16.to_be_bytes());
|
||||
hvcc.extend_from_slice(&[0x42, 0x01, 0x01]);
|
||||
|
||||
let mut frame_data = Vec::new();
|
||||
frame_data.extend_from_slice(&2u32.to_be_bytes());
|
||||
frame_data.extend_from_slice(&[0xAA, 0xBB]);
|
||||
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = HevcMux::new(&mut sink);
|
||||
mux.set_codec_private(hvcc);
|
||||
mux.write_frame(0, &frame_data).unwrap();
|
||||
// Second frame — no SPS re-emission.
|
||||
mux.write_frame(40_000_000, &frame_data).unwrap();
|
||||
mux.finish().unwrap();
|
||||
|
||||
// SPS NAL (7 bytes) + 2× frame NAL (6 bytes) = 19 bytes.
|
||||
assert_eq!(sink.len(), 7 + 6 + 6);
|
||||
// Start codes at offsets 0 (SPS), 7 (frame1), 13 (frame2).
|
||||
assert_eq!(&sink[0..4], &START_CODE);
|
||||
assert_eq!(&sink[7..11], &START_CODE);
|
||||
assert_eq!(&sink[13..17], &START_CODE);
|
||||
assert_eq!(sink[4], 0x42);
|
||||
assert_eq!(sink[11], 0xAA);
|
||||
assert_eq!(sink[17], 0xAA);
|
||||
}
|
||||
}
|
||||
+6
-6
@@ -1,10 +1,10 @@
|
||||
//! ISO sector reader — file-backed SectorReader for Blu-ray ISO images.
|
||||
//! ISO sector reader — file-backed SectorSource for Blu-ray ISO images.
|
||||
//!
|
||||
//! An ISO is a flat image of 2048-byte sectors. Sector N starts at byte offset N * 2048.
|
||||
//! Used by DiscStream::open_iso() and Disc::scan_image().
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::Path;
|
||||
@@ -32,12 +32,12 @@ impl IsoSectorReader {
|
||||
Ok(Self { file, capacity })
|
||||
}
|
||||
|
||||
pub fn capacity(&self) -> u32 {
|
||||
pub fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for IsoSectorReader {
|
||||
impl SectorSource for IsoSectorReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
@@ -73,7 +73,7 @@ mod tests {
|
||||
std::fs::write(&dir, &data).unwrap();
|
||||
|
||||
let mut reader = IsoSectorReader::open(dir.to_str().unwrap()).unwrap();
|
||||
assert_eq!(reader.capacity(), 4);
|
||||
assert_eq!(reader.capacity_sectors(), 4);
|
||||
|
||||
let mut buf = [0u8; 2048];
|
||||
reader.read_sectors(0, 1, &mut buf, true).unwrap();
|
||||
@@ -94,7 +94,7 @@ mod tests {
|
||||
std::fs::write(&dir, &data).unwrap();
|
||||
|
||||
let reader = IsoSectorReader::open(dir.to_str().unwrap()).unwrap();
|
||||
assert_eq!(reader.capacity(), 10);
|
||||
assert_eq!(reader.capacity_sectors(), 10);
|
||||
|
||||
std::fs::remove_file(&dir).ok();
|
||||
}
|
||||
|
||||
@@ -173,7 +173,6 @@ impl M2tsStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource/FrameSink in follow-up commit.
|
||||
impl crate::pes::Stream for M2tsStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
if let Some(frame) = self.pending_frames.pop_front() {
|
||||
@@ -284,27 +283,3 @@ impl crate::pes::Stream for M2tsStream {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// FrameSink sibling to the deprecated Stream impl; both coexist during the
|
||||
/// 0.18 deprecation window. Caller may pick either at the trait-object
|
||||
/// boundary — `Box<dyn Stream>` (deprecated) or `Box<dyn FrameSink>` (new).
|
||||
/// Use `M2tsStream::create(writer, title)` to construct the write half;
|
||||
/// calling `FrameSink::write` on an `M2tsStream::open(reader)` instance
|
||||
/// returns `StreamReadOnly`.
|
||||
#[allow(deprecated)] // delegating to deprecated Stream during the 0.18 deprecation window so callers don't see the deprecation twice.
|
||||
impl crate::pes::FrameSink for M2tsStream {
|
||||
fn write(&mut self, frame: &crate::pes::PesFrame) -> io::Result<()> {
|
||||
<Self as crate::pes::Stream>::write(self, frame)
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> io::Result<()> {
|
||||
// Why: Stream::finish takes &mut self, FrameSink::finish takes Box<Self>.
|
||||
// Re-borrow inside the box, call Stream::finish, drop the box.
|
||||
let mut s: Self = *self;
|
||||
<Self as crate::pes::Stream>::finish(&mut s)
|
||||
}
|
||||
|
||||
fn info(&self) -> &crate::disc::DiscTitle {
|
||||
<Self as crate::pes::Stream>::info(self)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,681 @@
|
||||
//! Standard MPEG-TS (188-byte packets) muxer — sequential-only.
|
||||
//!
|
||||
//! Distinct from `super::tsmux::TsMuxer` (BD-TS with 192-byte packets
|
||||
//! and the 4-byte TP_extra_header). This muxer emits the IETF / ISO/IEC
|
||||
//! 13818-1 wire format that ffmpeg, VLC, and `m2tsindex` consume
|
||||
//! out of the box. Use it for plain `.ts` / `.m2ts` files over a
|
||||
//! [`SequentialSink`](crate::io::sink::SequentialSink), and for
|
||||
//! MPEG-TS-over-UDP via [`UdpSocketSink`](crate::io::sink::UdpSocketSink).
|
||||
//!
|
||||
//! ## Wire format
|
||||
//!
|
||||
//! Every output packet is exactly 188 bytes:
|
||||
//!
|
||||
//! ```text
|
||||
//! sync_byte:8 = 0x47
|
||||
//! transport_error_indicator:1 = 0
|
||||
//! payload_unit_start_indicator:1
|
||||
//! transport_priority:1 = 0
|
||||
//! PID:13
|
||||
//! transport_scrambling_control:2 = 0
|
||||
//! adaptation_field_control:2
|
||||
//! continuity_counter:4
|
||||
//! [adaptation field, if signalled]
|
||||
//! [payload, if signalled]
|
||||
//! ```
|
||||
//!
|
||||
//! ## Wiring
|
||||
//!
|
||||
//! Single program (PMT PID `0x1000`, program number `1`):
|
||||
//! - PAT on PID `0x0000`
|
||||
//! - PMT on PID `0x1000`
|
||||
//! - Video (HEVC, stream_type `0x24`) on PID `0x0100`
|
||||
//! - First audio (AC3, stream_type `0x81`) on PID `0x0101`
|
||||
//! (or TrueHD, stream_type `0x83`, if hinted)
|
||||
//!
|
||||
//! ## Scope vs. full TS
|
||||
//!
|
||||
//! This is a deliberately minimal viable muxer:
|
||||
//! - One program, one video track, optionally one audio track.
|
||||
//! - PAT + PMT re-emitted every `PSI_INTERVAL_PACKETS` packets so a
|
||||
//! mid-stream receiver can lock on within ~100 ms at typical
|
||||
//! UHD bitrates.
|
||||
//! - PCR clock derived from video PTS (`pts - PCR_LEAD_90KHZ`),
|
||||
//! attached to the video PID's adaptation field every
|
||||
//! `PCR_INTERVAL_PACKETS` packets.
|
||||
//! - No language / descriptor tags, no SCTE-35 markers, no per-PID
|
||||
//! PMT version bumps, no SDT/EIT. Sufficient for "ffmpeg can play
|
||||
//! this back", not for full broadcast deployment.
|
||||
|
||||
use std::io::{self, Write};
|
||||
|
||||
mod packet;
|
||||
|
||||
use packet::{Packet, PacketWriter};
|
||||
|
||||
/// PID `0x0000` — PAT, mandated by spec.
|
||||
const PID_PAT: u16 = 0x0000;
|
||||
/// PID for the single program's PMT. `0x1000` is the conventional
|
||||
/// choice; anything outside reserved/null ranges works.
|
||||
const PID_PMT: u16 = 0x1000;
|
||||
/// PID for the video elementary stream.
|
||||
const PID_VIDEO: u16 = 0x0100;
|
||||
/// PID for the (optional) first audio elementary stream.
|
||||
const PID_AUDIO: u16 = 0x0101;
|
||||
/// Null PID — reserved by spec; never used here.
|
||||
#[allow(dead_code)]
|
||||
const PID_NULL: u16 = 0x1FFF;
|
||||
|
||||
/// Re-emit PAT/PMT every N TS packets. ~250 × 188 B = 47 KB; at
|
||||
/// 80 Mb/s UHD that's ~5 ms — well under typical receiver lock budget.
|
||||
const PSI_INTERVAL_PACKETS: u64 = 250;
|
||||
/// Re-stamp PCR every N TS packets carrying the video PID. Spec MAX
|
||||
/// is 100 ms; 40 packets × 188 B at moderate bitrate keeps us under.
|
||||
const PCR_INTERVAL_PACKETS: u64 = 40;
|
||||
/// PCR lead time — the PCR must precede the PTS of the first byte of
|
||||
/// the picture it timestamps. 200 ms in 90 kHz ticks.
|
||||
const PCR_LEAD_90KHZ: u64 = 90_000 / 5;
|
||||
|
||||
/// Stream-type codes from ISO/IEC 13818-1 Table 2-29 + later amendments.
|
||||
const STREAM_TYPE_HEVC: u8 = 0x24;
|
||||
const STREAM_TYPE_AC3: u8 = 0x81;
|
||||
const STREAM_TYPE_TRUEHD: u8 = 0x83;
|
||||
|
||||
/// Audio codec hint for [`M2tsMux::new`] / [`M2tsMux::set_audio`]. The
|
||||
/// muxer needs to know the codec to pick the right PMT `stream_type`
|
||||
/// and the right PES stream_id; it doesn't decode the audio.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum AudioCodec {
|
||||
/// AC-3 / E-AC-3, stream_type `0x81`, PES stream_id `0xBD`.
|
||||
Ac3,
|
||||
/// Dolby TrueHD, stream_type `0x83`, PES stream_id `0xBD`.
|
||||
TrueHd,
|
||||
}
|
||||
|
||||
impl AudioCodec {
|
||||
fn stream_type(self) -> u8 {
|
||||
match self {
|
||||
AudioCodec::Ac3 => STREAM_TYPE_AC3,
|
||||
AudioCodec::TrueHd => STREAM_TYPE_TRUEHD,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sequential MPEG-TS muxer with one video and optional one audio track.
|
||||
pub struct M2tsMux<W: Write> {
|
||||
out: PacketWriter<W>,
|
||||
/// hvcC parameter set bytes to prepend to the first video PES.
|
||||
/// Optional — if the upstream frames already carry inline params,
|
||||
/// callers can omit this.
|
||||
video_codec_private: Option<Vec<u8>>,
|
||||
/// Set on first video frame: have we emitted VPS/SPS/PPS?
|
||||
params_written: bool,
|
||||
/// Audio codec, if an audio track is configured. `None` = video-only.
|
||||
audio: Option<AudioCodec>,
|
||||
/// First seen PTS (90 kHz). All subsequent PTS / PCR values are
|
||||
/// relative to this so output streams start near t=0 and don't
|
||||
/// confuse downstream parsers that don't tolerate huge starting
|
||||
/// timestamps.
|
||||
base_pts_90k: Option<u64>,
|
||||
/// Per-PID continuity counter, 4 bits, monotonically increasing.
|
||||
cc_video: u8,
|
||||
cc_audio: u8,
|
||||
cc_pat: u8,
|
||||
cc_pmt: u8,
|
||||
/// Total packets written, used to gate PSI / PCR cadence.
|
||||
packets_written: u64,
|
||||
/// Video packets written since last PCR, used to gate PCR cadence.
|
||||
video_packets_since_pcr: u64,
|
||||
}
|
||||
|
||||
impl<W: Write> M2tsMux<W> {
|
||||
/// Construct a muxer wrapping `writer`. By default audio is
|
||||
/// disabled; call [`set_audio`](Self::set_audio) before the first
|
||||
/// frame to enable.
|
||||
pub fn new(writer: W) -> Self {
|
||||
Self {
|
||||
out: PacketWriter::new(writer),
|
||||
video_codec_private: None,
|
||||
params_written: false,
|
||||
audio: None,
|
||||
base_pts_90k: None,
|
||||
cc_video: 0,
|
||||
cc_audio: 0,
|
||||
cc_pat: 0,
|
||||
cc_pmt: 0,
|
||||
packets_written: 0,
|
||||
video_packets_since_pcr: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Provide the `HEVCDecoderConfigurationRecord` for video so the
|
||||
/// muxer prepends VPS/SPS/PPS Annex B NALs at stream start.
|
||||
pub fn set_video_codec_private(&mut self, hvcc: Vec<u8>) {
|
||||
self.video_codec_private = Some(hvcc);
|
||||
}
|
||||
|
||||
/// Enable a single audio track. Must be called before
|
||||
/// [`write_audio`](Self::write_audio).
|
||||
pub fn set_audio(&mut self, codec: AudioCodec) {
|
||||
self.audio = Some(codec);
|
||||
}
|
||||
|
||||
/// Write one video PES frame. `data` is either length-prefixed
|
||||
/// NALUs (MKV-style) or already Annex B; both are accepted.
|
||||
pub fn write_video(&mut self, pts_ns: i64, data: &[u8]) -> io::Result<()> {
|
||||
let pts_90k = self.base_relative_pts(pts_ns);
|
||||
// PCR comes "before" the PTS it timestamps; clamp at 0 for the
|
||||
// first frame so we don't underflow.
|
||||
let pcr = pts_90k.saturating_sub(PCR_LEAD_90KHZ);
|
||||
|
||||
// Annex-B-ify the frame and prepend VPS/SPS/PPS once.
|
||||
let mut es = Vec::with_capacity(data.len() + 64);
|
||||
if !self.params_written {
|
||||
if let Some(cp) = &self.video_codec_private {
|
||||
let payload = hvcc_payload(cp);
|
||||
if !payload.is_empty() {
|
||||
let params = super::hevc::length_prefixed_to_annex_b(&payload);
|
||||
es.extend_from_slice(¶ms);
|
||||
}
|
||||
}
|
||||
self.params_written = true;
|
||||
}
|
||||
let annex_b = super::hevc::length_prefixed_to_annex_b(data);
|
||||
es.extend_from_slice(&annex_b);
|
||||
|
||||
let pes = build_video_pes(pts_90k, &es);
|
||||
self.write_pes(PID_VIDEO, &pes, Some(pcr))
|
||||
}
|
||||
|
||||
/// Write one audio PES frame. Returns `Ok(())` and silently drops
|
||||
/// the frame if no audio track was configured — the design assumes
|
||||
/// the upstream picks tracks and won't ship audio to a video-only
|
||||
/// muxer, but defending against it keeps the API a single shape.
|
||||
pub fn write_audio(&mut self, pts_ns: i64, data: &[u8]) -> io::Result<()> {
|
||||
if self.audio.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
let pts_90k = self.base_relative_pts(pts_ns);
|
||||
let pes = build_audio_pes(pts_90k, data);
|
||||
self.write_pes(PID_AUDIO, &pes, None)
|
||||
}
|
||||
|
||||
/// Drain the underlying writer. No TS-level trailer is mandatory —
|
||||
/// receivers detect end-of-stream from socket close or file EOF.
|
||||
pub fn finish(&mut self) -> io::Result<()> {
|
||||
self.out.flush()
|
||||
}
|
||||
|
||||
/// Convert input PTS (nanoseconds) to 90 kHz ticks rebased on the
|
||||
/// first frame's PTS. Saturating at 0 keeps the math friendly when
|
||||
/// frames arrive slightly out of decode order.
|
||||
fn base_relative_pts(&mut self, pts_ns: i64) -> u64 {
|
||||
let raw_90k = if pts_ns > 0 {
|
||||
(pts_ns as u64) * 9 / 100_000
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let base = *self.base_pts_90k.get_or_insert(raw_90k);
|
||||
raw_90k.saturating_sub(base)
|
||||
}
|
||||
|
||||
/// Emit one PES payload as a chain of TS packets on `pid`. If `pcr`
|
||||
/// is provided the first packet carries an adaptation field with
|
||||
/// the PCR. PAT/PMT are re-emitted every `PSI_INTERVAL_PACKETS`.
|
||||
///
|
||||
/// Packet-size math (TS = 188 bytes, header = 4 bytes):
|
||||
/// - 184 B remain after the header for the adaptation-field area
|
||||
/// plus the payload area.
|
||||
/// - With AF body of `b` bytes and `s` stuffing bytes: AF total =
|
||||
/// `1 + b + s` (the leading `1` is the `adaptation_field_length`
|
||||
/// byte itself). Payload = `184 - (1 + b + s)`.
|
||||
/// - With no AF area at all: payload = `184`.
|
||||
///
|
||||
/// The fit-the-tail logic on the last packet of the PES uses
|
||||
/// stuffing rather than a separate small packet, which is the
|
||||
/// standard MPEG-TS convention.
|
||||
fn write_pes(&mut self, pid: u16, pes: &[u8], pcr: Option<u64>) -> io::Result<()> {
|
||||
// PSI cadence is enforced per TS packet — interleave a fresh
|
||||
// PAT+PMT into the packet stream every PSI_INTERVAL_PACKETS so
|
||||
// long single-PES emissions (e.g. one 60 KB video frame) don't
|
||||
// starve receivers tuning in mid-stream.
|
||||
let mut offset = 0;
|
||||
let mut first = true;
|
||||
while offset < pes.len() {
|
||||
self.maybe_emit_psi()?;
|
||||
|
||||
let attach_pcr = first
|
||||
&& (pid == PID_VIDEO)
|
||||
&& (pcr.is_some())
|
||||
&& (self.packets_written == 0
|
||||
|| self.video_packets_since_pcr >= PCR_INTERVAL_PACKETS);
|
||||
|
||||
let af_body: Vec<u8> = if attach_pcr {
|
||||
build_pcr_adaptation(pcr.unwrap_or(0))
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
let remaining = pes.len() - offset;
|
||||
// Capacity for payload given AF body and 1-byte AF length.
|
||||
// When AF body is empty we can still skip the AF entirely
|
||||
// and get the full 184 B; only invoke the AF when we'd
|
||||
// otherwise need stuffing.
|
||||
let (af_present, payload_len, stuffing): (bool, usize, usize) = if !af_body.is_empty() {
|
||||
// AF is mandatory (PCR). 1 byte length + body + stuffing
|
||||
// + payload = 184.
|
||||
let max_payload = 184 - 1 - af_body.len();
|
||||
let p = remaining.min(max_payload);
|
||||
let s = max_payload - p;
|
||||
(true, p, s)
|
||||
} else if remaining >= 184 {
|
||||
// Full payload packet — no AF at all.
|
||||
(false, 184, 0)
|
||||
} else {
|
||||
// Last (small) packet — stuff via empty AF.
|
||||
// 1 byte length + 0 body + stuffing + payload = 184.
|
||||
let max_payload = 183;
|
||||
let p = remaining.min(max_payload);
|
||||
let s = max_payload - p;
|
||||
(true, p, s)
|
||||
};
|
||||
|
||||
let cc = self.advance_cc(pid);
|
||||
let mut packet = Packet::new();
|
||||
packet.set_header(pid, first, true, af_present, cc);
|
||||
if af_present {
|
||||
packet.append_adaptation(&af_body, stuffing);
|
||||
}
|
||||
packet.append_payload(&pes[offset..offset + payload_len]);
|
||||
debug_assert_eq!(packet.len(), 188, "packet not 188 bytes");
|
||||
self.out.write_packet(&packet)?;
|
||||
|
||||
self.packets_written += 1;
|
||||
if pid == PID_VIDEO {
|
||||
if attach_pcr {
|
||||
self.video_packets_since_pcr = 0;
|
||||
} else {
|
||||
self.video_packets_since_pcr += 1;
|
||||
}
|
||||
}
|
||||
offset += payload_len;
|
||||
first = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn advance_cc(&mut self, pid: u16) -> u8 {
|
||||
let slot = match pid {
|
||||
PID_VIDEO => &mut self.cc_video,
|
||||
PID_AUDIO => &mut self.cc_audio,
|
||||
PID_PAT => &mut self.cc_pat,
|
||||
PID_PMT => &mut self.cc_pmt,
|
||||
_ => return 0,
|
||||
};
|
||||
let cc = *slot;
|
||||
*slot = (*slot + 1) & 0x0F;
|
||||
cc
|
||||
}
|
||||
|
||||
fn maybe_emit_psi(&mut self) -> io::Result<()> {
|
||||
if self.packets_written == 0 || self.packets_written % PSI_INTERVAL_PACKETS == 0 {
|
||||
self.emit_pat()?;
|
||||
self.emit_pmt()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn emit_pat(&mut self) -> io::Result<()> {
|
||||
let payload = build_pat(PID_PMT);
|
||||
let cc = self.advance_cc(PID_PAT);
|
||||
let mut packet = Packet::new();
|
||||
packet.set_header(PID_PAT, true, true, false, cc);
|
||||
packet.append_payload(&payload);
|
||||
packet.pad_to_188();
|
||||
self.out.write_packet(&packet)?;
|
||||
self.packets_written += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn emit_pmt(&mut self) -> io::Result<()> {
|
||||
let payload = build_pmt(self.audio);
|
||||
let cc = self.advance_cc(PID_PMT);
|
||||
let mut packet = Packet::new();
|
||||
packet.set_header(PID_PMT, true, true, false, cc);
|
||||
packet.append_payload(&payload);
|
||||
packet.pad_to_188();
|
||||
self.out.write_packet(&packet)?;
|
||||
self.packets_written += 1;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the raw hvcC bytes for handoff to `length_prefixed_to_annex_b`.
|
||||
/// hvcC layout: 22-byte fixed header, then `numOfArrays` arrays of
|
||||
/// `(nalType, numNalus, [nalLength:u16, NAL bytes]…)`. We convert this
|
||||
/// directly to a length-prefixed byte stream (NAL length is u16 in
|
||||
/// hvcC; widen to u32 for the standard length-prefixed encoding).
|
||||
fn hvcc_payload(hvcc: &[u8]) -> Vec<u8> {
|
||||
if hvcc.len() < 23 {
|
||||
return Vec::new();
|
||||
}
|
||||
let num_arrays = hvcc[22] as usize;
|
||||
let mut out = Vec::new();
|
||||
let mut offset = 23;
|
||||
for _ in 0..num_arrays {
|
||||
if offset + 3 > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
offset += 1;
|
||||
let num_nalus = u16::from_be_bytes([hvcc[offset], hvcc[offset + 1]]) as usize;
|
||||
offset += 2;
|
||||
for _ in 0..num_nalus {
|
||||
if offset + 2 > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
let nal_len = u16::from_be_bytes([hvcc[offset], hvcc[offset + 1]]) as usize;
|
||||
offset += 2;
|
||||
if offset + nal_len > hvcc.len() {
|
||||
break;
|
||||
}
|
||||
out.extend_from_slice(&(nal_len as u32).to_be_bytes());
|
||||
out.extend_from_slice(&hvcc[offset..offset + nal_len]);
|
||||
offset += nal_len;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Build a PES packet for a video access unit.
|
||||
fn build_video_pes(pts_90k: u64, es: &[u8]) -> Vec<u8> {
|
||||
build_pes_packet(0xE0, pts_90k, es, /* length_in_header */ false)
|
||||
}
|
||||
|
||||
/// Build a PES packet for an audio access unit.
|
||||
fn build_audio_pes(pts_90k: u64, es: &[u8]) -> Vec<u8> {
|
||||
// Audio PES: length is fillable when it fits in u16. We always
|
||||
// write the length so receivers don't have to scan for the next
|
||||
// start code.
|
||||
build_pes_packet(0xBD, pts_90k, es, /* length_in_header */ true)
|
||||
}
|
||||
|
||||
fn build_pes_packet(stream_id: u8, pts_90k: u64, es: &[u8], length_in_header: bool) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(es.len() + 14);
|
||||
out.extend_from_slice(&[0x00, 0x00, 0x01, stream_id]);
|
||||
// PES_packet_length: total bytes after this field. 3 flag bytes + 5
|
||||
// PTS bytes + es.len(). Zero means "unbounded" — used for video
|
||||
// where PES can exceed u16.
|
||||
let pes_len = 8 + es.len();
|
||||
if length_in_header && pes_len <= u16::MAX as usize {
|
||||
out.extend_from_slice(&(pes_len as u16).to_be_bytes());
|
||||
} else {
|
||||
out.extend_from_slice(&[0x00, 0x00]);
|
||||
}
|
||||
// Flags byte 1: 10 = MPEG-2 marker, then scrambling/priority/etc 0.
|
||||
out.push(0x80);
|
||||
// Flags byte 2: PTS flag (bit 7).
|
||||
out.push(0x80);
|
||||
// PES_header_data_length = 5 (just PTS).
|
||||
out.push(5);
|
||||
// PTS bytes — 33-bit timestamp split across 5 bytes with marker bits.
|
||||
let pts = pts_90k & 0x1_FFFF_FFFF;
|
||||
out.push(0x21 | (((pts >> 29) & 0x0E) as u8));
|
||||
out.push(((pts >> 22) & 0xFF) as u8);
|
||||
out.push(0x01 | (((pts >> 14) & 0xFE) as u8));
|
||||
out.push(((pts >> 7) & 0xFF) as u8);
|
||||
out.push(0x01 | (((pts << 1) & 0xFE) as u8));
|
||||
out.extend_from_slice(es);
|
||||
out
|
||||
}
|
||||
|
||||
/// Build the PAT payload (section, with pointer_field).
|
||||
fn build_pat(pmt_pid: u16) -> Vec<u8> {
|
||||
let mut section = Vec::new();
|
||||
section.push(0x00); // table_id = PAT
|
||||
// section_syntax_indicator(1) | '0'(1) | reserved(2) | section_length(12)
|
||||
// section_length covers from end of this field through CRC.
|
||||
// Body: transport_stream_id(2) + version/cni(1) + section/last_section(2) + program(4) = 9 bytes,
|
||||
// plus CRC(4) = 13. Encoded big-endian.
|
||||
section.extend_from_slice(&[0xB0, 13]);
|
||||
section.extend_from_slice(&[0x00, 0x01]); // transport_stream_id = 1
|
||||
section.push(0xC1); // reserved | version=0 | current_next=1
|
||||
section.push(0x00); // section_number
|
||||
section.push(0x00); // last_section_number
|
||||
section.extend_from_slice(&[0x00, 0x01]); // program_number = 1
|
||||
// reserved(3) | network_PID/program_map_PID(13)
|
||||
let pid_bytes = (0xE000u16 | (pmt_pid & 0x1FFF)).to_be_bytes();
|
||||
section.extend_from_slice(&pid_bytes);
|
||||
let crc = mpegts_crc32(§ion);
|
||||
section.extend_from_slice(&crc.to_be_bytes());
|
||||
// Prepend pointer_field=0 (section starts immediately).
|
||||
let mut payload = Vec::with_capacity(section.len() + 1);
|
||||
payload.push(0x00);
|
||||
payload.extend_from_slice(§ion);
|
||||
payload
|
||||
}
|
||||
|
||||
/// Build the PMT payload (section, with pointer_field).
|
||||
fn build_pmt(audio: Option<AudioCodec>) -> Vec<u8> {
|
||||
let mut section = Vec::new();
|
||||
section.push(0x02); // table_id = PMT
|
||||
// section_length filled in after we know the body size.
|
||||
let len_placeholder = section.len();
|
||||
section.extend_from_slice(&[0xB0, 0x00]);
|
||||
|
||||
section.extend_from_slice(&1u16.to_be_bytes()); // program_number
|
||||
section.push(0xC1); // reserved | version=0 | current_next=1
|
||||
section.push(0x00); // section_number
|
||||
section.push(0x00); // last_section_number
|
||||
// reserved(3) | PCR_PID(13)
|
||||
let pcr_pid = (0xE000u16 | (PID_VIDEO & 0x1FFF)).to_be_bytes();
|
||||
section.extend_from_slice(&pcr_pid);
|
||||
// program_info_length = 0
|
||||
section.extend_from_slice(&[0xF0, 0x00]);
|
||||
|
||||
// Video elementary stream entry.
|
||||
section.push(STREAM_TYPE_HEVC);
|
||||
let v_pid = (0xE000u16 | (PID_VIDEO & 0x1FFF)).to_be_bytes();
|
||||
section.extend_from_slice(&v_pid);
|
||||
section.extend_from_slice(&[0xF0, 0x00]); // ES_info_length = 0
|
||||
|
||||
if let Some(codec) = audio {
|
||||
section.push(codec.stream_type());
|
||||
let a_pid = (0xE000u16 | (PID_AUDIO & 0x1FFF)).to_be_bytes();
|
||||
section.extend_from_slice(&a_pid);
|
||||
section.extend_from_slice(&[0xF0, 0x00]);
|
||||
}
|
||||
|
||||
// Now patch section_length: covers everything after the length field
|
||||
// through the CRC, so (current body size - 3 bytes consumed by
|
||||
// table_id + 2 length bytes) + 4 (CRC).
|
||||
let section_len = section.len() - 3 + 4;
|
||||
section[len_placeholder] = 0xB0 | ((section_len >> 8) as u8 & 0x0F);
|
||||
section[len_placeholder + 1] = section_len as u8;
|
||||
|
||||
let crc = mpegts_crc32(§ion);
|
||||
section.extend_from_slice(&crc.to_be_bytes());
|
||||
|
||||
let mut payload = Vec::with_capacity(section.len() + 1);
|
||||
payload.push(0x00);
|
||||
payload.extend_from_slice(§ion);
|
||||
payload
|
||||
}
|
||||
|
||||
/// Build the adaptation field carrying a PCR (no other flags).
|
||||
fn build_pcr_adaptation(pcr_90k: u64) -> Vec<u8> {
|
||||
// adaptation_field_length is set by `Packet::append_adaptation`
|
||||
// — this function returns just the field body.
|
||||
//
|
||||
// Layout: discontinuity_indicator(1) | random_access(1) |
|
||||
// elementary_stream_priority(1) | PCR_flag(1) | OPCR_flag(1) |
|
||||
// splicing_point_flag(1) | transport_private_data_flag(1) |
|
||||
// adaptation_field_extension_flag(1) | PCR(48b).
|
||||
let mut af = vec![0x50]; // PCR_flag=1, random_access_indicator=1
|
||||
let pcr_base = pcr_90k & 0x1_FFFF_FFFF; // 33-bit
|
||||
let pcr_ext: u16 = 0; // 9-bit, we keep it zero (no sub-tick precision)
|
||||
// Encode PCR: 33b base | 6b reserved | 9b extension = 48b
|
||||
af.push((pcr_base >> 25) as u8);
|
||||
af.push((pcr_base >> 17) as u8);
|
||||
af.push((pcr_base >> 9) as u8);
|
||||
af.push((pcr_base >> 1) as u8);
|
||||
af.push(((pcr_base << 7) as u8 & 0x80) | 0x7E | ((pcr_ext >> 8) as u8 & 0x01));
|
||||
af.push(pcr_ext as u8);
|
||||
af
|
||||
}
|
||||
|
||||
/// MPEG-TS CRC-32 (poly 0x04C11DB7, init 0xFFFFFFFF, no reflection, no
|
||||
/// final XOR). Implementation: bitwise so we don't need a table —
|
||||
/// PSI sections are tiny, the cost is negligible.
|
||||
fn mpegts_crc32(data: &[u8]) -> u32 {
|
||||
let mut crc: u32 = 0xFFFF_FFFF;
|
||||
for &b in data {
|
||||
crc ^= (b as u32) << 24;
|
||||
for _ in 0..8 {
|
||||
if crc & 0x8000_0000 != 0 {
|
||||
crc = (crc << 1) ^ 0x04C1_1DB7;
|
||||
} else {
|
||||
crc <<= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
crc
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// All emitted bytes must align to 188-byte packet boundaries and
|
||||
/// every packet must start with `0x47`.
|
||||
fn assert_ts_well_formed(buf: &[u8]) {
|
||||
assert_eq!(
|
||||
buf.len() % 188,
|
||||
0,
|
||||
"stream not packet-aligned: {} bytes",
|
||||
buf.len()
|
||||
);
|
||||
for (i, chunk) in buf.chunks(188).enumerate() {
|
||||
assert_eq!(chunk[0], 0x47, "packet {} missing sync byte", i);
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_pids(buf: &[u8]) -> Vec<u16> {
|
||||
buf.chunks(188)
|
||||
.map(|p| u16::from_be_bytes([p[1] & 0x1F, p[2]]))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crc32_is_self_validating() {
|
||||
// The MPEG-TS CRC has the property that prepending a single
|
||||
// bit-flip changes the output; running it over its own input +
|
||||
// CRC yields a fixed magic constant (the CRC residue). Rather
|
||||
// than hardcoding sample bytes, verify the underlying algorithm
|
||||
// by checking that two distinct inputs produce distinct CRCs
|
||||
// and that the same input is deterministic.
|
||||
let a = [
|
||||
0u8, 0xB0, 0x0D, 0x00, 0x01, 0xC1, 0x00, 0x00, 0x00, 0x01, 0xE1, 0x00,
|
||||
];
|
||||
let mut b = a;
|
||||
b[5] ^= 0x01; // flip one bit
|
||||
let crc_a = mpegts_crc32(&a);
|
||||
let crc_b = mpegts_crc32(&b);
|
||||
assert_ne!(crc_a, crc_b);
|
||||
assert_eq!(crc_a, mpegts_crc32(&a)); // deterministic
|
||||
// Sanity: all-zero input ⇒ CRC = 0 (init XORs but the
|
||||
// shift/feedback cancels for zero data after init drains).
|
||||
// We don't assert exact value — that depends on poly choice —
|
||||
// but check it's not the same as for non-zero data.
|
||||
let crc_zero = mpegts_crc32(&[0u8; 12]);
|
||||
assert_ne!(crc_zero, crc_a);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_only_mux_emits_pat_pmt_then_video() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = M2tsMux::new(&mut sink);
|
||||
// One small video frame, no codec_private (so no params NAL inline).
|
||||
let mut frame = Vec::new();
|
||||
frame.extend_from_slice(&4u32.to_be_bytes());
|
||||
frame.extend_from_slice(&[0x40, 0x01, 0x0C, 0x01]);
|
||||
mux.write_video(0, &frame).unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
assert_ts_well_formed(&sink);
|
||||
let pids = extract_pids(&sink);
|
||||
// First two packets: PAT, PMT. At least one video packet after.
|
||||
assert_eq!(pids[0], PID_PAT);
|
||||
assert_eq!(pids[1], PID_PMT);
|
||||
assert!(pids.iter().any(|p| *p == PID_VIDEO));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_track_appears_in_pmt_and_stream() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = M2tsMux::new(&mut sink);
|
||||
mux.set_audio(AudioCodec::Ac3);
|
||||
// Video + audio frame pair.
|
||||
let mut frame = Vec::new();
|
||||
frame.extend_from_slice(&3u32.to_be_bytes());
|
||||
frame.extend_from_slice(&[0x40, 0x01, 0x0C]);
|
||||
mux.write_video(0, &frame).unwrap();
|
||||
mux.write_audio(20_000_000, &[0x0B, 0x77, 0x12, 0x34])
|
||||
.unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
assert_ts_well_formed(&sink);
|
||||
let pids = extract_pids(&sink);
|
||||
assert!(pids.iter().any(|p| *p == PID_VIDEO));
|
||||
assert!(pids.iter().any(|p| *p == PID_AUDIO));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn psi_re_emits_at_interval() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = M2tsMux::new(&mut sink);
|
||||
// Build a frame large enough to span > PSI_INTERVAL_PACKETS TS packets.
|
||||
// 184 B payload per packet ⇒ ~250 packets = ~46 KB elementary stream.
|
||||
let big: Vec<u8> = (0..(60 * 1024)).map(|i| (i & 0xff) as u8).collect();
|
||||
let mut frame = Vec::new();
|
||||
frame.extend_from_slice(&(big.len() as u32).to_be_bytes());
|
||||
frame.extend_from_slice(&big);
|
||||
mux.write_video(0, &frame).unwrap();
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
assert_ts_well_formed(&sink);
|
||||
let pids = extract_pids(&sink);
|
||||
// Count PAT/PMT pairs — must be at least 2 given the input size.
|
||||
let pat_count = pids.iter().filter(|p| **p == PID_PAT).count();
|
||||
let pmt_count = pids.iter().filter(|p| **p == PID_PMT).count();
|
||||
assert!(pat_count >= 2, "expected ≥2 PAT, got {}", pat_count);
|
||||
assert!(pmt_count >= 2, "expected ≥2 PMT, got {}", pmt_count);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn continuity_counter_increments_per_pid() {
|
||||
let mut sink: Vec<u8> = Vec::new();
|
||||
let mut mux = M2tsMux::new(&mut sink);
|
||||
// Three small video frames to get a sequence of video TS packets.
|
||||
for pts in [0i64, 40_000_000, 80_000_000] {
|
||||
let mut frame = Vec::new();
|
||||
frame.extend_from_slice(&3u32.to_be_bytes());
|
||||
frame.extend_from_slice(&[0xAA, 0xBB, 0xCC]);
|
||||
mux.write_video(pts, &frame).unwrap();
|
||||
}
|
||||
mux.finish().unwrap();
|
||||
drop(mux);
|
||||
|
||||
// Collect CCs for video packets in order.
|
||||
let ccs: Vec<u8> = sink
|
||||
.chunks(188)
|
||||
.filter(|p| u16::from_be_bytes([p[1] & 0x1F, p[2]]) == PID_VIDEO)
|
||||
.map(|p| p[3] & 0x0F)
|
||||
.collect();
|
||||
for w in ccs.windows(2) {
|
||||
assert_eq!(w[1], (w[0] + 1) & 0x0F);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
//! 188-byte MPEG-TS packet builder + writer.
|
||||
//!
|
||||
//! Internal helper for `super::M2tsMux`. Not public API — exposes raw
|
||||
//! byte layout so the parent module can compose PSI / PCR / PES bytes
|
||||
//! without each caller re-implementing the 188-byte boundary math.
|
||||
|
||||
use std::io::{self, Write};
|
||||
|
||||
const TS_PACKET_SIZE: usize = 188;
|
||||
const SYNC_BYTE: u8 = 0x47;
|
||||
const STUFF_BYTE: u8 = 0xFF;
|
||||
|
||||
/// One TS packet under construction. Always emits 188 bytes when
|
||||
/// [`pad_to_188`](Self::pad_to_188) is called; if it's not called the
|
||||
/// caller is responsible for filling the packet exactly.
|
||||
pub(super) struct Packet {
|
||||
buf: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Packet {
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
buf: Vec::with_capacity(TS_PACKET_SIZE),
|
||||
}
|
||||
}
|
||||
|
||||
/// Write the 4-byte TS packet header.
|
||||
///
|
||||
/// * `pid` — 13-bit PID
|
||||
/// * `payload_unit_start` — first packet of a PES / PSI section
|
||||
/// * `has_payload` — packet carries any payload bytes
|
||||
/// * `has_adaptation` — packet carries an adaptation field
|
||||
/// * `cc` — 4-bit continuity counter
|
||||
pub(super) fn set_header(
|
||||
&mut self,
|
||||
pid: u16,
|
||||
payload_unit_start: bool,
|
||||
has_payload: bool,
|
||||
has_adaptation: bool,
|
||||
cc: u8,
|
||||
) {
|
||||
self.buf.clear();
|
||||
self.buf.push(SYNC_BYTE);
|
||||
let pus_bit = if payload_unit_start { 0x40 } else { 0 };
|
||||
// transport_error_indicator(1)=0 | payload_unit_start(1) | transport_priority(1)=0 | PID(5 high)
|
||||
self.buf.push(pus_bit | ((pid >> 8) as u8 & 0x1F));
|
||||
self.buf.push(pid as u8);
|
||||
// transport_scrambling_control(2)=0 | adaptation_field_control(2) | continuity_counter(4)
|
||||
let afc = match (has_adaptation, has_payload) {
|
||||
(false, false) => 0b00, // reserved — should not happen
|
||||
(false, true) => 0b01, // payload only
|
||||
(true, false) => 0b10, // adaptation only
|
||||
(true, true) => 0b11, // both
|
||||
};
|
||||
self.buf.push((afc << 4) | (cc & 0x0F));
|
||||
}
|
||||
|
||||
/// Append the adaptation field after the header.
|
||||
///
|
||||
/// `body` is the adaptation field body (flags byte plus optional PCR
|
||||
/// and so on). `stuffing` is the number of `0xFF` stuffing bytes to
|
||||
/// append after the body. The first byte of the field
|
||||
/// (`adaptation_field_length`) is computed here from
|
||||
/// `body.len() + stuffing`.
|
||||
pub(super) fn append_adaptation(&mut self, body: &[u8], stuffing: usize) {
|
||||
let af_len = body.len() + stuffing;
|
||||
debug_assert!(af_len <= 183, "adaptation field overflow");
|
||||
self.buf.push(af_len as u8);
|
||||
self.buf.extend_from_slice(body);
|
||||
for _ in 0..stuffing {
|
||||
self.buf.push(STUFF_BYTE);
|
||||
}
|
||||
}
|
||||
|
||||
/// Append payload bytes.
|
||||
pub(super) fn append_payload(&mut self, payload: &[u8]) {
|
||||
self.buf.extend_from_slice(payload);
|
||||
debug_assert!(self.buf.len() <= TS_PACKET_SIZE, "packet overflow");
|
||||
}
|
||||
|
||||
/// Pad the packet to exactly 188 bytes with `0xFF` bytes — used by
|
||||
/// PSI emit paths where the section is much smaller than 184 bytes.
|
||||
/// For PSI packets only — payload-carrying packets reserve room for
|
||||
/// stuffing via `append_adaptation`.
|
||||
pub(super) fn pad_to_188(&mut self) {
|
||||
while self.buf.len() < TS_PACKET_SIZE {
|
||||
self.buf.push(STUFF_BYTE);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn bytes(&self) -> &[u8] {
|
||||
&self.buf
|
||||
}
|
||||
|
||||
pub(super) fn len(&self) -> usize {
|
||||
self.buf.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Buffered writer for assembled TS packets. Owns the underlying sink.
|
||||
pub(super) struct PacketWriter<W: Write> {
|
||||
inner: W,
|
||||
}
|
||||
|
||||
impl<W: Write> PacketWriter<W> {
|
||||
pub(super) fn new(inner: W) -> Self {
|
||||
Self { inner }
|
||||
}
|
||||
|
||||
pub(super) fn write_packet(&mut self, packet: &Packet) -> io::Result<()> {
|
||||
let bytes = packet.bytes();
|
||||
debug_assert_eq!(bytes.len(), TS_PACKET_SIZE);
|
||||
self.inner.write_all(bytes)
|
||||
}
|
||||
|
||||
pub(super) fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn pad_fills_to_188() {
|
||||
let mut p = Packet::new();
|
||||
p.set_header(0x100, true, true, false, 0);
|
||||
p.append_payload(&[1, 2, 3]);
|
||||
p.pad_to_188();
|
||||
assert_eq!(p.bytes().len(), 188);
|
||||
assert_eq!(p.bytes()[0], SYNC_BYTE);
|
||||
// After 4-byte header + 3 payload, byte 7 starts stuffing.
|
||||
assert_eq!(p.bytes()[7], STUFF_BYTE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_pid_round_trips() {
|
||||
let mut p = Packet::new();
|
||||
p.set_header(0x1ABC, false, true, false, 0xA);
|
||||
let pid = u16::from_be_bytes([p.bytes()[1] & 0x1F, p.bytes()[2]]);
|
||||
assert_eq!(pid, 0x1ABC);
|
||||
assert_eq!(p.bytes()[3] & 0x0F, 0xA);
|
||||
}
|
||||
}
|
||||
@@ -92,7 +92,6 @@ impl MkvStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource/FrameSink in follow-up commit.
|
||||
impl crate::pes::Stream for MkvStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
let rs = match self.mode {
|
||||
@@ -189,35 +188,6 @@ impl crate::pes::Stream for MkvStream {
|
||||
}
|
||||
}
|
||||
|
||||
/// FrameSink sibling to the deprecated Stream impl; both coexist during the
|
||||
/// 0.18 deprecation window. Caller may pick either at the trait-object
|
||||
/// boundary — `Box<dyn Stream>` (deprecated) or `Box<dyn FrameSink>` (new).
|
||||
/// Use `MkvStream::create(writer, title)` to construct the write half;
|
||||
/// calling `FrameSink::write` on a `MkvStream::open(reader)` instance returns
|
||||
/// `StreamReadOnly`. `finish` is where the Cues index is written, so it must
|
||||
/// be called for the resulting MKV to be seekable.
|
||||
#[allow(deprecated)] // delegating to deprecated Stream during the 0.18 deprecation window so callers don't see the deprecation twice.
|
||||
impl crate::pes::FrameSink for MkvStream {
|
||||
fn write(&mut self, frame: &crate::pes::PesFrame) -> io::Result<()> {
|
||||
<Self as crate::pes::Stream>::write(self, frame)
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> io::Result<()> {
|
||||
// Why: Stream::finish takes &mut self, FrameSink::finish takes Box<Self>.
|
||||
// Re-borrow inside the box, call Stream::finish, drop the box.
|
||||
// The inner `MkvMuxer` is owned via `Option`, and `Stream::finish`
|
||||
// already takes it via `Option::take()` — moving `*self` out of the
|
||||
// box hands it the same field by-value, so the muxer's own
|
||||
// by-value `finish()` runs correctly.
|
||||
let mut s: Self = *self;
|
||||
<Self as crate::pes::Stream>::finish(&mut s)
|
||||
}
|
||||
|
||||
fn info(&self) -> &crate::disc::DiscTitle {
|
||||
<Self as crate::pes::Stream>::info(self)
|
||||
}
|
||||
}
|
||||
|
||||
// ── MKV header parsing (read side) ────────────────────────────
|
||||
|
||||
/// Returns (DiscTitle, codec_privates: Vec<(track_number, codec_private_bytes)>)
|
||||
|
||||
@@ -33,6 +33,17 @@ pub(crate) mod m2ts;
|
||||
/// to round-trip codec_privates that don't fit inside the underlying format).
|
||||
/// Exposed for integration tests that exercise the wire format directly.
|
||||
pub mod meta;
|
||||
|
||||
// ── Phase 3 sequential muxers ──────────────────────────────────────────────
|
||||
//
|
||||
// New container muxers that consume PES frames and write to a
|
||||
// `SequentialSink`. They are NOT refactors of the existing `MkvStream` /
|
||||
// `M2tsStream` (which round-trip via the legacy `Stream` trait + the
|
||||
// BD-TS framing); they're sequential-only and target the Phase 2 sink
|
||||
// split end-to-end.
|
||||
pub mod fmp4;
|
||||
pub mod hevc;
|
||||
pub mod m2ts_mux;
|
||||
pub(crate) mod mkv;
|
||||
pub(crate) mod mkvstream;
|
||||
pub(crate) mod network;
|
||||
|
||||
@@ -70,7 +70,6 @@ impl NetworkStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource/FrameSink in follow-up commit.
|
||||
impl crate::pes::Stream for NetworkStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
match &mut self.mode {
|
||||
@@ -109,30 +108,6 @@ impl crate::pes::Stream for NetworkStream {
|
||||
}
|
||||
}
|
||||
|
||||
/// FrameSink sibling to the deprecated Stream impl; both coexist during the
|
||||
/// 0.18 deprecation window. Caller may pick either at the trait-object
|
||||
/// boundary — `Box<dyn Stream>` (deprecated) or `Box<dyn FrameSink>` (new).
|
||||
/// Use `NetworkStream::connect(addr).meta(title)` to construct the write
|
||||
/// half; calling `FrameSink::write` on `NetworkStream::listen(addr)` returns
|
||||
/// `StreamReadOnly`.
|
||||
#[allow(deprecated)] // delegating to deprecated Stream during the 0.18 deprecation window so callers don't see the deprecation twice.
|
||||
impl crate::pes::FrameSink for NetworkStream {
|
||||
fn write(&mut self, frame: &crate::pes::PesFrame) -> io::Result<()> {
|
||||
<Self as crate::pes::Stream>::write(self, frame)
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> io::Result<()> {
|
||||
// Why: Stream::finish takes &mut self, FrameSink::finish takes Box<Self>.
|
||||
// Re-borrow inside the box, call Stream::finish, drop the box.
|
||||
let mut s: Self = *self;
|
||||
<Self as crate::pes::Stream>::finish(&mut s)
|
||||
}
|
||||
|
||||
fn info(&self) -> &DiscTitle {
|
||||
<Self as crate::pes::Stream>::info(self)
|
||||
}
|
||||
}
|
||||
|
||||
// NetworkStream is PES-only — no IOStream/Read/Write byte interface.
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -182,7 +157,6 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
#[ignore] // Requires TCP; may be flaky in CI environments
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource/FrameSink in follow-up commit.
|
||||
fn network_pes_roundtrip() {
|
||||
use crate::pes;
|
||||
|
||||
|
||||
+4
-34
@@ -16,7 +16,6 @@ impl NullStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSink in follow-up commit.
|
||||
impl crate::pes::Stream for NullStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
Ok(None)
|
||||
@@ -32,43 +31,16 @@ impl crate::pes::Stream for NullStream {
|
||||
}
|
||||
}
|
||||
|
||||
/// FrameSink sibling to the deprecated Stream impl; both coexist during the
|
||||
/// 0.18 deprecation window. Caller may pick either at the trait-object
|
||||
/// boundary — `Box<dyn Stream>` (deprecated) or `Box<dyn FrameSink>` (new).
|
||||
/// The deprecation-window callers eventually migrate; this impl exists so
|
||||
/// new callers can target `FrameSink` without waiting for the rest of the
|
||||
/// migration to complete.
|
||||
#[allow(deprecated)] // delegating to deprecated Stream during the 0.18 deprecation window so callers don't see the deprecation twice.
|
||||
impl crate::pes::FrameSink for NullStream {
|
||||
fn write(&mut self, frame: &crate::pes::PesFrame) -> io::Result<()> {
|
||||
<Self as crate::pes::Stream>::write(self, frame)
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> io::Result<()> {
|
||||
// Why: Stream::finish takes &mut self, FrameSink::finish takes Box<Self>.
|
||||
// Re-borrow inside the box, call Stream::finish, drop the box.
|
||||
let mut s: Self = *self;
|
||||
<Self as crate::pes::Stream>::finish(&mut s)
|
||||
}
|
||||
|
||||
fn info(&self) -> &DiscTitle {
|
||||
<Self as crate::pes::Stream>::info(self)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::pes::FrameSink;
|
||||
use crate::pes::Stream;
|
||||
|
||||
/// Smallest credible witness that the new FrameSink impl on a concrete
|
||||
/// `mux/*` sink works through the trait object: build a boxed
|
||||
/// `dyn FrameSink`, write a frame, finish it. The trait-bridge correctness
|
||||
/// is what's being verified — not NullStream-specific behaviour.
|
||||
/// Verify NullStream routes through the `Stream` trait object cleanly.
|
||||
#[test]
|
||||
fn frame_sink_via_dyn_object_writes_and_finishes() {
|
||||
fn stream_via_dyn_object_writes_and_finishes() {
|
||||
let title = DiscTitle::empty();
|
||||
let mut sink: Box<dyn FrameSink> = Box::new(NullStream::new(&title));
|
||||
let mut sink: Box<dyn Stream> = Box::new(NullStream::new(&title));
|
||||
|
||||
let frame = crate::pes::PesFrame {
|
||||
track: 0,
|
||||
@@ -77,9 +49,7 @@ mod tests {
|
||||
data: vec![0x01, 0x02, 0x03],
|
||||
};
|
||||
sink.write(&frame).unwrap();
|
||||
// info() routes through the trait object.
|
||||
let _ = sink.info();
|
||||
// finish() consumes the Box<Self> — must compile and run.
|
||||
sink.finish().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
+6
-6
@@ -167,7 +167,6 @@ pub struct InputOptions {
|
||||
}
|
||||
|
||||
/// Open a PES input stream (produces PES frames).
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource in follow-up commit.
|
||||
pub fn input(url: &str, opts: &InputOptions) -> io::Result<Box<dyn crate::pes::Stream>> {
|
||||
let parsed = parse_url(url);
|
||||
match parsed {
|
||||
@@ -187,7 +186,7 @@ pub fn input(url: &str, opts: &InputOptions) -> io::Result<Box<dyn crate::pes::S
|
||||
None => crate::disc::ScanOptions::default(),
|
||||
};
|
||||
let mut reader = super::iso::IsoSectorReader::open(&path.to_string_lossy())?;
|
||||
let capacity = reader.capacity();
|
||||
let capacity = reader.capacity_sectors();
|
||||
let disc = crate::disc::Disc::scan_image(&mut reader, capacity, &scan_opts)
|
||||
.map_err(|e| -> io::Error { e.into() })?;
|
||||
if disc.titles.is_empty() {
|
||||
@@ -239,7 +238,6 @@ pub fn input(url: &str, opts: &InputOptions) -> io::Result<Box<dyn crate::pes::S
|
||||
}
|
||||
|
||||
/// Open a PES output stream (consumes PES frames).
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSink in follow-up commit.
|
||||
pub fn output(
|
||||
url: &str,
|
||||
title: &crate::disc::DiscTitle,
|
||||
@@ -252,11 +250,13 @@ pub fn output(
|
||||
// writeback) so a UHD-scale MKV mux to slow / network-attached
|
||||
// staging doesn't hit the dirty-page burst pathology that
|
||||
// sweep already side-steps. BufWriter sits on top to coalesce
|
||||
// mux's many small EBML element writes.
|
||||
// mux's many small EBML element writes. Pre-reserve the
|
||||
// target's worth of extents on Linux via fallocate(KEEP_SIZE)
|
||||
// to reduce extent fragmentation during the mux.
|
||||
let writer: Box<dyn super::WriteSeek + Send> =
|
||||
Box::new(std::io::BufWriter::with_capacity(
|
||||
IO_BUF_SIZE,
|
||||
crate::io::WritebackFile::create(path)?,
|
||||
crate::io::WritebackFile::create_with_size_hint(path, title.size_bytes)?,
|
||||
));
|
||||
Ok(Box::new(MkvStream::create(writer, title)?))
|
||||
}
|
||||
@@ -264,7 +264,7 @@ pub fn output(
|
||||
validate_file_path(path, "m2ts")?;
|
||||
let writer = std::io::BufWriter::with_capacity(
|
||||
IO_BUF_SIZE,
|
||||
crate::io::WritebackFile::create(path)?,
|
||||
crate::io::WritebackFile::create_with_size_hint(path, title.size_bytes)?,
|
||||
);
|
||||
Ok(Box::new(M2tsStream::create(writer, title)?))
|
||||
}
|
||||
|
||||
@@ -60,7 +60,6 @@ impl StdioStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)] // 0.18 trait split: migrate to FrameSource/FrameSink in follow-up commit.
|
||||
impl crate::pes::Stream for StdioStream {
|
||||
fn read(&mut self) -> io::Result<Option<crate::pes::PesFrame>> {
|
||||
self.ensure_header_read()?;
|
||||
@@ -105,26 +104,3 @@ impl crate::pes::Stream for StdioStream {
|
||||
self.header_read || self.writer.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
/// FrameSink sibling to the deprecated Stream impl; both coexist during the
|
||||
/// 0.18 deprecation window. Caller may pick either at the trait-object
|
||||
/// boundary — `Box<dyn Stream>` (deprecated) or `Box<dyn FrameSink>` (new).
|
||||
/// Use `StdioStream::output(title)` to construct the write half; calling
|
||||
/// `FrameSink::write` on a `StdioStream::input()` returns `StreamReadOnly`.
|
||||
#[allow(deprecated)] // delegating to deprecated Stream during the 0.18 deprecation window so callers don't see the deprecation twice.
|
||||
impl crate::pes::FrameSink for StdioStream {
|
||||
fn write(&mut self, frame: &crate::pes::PesFrame) -> io::Result<()> {
|
||||
<Self as crate::pes::Stream>::write(self, frame)
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> io::Result<()> {
|
||||
// Why: Stream::finish takes &mut self, FrameSink::finish takes Box<Self>.
|
||||
// Re-borrow inside the box, call Stream::finish, drop the box.
|
||||
let mut s: Self = *self;
|
||||
<Self as crate::pes::Stream>::finish(&mut s)
|
||||
}
|
||||
|
||||
fn info(&self) -> &DiscTitle {
|
||||
<Self as crate::pes::Stream>::info(self)
|
||||
}
|
||||
}
|
||||
|
||||
+44
-233
@@ -78,19 +78,20 @@ impl PesFrame {
|
||||
}
|
||||
}
|
||||
|
||||
/// Deprecated; use [`FrameSource`] for read-only sources or [`FrameSink`]
|
||||
/// for write-only sinks. The runtime direction-error semantics
|
||||
/// (`StreamReadOnly` / `StreamWriteOnly` from a wrong-direction call) are
|
||||
/// removed in 0.18 — direction is type-checked.
|
||||
#[deprecated(
|
||||
since = "0.18.0",
|
||||
note = "use FrameSource (read-only) or FrameSink (write-only) instead"
|
||||
)]
|
||||
pub trait Stream {
|
||||
/// Read the next frame, or `Ok(None)` at end of stream.
|
||||
/// A PES frame stream. One trait per format — same type opens for read
|
||||
/// (`open()` / `listen()` / `input()`) or write (`create()` / `connect()` /
|
||||
/// `output()`). Calling the wrong-direction method returns a typed
|
||||
/// `StreamReadOnly` / `StreamWriteOnly` error.
|
||||
///
|
||||
/// `Send` is required so streams can move across the producer/consumer
|
||||
/// threads in autorip's mux pipeline.
|
||||
pub trait Stream: Send {
|
||||
/// Read the next frame, or `Ok(None)` at end of stream. Returns
|
||||
/// `StreamWriteOnly` on a stream opened for writing.
|
||||
fn read(&mut self) -> std::io::Result<Option<PesFrame>>;
|
||||
|
||||
/// Write a frame to the sink.
|
||||
/// Write a frame to the sink. Returns `StreamReadOnly` on a stream
|
||||
/// opened for reading.
|
||||
fn write(&mut self, frame: &PesFrame) -> std::io::Result<()>;
|
||||
|
||||
/// Finalize the stream: flush buffered frames, write any container
|
||||
@@ -116,115 +117,10 @@ pub trait Stream {
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-only source of PES frames.
|
||||
///
|
||||
/// Replaces the read half of the deprecated [`Stream`] trait. Implementors
|
||||
/// produce frames via [`read`](FrameSource::read) and never accept writes —
|
||||
/// passing a `FrameSource` where a sink is expected is a compile error,
|
||||
/// not a runtime `E9001`.
|
||||
///
|
||||
/// `info()` returns the source's `DiscTitle` metadata (track list, codec
|
||||
/// info, duration). It must be stable across the lifetime of the source.
|
||||
///
|
||||
/// `codec_private(track)` exposes per-track codec initialization data
|
||||
/// (H.264 SPS/PPS, HEVC VPS/SPS/PPS, AC-3 fscod, etc.) that downstream
|
||||
/// muxers may need before any frame is written. `headers_ready()` returns
|
||||
/// false until enough input frames have been seen to populate every video
|
||||
/// track's codec-private blob — callers buffer frames they read until
|
||||
/// `headers_ready()` returns true.
|
||||
pub trait FrameSource: Send {
|
||||
/// Read the next frame, or `Ok(None)` at end of stream.
|
||||
fn read(&mut self) -> std::io::Result<Option<PesFrame>>;
|
||||
|
||||
/// Source metadata. Stable across reads — implementors must return a
|
||||
/// consistent reference for the lifetime of the source.
|
||||
fn info(&self) -> &crate::disc::DiscTitle;
|
||||
|
||||
/// Codec initialization data for a track (SPS/PPS, AC-3 fscod, etc.).
|
||||
/// `None` for tracks that don't need codec_private (raw passthrough).
|
||||
fn codec_private(&self, _track: usize) -> Option<Vec<u8>> {
|
||||
None
|
||||
}
|
||||
|
||||
/// True when `codec_private` is available for every video track —
|
||||
/// callers buffer input frames until this flips, since some output
|
||||
/// formats (MKV) can't write frames without codec init data.
|
||||
fn headers_ready(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// Write-only sink of PES frames.
|
||||
///
|
||||
/// Replaces the write half of the deprecated [`Stream`] trait. Implementors
|
||||
/// accept frames via [`write`](FrameSink::write) and finalize via
|
||||
/// [`finish`](FrameSink::finish) — passing a `FrameSink` where a source is
|
||||
/// expected is a compile error, not a runtime `E9000`.
|
||||
///
|
||||
/// `finish` takes `Box<Self>` (rather than `&mut self` like `Stream::finish`)
|
||||
/// so that finalization is a one-shot terminal operation: callers cannot use
|
||||
/// the sink after `finish` returns. This is the standard idiom for terminal
|
||||
/// methods on `dyn Trait` objects.
|
||||
pub trait FrameSink: Send {
|
||||
/// Write a frame to the sink.
|
||||
fn write(&mut self, frame: &PesFrame) -> std::io::Result<()>;
|
||||
|
||||
/// Finalize the sink: flush buffered frames, write any container index
|
||||
/// (e.g. MKV `Cues`), close the underlying file/socket. Consumes the
|
||||
/// sink — callers cannot use it afterwards.
|
||||
fn finish(self: Box<Self>) -> std::io::Result<()>;
|
||||
|
||||
/// Sink metadata. Stable across writes — implementors must return a
|
||||
/// consistent reference for the lifetime of the sink.
|
||||
fn info(&self) -> &crate::disc::DiscTitle;
|
||||
}
|
||||
|
||||
// Bridge: any **`Send`** type implementing the deprecated `Stream` trait
|
||||
// is also a `FrameSource`. This lets existing concrete `Stream` impls in
|
||||
// `mux/*` satisfy `FrameSource` bounds without per-type migration during
|
||||
// the 0.18 deprecation window.
|
||||
//
|
||||
// **Send caveat (read me before tightening `Stream` itself).** This
|
||||
// blanket carries a `T: Send` bound rather than promoting `Send` to a
|
||||
// supertrait of `Stream`, because not every concrete in-tree `Stream`
|
||||
// impl is `Send`: `MkvStream` and `M2tsStream` carry `Box<dyn Read>`
|
||||
// and `Box<dyn Write>` fields whose trait objects don't include `Send`.
|
||||
// Adding `Stream: Send` would force a wider audit (every `Box<dyn Read>`
|
||||
// becomes `Box<dyn Read + Send>`) than this commit is taking on, and
|
||||
// the type-level migration target is `FrameSource` / `FrameSink`
|
||||
// directly anyway. Consequence: coercing a non-Send `Box<dyn Stream>`
|
||||
// (the return shape of `crate::mux::input` / `output`) to
|
||||
// `Box<dyn FrameSource>` will fail with a `T: Send` trait-bound error.
|
||||
// The fix on the consumer side is to construct a Send-compliant
|
||||
// `FrameSource` / `FrameSink` directly rather than relying on this
|
||||
// bridge for non-Send streams.
|
||||
//
|
||||
// Note: `FrameSink` cannot be blanket-impl'd from `Stream` because
|
||||
// `Stream::finish` takes `&mut self` while `FrameSink::finish` takes
|
||||
// `Box<Self>`; concrete types will be migrated in a follow-up commit.
|
||||
#[allow(deprecated)]
|
||||
impl<T: Stream + Send + ?Sized> FrameSource for T {
|
||||
fn read(&mut self) -> std::io::Result<Option<PesFrame>> {
|
||||
<Self as Stream>::read(self)
|
||||
}
|
||||
|
||||
fn info(&self) -> &crate::disc::DiscTitle {
|
||||
<Self as Stream>::info(self)
|
||||
}
|
||||
|
||||
fn codec_private(&self, track: usize) -> Option<Vec<u8>> {
|
||||
<Self as Stream>::codec_private(self, track)
|
||||
}
|
||||
|
||||
fn headers_ready(&self) -> bool {
|
||||
<Self as Stream>::headers_ready(self)
|
||||
}
|
||||
}
|
||||
|
||||
/// Wraps any output stream and counts bytes written.
|
||||
///
|
||||
/// Progress tracking is a CLI concern — streams don't know their size.
|
||||
/// Wrap the output with CountingStream, then query bytes_written().
|
||||
/// Wrap the output with `CountingStream`, then query `bytes_written()`.
|
||||
///
|
||||
/// ```text
|
||||
/// let mut output = CountingStream::new(libfreemkv::output(dest, &title)?);
|
||||
@@ -233,13 +129,11 @@ impl<T: Stream + Send + ?Sized> FrameSource for T {
|
||||
/// let pct = output.bytes_written() as f64 / total as f64;
|
||||
/// }
|
||||
/// ```
|
||||
#[allow(deprecated)]
|
||||
pub struct CountingStream {
|
||||
inner: Box<dyn Stream>,
|
||||
written: u64,
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
impl CountingStream {
|
||||
pub fn new(inner: Box<dyn Stream>) -> Self {
|
||||
Self { inner, written: 0 }
|
||||
@@ -251,7 +145,6 @@ impl CountingStream {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
impl Stream for CountingStream {
|
||||
fn read(&mut self) -> std::io::Result<Option<PesFrame>> {
|
||||
self.inner.read()
|
||||
@@ -293,88 +186,35 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Direct `FrameSource` impl (not via the deprecated `Stream` blanket).
|
||||
struct MockSource {
|
||||
frames: std::vec::IntoIter<PesFrame>,
|
||||
/// Minimal in-memory `Stream` for trait-shape tests. `read` replays
|
||||
/// pre-seeded frames; `write` collects them.
|
||||
struct MockStream {
|
||||
read_queue: std::vec::IntoIter<PesFrame>,
|
||||
written: Vec<PesFrame>,
|
||||
title: DiscTitle,
|
||||
}
|
||||
|
||||
impl MockSource {
|
||||
fn new(frames: Vec<PesFrame>) -> Self {
|
||||
impl MockStream {
|
||||
fn new(read_frames: Vec<PesFrame>) -> Self {
|
||||
Self {
|
||||
frames: frames.into_iter(),
|
||||
read_queue: read_frames.into_iter(),
|
||||
written: Vec::new(),
|
||||
title: DiscTitle::empty(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FrameSource for MockSource {
|
||||
impl Stream for MockStream {
|
||||
fn read(&mut self) -> std::io::Result<Option<PesFrame>> {
|
||||
Ok(self.frames.next())
|
||||
Ok(self.read_queue.next())
|
||||
}
|
||||
|
||||
fn info(&self) -> &DiscTitle {
|
||||
&self.title
|
||||
}
|
||||
}
|
||||
|
||||
/// Direct `FrameSink` impl (not via the deprecated `Stream` blanket).
|
||||
struct MockSink {
|
||||
collected: Vec<PesFrame>,
|
||||
title: DiscTitle,
|
||||
}
|
||||
|
||||
impl MockSink {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
collected: Vec::new(),
|
||||
title: DiscTitle::empty(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FrameSink for MockSink {
|
||||
fn write(&mut self, frame: &PesFrame) -> std::io::Result<()> {
|
||||
self.collected.push(frame.clone());
|
||||
self.written.push(frame.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> std::io::Result<()> {
|
||||
// Drop self; in real sinks this is where flush/fsync/close happens.
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn info(&self) -> &DiscTitle {
|
||||
&self.title
|
||||
}
|
||||
}
|
||||
|
||||
/// Variant of `MockSink` whose `finish` returns the collected frames so
|
||||
/// the test can assert on them after the consuming `Box<Self>` call.
|
||||
struct CollectingSink {
|
||||
collected: Vec<PesFrame>,
|
||||
title: DiscTitle,
|
||||
}
|
||||
|
||||
impl CollectingSink {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
collected: Vec::new(),
|
||||
title: DiscTitle::empty(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FrameSink for CollectingSink {
|
||||
fn write(&mut self, frame: &PesFrame) -> std::io::Result<()> {
|
||||
self.collected.push(frame.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(self: Box<Self>) -> std::io::Result<()> {
|
||||
// Real CollectingSink consumers would expose `take()` before
|
||||
// finish; this trait method just confirms the boxed signature
|
||||
// compiles and runs.
|
||||
fn finish(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -384,72 +224,43 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_source_yields_frames_then_eof() {
|
||||
fn stream_read_yields_frames_then_eof() {
|
||||
let frames = vec![make_frame(0, 0), make_frame(1, 1_000), make_frame(0, 2_000)];
|
||||
let mut src = MockSource::new(frames.clone());
|
||||
let mut s = MockStream::new(frames.clone());
|
||||
|
||||
let f0 = src.read().unwrap().expect("first frame");
|
||||
let f0 = s.read().unwrap().expect("first frame");
|
||||
assert_eq!(f0.track, frames[0].track);
|
||||
assert_eq!(f0.pts, frames[0].pts);
|
||||
assert!(f0.keyframe);
|
||||
|
||||
let f1 = src.read().unwrap().expect("second frame");
|
||||
assert_eq!(f1.track, frames[1].track);
|
||||
let f1 = s.read().unwrap().expect("second frame");
|
||||
assert_eq!(f1.pts, frames[1].pts);
|
||||
|
||||
let f2 = src.read().unwrap().expect("third frame");
|
||||
assert_eq!(f2.track, frames[2].track);
|
||||
let f2 = s.read().unwrap().expect("third frame");
|
||||
assert_eq!(f2.pts, frames[2].pts);
|
||||
|
||||
assert!(src.read().unwrap().is_none());
|
||||
assert!(src.read().unwrap().is_none()); // idempotent at EOF
|
||||
assert!(s.read().unwrap().is_none());
|
||||
assert!(s.read().unwrap().is_none()); // idempotent at EOF
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_sink_collects_then_finishes() {
|
||||
let mut sink = MockSink::new();
|
||||
fn stream_write_collects_then_finishes() {
|
||||
let mut s = MockStream::new(Vec::new());
|
||||
let frames = [make_frame(0, 0), make_frame(1, 100), make_frame(2, 200)];
|
||||
|
||||
for f in &frames {
|
||||
sink.write(f).unwrap();
|
||||
s.write(f).unwrap();
|
||||
}
|
||||
assert_eq!(sink.collected.len(), 3);
|
||||
assert_eq!(sink.collected[0].pts, 0);
|
||||
assert_eq!(sink.collected[1].pts, 100);
|
||||
assert_eq!(sink.collected[2].pts, 200);
|
||||
|
||||
// Box-and-finish — the `self: Box<Self>` shape must compile and run.
|
||||
Box::new(sink).finish().unwrap();
|
||||
assert_eq!(s.written.len(), 3);
|
||||
s.finish().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_sink_via_dyn_object() {
|
||||
let frames = [make_frame(0, 0), make_frame(0, 33)];
|
||||
let mut sink: Box<dyn FrameSink> = Box::new(CollectingSink::new());
|
||||
|
||||
for f in &frames {
|
||||
sink.write(f).unwrap();
|
||||
}
|
||||
// info() routes through the trait object.
|
||||
let _ = sink.info();
|
||||
sink.finish().unwrap();
|
||||
}
|
||||
|
||||
/// The deprecated blanket impl: any concrete `Stream` should also act as
|
||||
/// a `FrameSource`. `NullStream` has the simplest constructor of every
|
||||
/// concrete `Stream` impl in `mux/*`, so it's the smallest credible
|
||||
/// witness that the bridge compiles and dispatches correctly.
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn deprecated_stream_satisfies_frame_source() {
|
||||
let title = DiscTitle::empty();
|
||||
let mut null = crate::mux::NullStream::new(&title);
|
||||
|
||||
let src: &mut dyn FrameSource = &mut null;
|
||||
// NullStream::read returns Ok(None) — it's a write-only sink.
|
||||
assert!(src.read().unwrap().is_none());
|
||||
// info() forwards through the blanket impl.
|
||||
let _ = src.info();
|
||||
assert!(src.headers_ready());
|
||||
fn stream_via_dyn_object() {
|
||||
let mut s: Box<dyn Stream> = Box::new(MockStream::new(vec![make_frame(0, 0)]));
|
||||
let frame = s.read().unwrap().expect("first frame");
|
||||
s.write(&frame).unwrap();
|
||||
let _ = s.info();
|
||||
s.finish().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
//! Linux `statfs64`-based filesystem type detection.
|
||||
//!
|
||||
//! Recognised local-FS magics: ext2/3/4, xfs, btrfs, tmpfs. NFS is the
|
||||
//! one network FS this layer cares about (the buffering decision keys
|
||||
//! off it). Anything else maps to [`FsType::Unknown`].
|
||||
|
||||
use std::ffi::CString;
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
use std::path::Path;
|
||||
|
||||
use super::FsType;
|
||||
|
||||
// Magic numbers from `<linux/magic.h>`. Kept literal here so we don't
|
||||
// depend on libc exposing each one — only `NFS_SUPER_MAGIC` is
|
||||
// guaranteed to be present across libc / musl revisions.
|
||||
const EXT2_SUPER_MAGIC: i64 = 0xEF53;
|
||||
const XFS_SUPER_MAGIC: i64 = 0x5846_5342;
|
||||
const BTRFS_SUPER_MAGIC: i64 = 0x9123_683E;
|
||||
const TMPFS_MAGIC: i64 = 0x0102_1994;
|
||||
|
||||
pub(super) fn detect_impl(path: &Path) -> FsType {
|
||||
let cpath = match CString::new(path.as_os_str().as_bytes()) {
|
||||
Ok(c) => c,
|
||||
Err(_) => return FsType::Unknown,
|
||||
};
|
||||
// `statfs64` is repr(C); zeroing is the documented init pattern for
|
||||
// the kernel uapi struct.
|
||||
let mut buf: libc::statfs64 = unsafe { std::mem::zeroed() };
|
||||
let rc = unsafe { libc::statfs64(cpath.as_ptr(), &mut buf) };
|
||||
if rc != 0 {
|
||||
return FsType::Unknown;
|
||||
}
|
||||
// `f_type` is signed (`__fsword_t`) on glibc and unsigned
|
||||
// (`c_ulong`) on musl. Cast both sides to i64 for a portable
|
||||
// comparison. On glibc x86_64 both already are i64 — clippy flags
|
||||
// the cast as unnecessary on that target only, but we need it for
|
||||
// musl, so silence the lint.
|
||||
#[allow(clippy::unnecessary_cast)]
|
||||
let f_type = buf.f_type as i64;
|
||||
#[allow(clippy::unnecessary_cast)]
|
||||
let nfs_magic = libc::NFS_SUPER_MAGIC as i64;
|
||||
if f_type == nfs_magic {
|
||||
return FsType::Nfs;
|
||||
}
|
||||
match f_type {
|
||||
EXT2_SUPER_MAGIC | XFS_SUPER_MAGIC | BTRFS_SUPER_MAGIC | TMPFS_MAGIC => FsType::Local,
|
||||
_ => FsType::Unknown,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
//! macOS `statfs`-based filesystem type detection.
|
||||
//!
|
||||
//! macOS exposes a textual `f_fstypename` field (e.g. `"apfs"`, `"hfs"`,
|
||||
//! `"nfs"`, `"smbfs"`) on its `statfs` struct, which is far more
|
||||
//! reliable than chasing magic numbers. Anything starting with `"nfs"`
|
||||
//! counts as NFS; anything else recognised maps to `Local`; otherwise
|
||||
//! `Unknown`.
|
||||
|
||||
use std::ffi::CString;
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
use std::path::Path;
|
||||
|
||||
use super::FsType;
|
||||
|
||||
pub(super) fn detect_impl(path: &Path) -> FsType {
|
||||
let cpath = match CString::new(path.as_os_str().as_bytes()) {
|
||||
Ok(c) => c,
|
||||
Err(_) => return FsType::Unknown,
|
||||
};
|
||||
let mut buf: libc::statfs = unsafe { std::mem::zeroed() };
|
||||
let rc = unsafe { libc::statfs(cpath.as_ptr(), &mut buf) };
|
||||
if rc != 0 {
|
||||
return FsType::Unknown;
|
||||
}
|
||||
// `f_fstypename` is a NUL-terminated C string of length MFSTYPENAMELEN.
|
||||
// SAFETY: libc guarantees the field is initialised to a NUL-terminated
|
||||
// string by a successful statfs.
|
||||
let name_ptr = buf.f_fstypename.as_ptr();
|
||||
let cstr = unsafe { std::ffi::CStr::from_ptr(name_ptr) };
|
||||
let name = cstr.to_bytes();
|
||||
if name.starts_with(b"nfs") {
|
||||
return FsType::Nfs;
|
||||
}
|
||||
// Recognised local types. SMB is not NFS but the buffering-policy
|
||||
// outcome on macOS is the same as for any other local FS — there's
|
||||
// no `WritebackFile` machinery to opt out of on this OS.
|
||||
match name {
|
||||
b"apfs" | b"hfs" | b"exfat" | b"msdos" | b"tmpfs" | b"smbfs" | b"webdav" => FsType::Local,
|
||||
_ => FsType::Unknown,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
//! Filesystem-type detection.
|
||||
//!
|
||||
//! The buffering architecture (Phase 2) selects different output sinks
|
||||
//! for local vs network filesystems: NFS gets the adaptive
|
||||
//! `WritebackFile` machinery on Linux; local disks get `LocalFileSink`
|
||||
//! and rely on the kernel's default writeback policy. This module
|
||||
//! provides the construction-site primitive that picks which one.
|
||||
//!
|
||||
//! Per the per-OS file-split convention, the actual `statfs` call lives
|
||||
//! in the matching platform file (`linux.rs`, `macos.rs`, `windows.rs`,
|
||||
//! `other.rs`); this `mod.rs` exposes only the cross-platform enum and
|
||||
//! the `detect` entry point.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
/// What kind of filesystem a path lives on, to the extent we can tell
|
||||
/// cheaply at construction time.
|
||||
///
|
||||
/// `Unknown` is the fail-open default: a misdetection here should not
|
||||
/// be load-bearing for correctness, only for the choice of sink (and
|
||||
/// hence buffering policy). Callers that need a binary local/non-local
|
||||
/// answer should treat `Unknown` as "probably local".
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum FsType {
|
||||
/// A local on-disk filesystem (ext4, xfs, btrfs, apfs, ntfs, …).
|
||||
Local,
|
||||
/// A network filesystem with NFS-like semantics. The current
|
||||
/// detector lumps SMB / UNC into this on Windows because the
|
||||
/// buffering policy outcome is the same.
|
||||
Nfs,
|
||||
/// `statfs` failed, the filesystem type is not on our recognised
|
||||
/// list, or we are on an OS without a real implementation.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
#[cfg(target_os = "macos")]
|
||||
mod macos;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
mod other;
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use linux::detect_impl;
|
||||
#[cfg(target_os = "macos")]
|
||||
use macos::detect_impl;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
use other::detect_impl;
|
||||
#[cfg(target_os = "windows")]
|
||||
use windows::detect_impl;
|
||||
|
||||
/// Best-effort classification of the filesystem under `path`.
|
||||
///
|
||||
/// Falls back to [`FsType::Unknown`] on any syscall error or unrecognised
|
||||
/// filesystem signature. Never panics. Never blocks beyond the cost of
|
||||
/// a single `statfs(2)` (Unix) or a string check (Windows).
|
||||
pub fn detect(path: &Path) -> FsType {
|
||||
detect_impl(path)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn detect_does_not_panic_on_missing_path() {
|
||||
// Non-existent path should fall through to Unknown, not panic.
|
||||
let p = std::path::Path::new("/this/path/should/not/exist/freemkv-test");
|
||||
let r = detect(p);
|
||||
// We don't assert == Unknown because Windows' heuristic looks at
|
||||
// the leading bytes and might still classify; just confirm the
|
||||
// call returns rather than panicking.
|
||||
let _ = r;
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
#[test]
|
||||
fn macos_tmp_is_local() {
|
||||
// `/tmp` on macOS dev rigs is APFS via the symlink to
|
||||
// `/private/tmp`. Either way, never NFS.
|
||||
let r = detect(std::path::Path::new("/tmp"));
|
||||
assert!(
|
||||
matches!(r, FsType::Local | FsType::Unknown),
|
||||
"expected Local or Unknown for /tmp on macOS, got {r:?}",
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn linux_tmp_is_local_or_unknown() {
|
||||
// `/tmp` is tmpfs on most distros (which we recognise) but
|
||||
// could be ext4 on others. NFS would be unusual.
|
||||
let r = detect(std::path::Path::new("/tmp"));
|
||||
assert!(
|
||||
matches!(r, FsType::Local | FsType::Unknown),
|
||||
"expected Local or Unknown for /tmp on Linux, got {r:?}",
|
||||
);
|
||||
}
|
||||
|
||||
/// Real NFS exercise needs an actual NFS mount and isn't available
|
||||
/// in CI. Kept here as a manual probe.
|
||||
#[test]
|
||||
#[ignore = "needs an NFS mount path (e.g. /mnt/nfs/...) to validate live"]
|
||||
fn nfs_path_classified_as_nfs() {
|
||||
// Operator passes the mount as FREEMKV_NFS_PROBE; gated behind
|
||||
// `--ignored` because there's no portable NFS path.
|
||||
let p = std::env::var("FREEMKV_NFS_PROBE").expect("set FREEMKV_NFS_PROBE");
|
||||
assert_eq!(detect(std::path::Path::new(&p)), FsType::Nfs);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
//! Fallback fs-type detection for platforms without a specific impl.
|
||||
//!
|
||||
//! Always returns [`FsType::Unknown`]. Callers treat that as
|
||||
//! "probably local" and pick `LocalFileSink`.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use super::FsType;
|
||||
|
||||
pub(super) fn detect_impl(_path: &Path) -> FsType {
|
||||
FsType::Unknown
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
//! Windows filesystem-type detection.
|
||||
//!
|
||||
//! Heuristic-only: any UNC path (`\\server\share\...`) is treated as a
|
||||
//! network mount and bucketed into `Nfs`. Strictly, SMB is not NFS, but
|
||||
//! the buffering-policy outcome for our purposes is the same — there is
|
||||
//! no platform `WritebackFile` machinery on Windows yet, so the worst
|
||||
//! case of a false positive is using `LocalFileSink` regardless. A
|
||||
//! proper `GetVolumeInformation` query is a Phase 4 concern.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use super::FsType;
|
||||
|
||||
pub(super) fn detect_impl(path: &Path) -> FsType {
|
||||
// `Path::starts_with("\\\\")` won't match because component-wise
|
||||
// matching strips the prefix. Look at the raw OsStr instead.
|
||||
let s = path.as_os_str();
|
||||
// OsStr -> [u16] is the platform-correct way on Windows, but
|
||||
// checking the leading bytes via `to_string_lossy` is good enough
|
||||
// for a UNC prefix probe and works on any encoding.
|
||||
let lossy = s.to_string_lossy();
|
||||
if lossy.starts_with("\\\\") || lossy.starts_with("//") {
|
||||
return FsType::Nfs;
|
||||
}
|
||||
FsType::Local
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! Platform-specific drive initialization and disc probing.
|
||||
|
||||
pub mod fs_type;
|
||||
pub mod mt1959;
|
||||
|
||||
use crate::error::Result;
|
||||
|
||||
+12
-81
@@ -1,83 +1,19 @@
|
||||
//! File-backed sector I/O — read and write 2048-byte sectors against
|
||||
//! an ISO image on disk.
|
||||
//! File-backed sector sink — write 2048-byte sectors to an ISO image
|
||||
//! on disk.
|
||||
//!
|
||||
//! [`FileSectorSource`] is the read side (open-only). [`FileSectorSink`]
|
||||
//! is the write side (create or open-rw); writes go through
|
||||
//! [`crate::io::WritebackFile`] so big sequential ISO writes share
|
||||
//! the same bounded-cache writeback pipeline used by sweep / patch /
|
||||
//! mux.
|
||||
//! The read-side counterpart ([`crate::io::file_sector_source::FileSectorSource`])
|
||||
//! lives under `io/` because its internals (read-ahead buffer, per-OS
|
||||
//! `fadvise`/`F_RDADVISE` hints) are I/O infrastructure rather than
|
||||
//! sector-trait business logic. Both types remain re-exported at
|
||||
//! [`crate::sector`] for ergonomic imports.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
use super::{SectorReader, SectorSink};
|
||||
|
||||
/// SectorSource backed by a file (ISO image).
|
||||
///
|
||||
/// Seeks to `lba * 2048`, reads `count * 2048` bytes per call. The
|
||||
/// file is held directly: every `read_sectors` call performs an
|
||||
/// absolute seek, so a wrapping `BufReader` would have its buffer
|
||||
/// invalidated on every call (its internal cursor moves with the
|
||||
/// `Seek` impl) — pure overhead. Callers that benefit from buffered
|
||||
/// reads should compose their own `BufReader` at the `read_sectors`
|
||||
/// granularity they care about.
|
||||
pub struct FileSectorSource {
|
||||
file: File,
|
||||
capacity: u32,
|
||||
}
|
||||
|
||||
impl FileSectorSource {
|
||||
/// Open an existing ISO file for reading. Capacity is derived
|
||||
/// from `metadata().len() / 2048`. Returns
|
||||
/// [`Error::IsoTooLarge`] if the file would exceed the 32-bit
|
||||
/// LBA address space (~8 TB).
|
||||
pub fn open(path: &Path) -> std::io::Result<Self> {
|
||||
let file = File::open(path)?;
|
||||
let len = file.metadata()?.len();
|
||||
let sectors = len / 2048;
|
||||
if sectors > u32::MAX as u64 {
|
||||
return Err(Error::IsoTooLarge {
|
||||
path: path.to_string_lossy().into_owned(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let capacity = sectors as u32;
|
||||
Ok(Self { file, capacity })
|
||||
}
|
||||
}
|
||||
|
||||
// Implement the legacy `SectorReader` trait. The blanket impl in
|
||||
// `super` produces the `SectorSource` impl automatically — no need
|
||||
// to write both, and writing both would conflict. This keeps the
|
||||
// 0.17 method-resolution path intact (callers with `SectorReader`
|
||||
// in scope can still write `fsr.read_sectors(..)` against a
|
||||
// `FileSectorSource`).
|
||||
impl SectorReader for FileSectorSource {
|
||||
fn capacity(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
let offset = lba as u64 * 2048;
|
||||
let bytes = count as usize * 2048;
|
||||
self.file
|
||||
.seek(SeekFrom::Start(offset))
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
self.file
|
||||
.read_exact(&mut buf[..bytes])
|
||||
.map_err(|e| Error::IoError { source: e })?;
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
use super::SectorSink;
|
||||
|
||||
/// SectorSink backed by a file (ISO image).
|
||||
///
|
||||
@@ -146,13 +82,8 @@ impl SectorSink for FileSectorSink {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
// Bring the 0.18 trait into scope (not super::*: the super
|
||||
// module also re-exports the legacy `SectorReader`, and
|
||||
// having both `SectorReader::read_sectors` and
|
||||
// `SectorSource::read_sectors` visible would force every
|
||||
// call site to disambiguate). External consumers see the
|
||||
// same surface this test exercises.
|
||||
use super::{FileSectorSink, FileSectorSource};
|
||||
use super::FileSectorSink;
|
||||
use crate::io::file_sector_source::FileSectorSource;
|
||||
use crate::sector::{SectorSink, SectorSource};
|
||||
use tempfile::tempdir;
|
||||
|
||||
|
||||
+62
-150
@@ -1,29 +1,17 @@
|
||||
//! Sector-level I/O traits.
|
||||
//!
|
||||
//! 0.18 splits the unidirectional read trait from a write trait at
|
||||
//! the sector layer, so the type system catches "wrong direction"
|
||||
//! mistakes at compile time instead of runtime. See
|
||||
//! `(internal)/memory/0_18_redesign.md`.
|
||||
//! The sector layer is direction-typed: [`SectorSource`] reads
|
||||
//! 2048-byte sectors, [`SectorSink`] writes them. Concrete impls
|
||||
//! never do both — physical drives are read-only, file-backed
|
||||
//! ISO images are opened for read OR write at construction time.
|
||||
//!
|
||||
//! - [`SectorSource`] reads 2048-byte sectors. Implemented by
|
||||
//! `Drive` (via the legacy [`SectorReader`] alias) and
|
||||
//! [`FileSectorSource`] (ISO-backed).
|
||||
//! - [`SectorSink`] writes 2048-byte sectors. Implemented by
|
||||
//! [`FileSectorSink`] (ISO-backed) and, in later commits, by
|
||||
//! sweep/patch consumer adapters.
|
||||
//! - [`SectorSource`] is implemented by `Drive` (hardware) and
|
||||
//! [`FileSectorSource`] / `IsoSectorReader` (file-backed).
|
||||
//! - [`SectorSink`] is implemented by [`FileSectorSink`]
|
||||
//! (ISO-backed) and sweep/patch consumer adapters.
|
||||
//! - [`DecryptingSectorSource`] is a decorator that wraps any
|
||||
//! `SectorSource` and applies the existing AACS / CSS in-place
|
||||
//! decrypt to plaintext-out.
|
||||
//!
|
||||
//! [`SectorReader`] is the 0.17 read trait. It stays on through
|
||||
//! the 0.18 migration window so existing call sites
|
||||
//! (`Drive`, `IsoSectorReader`, `BufferedSectorReader`,
|
||||
//! `DiscStream`, `verify`) compile unchanged. A blanket impl
|
||||
//! forwards every `SectorReader` impl to `SectorSource`, so new
|
||||
//! code should target `SectorSource` / `SectorSink` directly. The
|
||||
//! formal `#[deprecated]` attribute lands once the internal
|
||||
//! callers have migrated; see the comment on `SectorReader` for
|
||||
//! why this commit holds it back.
|
||||
//! `SectorSource` and applies AACS / CSS in-place decrypt to
|
||||
//! yield plaintext sectors.
|
||||
|
||||
pub mod decrypting;
|
||||
pub mod file;
|
||||
@@ -32,13 +20,14 @@ use crate::error::Result;
|
||||
|
||||
/// Read 2048-byte sectors from a disc, image, or composed source.
|
||||
///
|
||||
/// Direction-typed: a `SectorSource` cannot be written to. Wrap the
|
||||
/// inner source in [`DecryptingSectorSource`] to get plaintext
|
||||
/// sectors out of an encrypted disc.
|
||||
/// Wrap the inner source in [`DecryptingSectorSource`] to get
|
||||
/// plaintext sectors out of an encrypted disc.
|
||||
pub trait SectorSource: Send {
|
||||
/// Total capacity in sectors, if known. Returns 0 when unknown
|
||||
/// Total capacity in sectors, if known. Default `0` = unknown
|
||||
/// (e.g. live drives that haven't completed `READ CAPACITY` yet).
|
||||
fn capacity_sectors(&self) -> u32;
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
0
|
||||
}
|
||||
|
||||
/// Read `count` sectors starting at `lba` into `buf`.
|
||||
/// `buf` must be at least `count * 2048` bytes.
|
||||
@@ -60,10 +49,52 @@ pub trait SectorSource: Send {
|
||||
fn set_speed(&mut self, _kbs: u16) {}
|
||||
}
|
||||
|
||||
// Forwarding impls so `Box<dyn SectorSource>` and `&mut dyn SectorSource`
|
||||
// satisfy the `SectorSource` trait bound when wrapped by generic
|
||||
// decorators like `DecryptingSectorSource<S: SectorSource>`.
|
||||
impl SectorSource for Box<dyn SectorSource> {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
(**self).capacity_sectors()
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize> {
|
||||
(**self).read_sectors(lba, count, buf, recovery)
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
(**self).set_speed(kbs)
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for &mut (dyn SectorSource + '_) {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
(**self).capacity_sectors()
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize> {
|
||||
(**self).read_sectors(lba, count, buf, recovery)
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
(**self).set_speed(kbs)
|
||||
}
|
||||
}
|
||||
|
||||
/// Write 2048-byte sectors to a disc image or composed sink.
|
||||
///
|
||||
/// Direction-typed: a `SectorSink` cannot be read from. The
|
||||
/// terminal [`finish`] takes `Box<Self>` so it can run on `dyn
|
||||
/// The terminal [`finish`] takes `Box<Self>` so it can run on `dyn
|
||||
/// SectorSink` and consume the sink (`fsync` + close).
|
||||
///
|
||||
/// [`finish`]: SectorSink::finish
|
||||
@@ -78,125 +109,6 @@ pub trait SectorSink: Send {
|
||||
fn finish(self: Box<Self>) -> Result<()>;
|
||||
}
|
||||
|
||||
/// 0.17 read trait. Slated for removal once internal call sites
|
||||
/// migrate to [`SectorSource`] in follow-up commits; until then
|
||||
/// it remains the trait that `Drive`, `IsoSectorReader`,
|
||||
/// `BufferedSectorReader`, and existing `&mut dyn SectorReader`
|
||||
/// signatures use unchanged.
|
||||
///
|
||||
/// New code should implement [`SectorSource`] directly. The
|
||||
/// blanket impl below makes any `SectorReader` automatically
|
||||
/// usable wherever a `SectorSource` is expected, so a one-way
|
||||
/// migration off `SectorReader` is possible per-callsite without
|
||||
/// touching the impls.
|
||||
//
|
||||
// NOTE: not marked `#[deprecated]` in this commit — `cargo clippy
|
||||
// -- -D warnings` (the CI gauntlet) treats deprecation as an
|
||||
// error, and the existing `Drive` / `udf::BufferedSectorReader` /
|
||||
// `mux::DiscStream` / `verify` call sites all go through this
|
||||
// trait. The deprecation attribute lands together with the
|
||||
// migration commits that move those call sites to
|
||||
// `SectorSource`. The behavioural contract — "this trait is
|
||||
// going away in 0.18" — is documented above and tracked in
|
||||
// `(internal)/memory/0_18_redesign.md`.
|
||||
pub trait SectorReader: Send {
|
||||
/// Read `count` sectors starting at `lba` into `buf`.
|
||||
/// See [`SectorSource::read_sectors`] for semantics.
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize>;
|
||||
|
||||
/// Total capacity in sectors, if known.
|
||||
fn capacity(&self) -> u32 {
|
||||
0
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, _kbs: u16) {}
|
||||
}
|
||||
|
||||
// Blanket impl: anything implementing the legacy `SectorReader`
|
||||
// trait automatically satisfies `SectorSource`. This is what keeps
|
||||
// existing impls (`Drive`, `IsoSectorReader`, `BufferedSectorReader`,
|
||||
// etc.) compiling without source changes during the migration. The
|
||||
// reverse direction (impl SectorReader for SectorSource) is
|
||||
// intentionally NOT provided — new code targets the new trait.
|
||||
impl<T: SectorReader + ?Sized> SectorSource for T {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
<T as SectorReader>::capacity(self)
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize> {
|
||||
<T as SectorReader>::read_sectors(self, lba, count, buf, recovery)
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
<T as SectorReader>::set_speed(self, kbs)
|
||||
}
|
||||
}
|
||||
|
||||
// Forwarding impls so callers can wrap `&mut dyn SectorReader` /
|
||||
// `Box<dyn SectorReader>` in [`DecryptingSectorSource`] without
|
||||
// having to unbox or re-borrow inside the lib's hot paths. The
|
||||
// generic `&mut T` / `Box<T>` blankets would conflict with the
|
||||
// `SectorReader → SectorSource` blanket above (a downstream crate
|
||||
// could `impl SectorReader for &mut U`); the specific
|
||||
// `dyn SectorReader` instantiations are unambiguous because
|
||||
// `SectorReader` is the very trait whose `dyn` we're targeting.
|
||||
impl SectorSource for &mut (dyn SectorReader + '_) {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
<dyn SectorReader as SectorReader>::capacity(*self)
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize> {
|
||||
<dyn SectorReader as SectorReader>::read_sectors(*self, lba, count, buf, recovery)
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
<dyn SectorReader as SectorReader>::set_speed(*self, kbs)
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for Box<dyn SectorReader> {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
<dyn SectorReader as SectorReader>::capacity(&**self)
|
||||
}
|
||||
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
recovery: bool,
|
||||
) -> Result<usize> {
|
||||
<dyn SectorReader as SectorReader>::read_sectors(&mut **self, lba, count, buf, recovery)
|
||||
}
|
||||
|
||||
fn set_speed(&mut self, kbs: u16) {
|
||||
<dyn SectorReader as SectorReader>::set_speed(&mut **self, kbs)
|
||||
}
|
||||
}
|
||||
|
||||
pub use crate::io::file_sector_source::FileSectorSource;
|
||||
pub use decrypting::DecryptingSectorSource;
|
||||
pub use file::{FileSectorSink, FileSectorSource};
|
||||
|
||||
// Backwards-compat alias for the public API. `FileSectorReader` is
|
||||
// the 0.17 name; new code uses `FileSectorSource`. Both point at
|
||||
// the same type. The `#[deprecated]` attribute lands together with
|
||||
// the migration commits that retire the alias from internal uses.
|
||||
pub type FileSectorReader = FileSectorSource;
|
||||
pub use file::FileSectorSink;
|
||||
|
||||
+17
-17
@@ -19,7 +19,7 @@
|
||||
//! BD-ROM Part 3 — Blu-ray filesystem profile
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
|
||||
/// A UDF filesystem parsed from disc.
|
||||
#[derive(Debug)]
|
||||
@@ -84,7 +84,7 @@ impl UdfFs {
|
||||
/// Reads sector by sector from disc — no buffering.
|
||||
/// Get the absolute starting LBA of a file on disc.
|
||||
/// Used by the rip pipeline to locate m2ts content sectors.
|
||||
pub fn file_start_lba(&self, reader: &mut dyn SectorReader, path: &str) -> Result<u32> {
|
||||
pub fn file_start_lba(&self, reader: &mut dyn SectorSource, path: &str) -> Result<u32> {
|
||||
let parts: Vec<&str> = path.trim_matches('/').split('/').collect();
|
||||
let mut current = &self.root;
|
||||
for part in &parts[..parts.len() - 1] {
|
||||
@@ -115,7 +115,7 @@ impl UdfFs {
|
||||
Ok(self.partition_start + data_lba)
|
||||
}
|
||||
|
||||
pub fn read_file(&self, reader: &mut dyn SectorReader, path: &str) -> Result<Vec<u8>> {
|
||||
pub fn read_file(&self, reader: &mut dyn SectorSource, path: &str) -> Result<Vec<u8>> {
|
||||
let parts: Vec<&str> = path.trim_matches('/').split('/').collect();
|
||||
let mut current = &self.root;
|
||||
|
||||
@@ -175,7 +175,7 @@ impl UdfFs {
|
||||
///
|
||||
/// Skips: STREAM/ (video), BACKUP/, DUPLICATE/,
|
||||
/// MKB_RO.inf, ContentHash*, ContentRevocation*
|
||||
pub fn metadata_sector_ranges(&self, reader: &mut dyn SectorReader) -> Result<Vec<(u32, u32)>> {
|
||||
pub fn metadata_sector_ranges(&self, reader: &mut dyn SectorSource) -> Result<Vec<(u32, u32)>> {
|
||||
let mut ranges = Vec::new();
|
||||
|
||||
// UDF structure: sector 0 through end of metadata partition
|
||||
@@ -194,7 +194,7 @@ impl UdfFs {
|
||||
|
||||
/// All sector ranges that contain data (metadata + all files including STREAM).
|
||||
/// For full disc-to-ISO dumps — reads only allocated sectors, skips gaps.
|
||||
pub fn all_sector_ranges(&self, reader: &mut dyn SectorReader) -> Result<Vec<(u32, u32)>> {
|
||||
pub fn all_sector_ranges(&self, reader: &mut dyn SectorSource) -> Result<Vec<(u32, u32)>> {
|
||||
let mut ranges = Vec::new();
|
||||
|
||||
// UDF structure sectors
|
||||
@@ -212,7 +212,7 @@ impl UdfFs {
|
||||
|
||||
fn collect_all_file_ranges(
|
||||
&self,
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
entry: &DirEntry,
|
||||
ranges: &mut Vec<(u32, u32)>,
|
||||
) -> Result<()> {
|
||||
@@ -238,7 +238,7 @@ impl UdfFs {
|
||||
|
||||
fn collect_file_ranges(
|
||||
&self,
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
entry: &DirEntry,
|
||||
ranges: &mut Vec<(u32, u32)>,
|
||||
) -> Result<()> {
|
||||
@@ -276,7 +276,7 @@ impl UdfFs {
|
||||
/// Read an Extended File Entry (tag 266) or File Entry (tag 261)
|
||||
/// and return its first allocation extent: (data_lba, data_length).
|
||||
/// The data_lba is partition-relative.
|
||||
fn read_icb_extent(&self, reader: &mut dyn SectorReader, meta_lba: u32) -> Result<(u32, u32)> {
|
||||
fn read_icb_extent(&self, reader: &mut dyn SectorSource, meta_lba: u32) -> Result<(u32, u32)> {
|
||||
let extents = self.read_icb_extents(reader, meta_lba)?;
|
||||
extents.first().copied().ok_or(Error::DiscRead {
|
||||
sector: 0,
|
||||
@@ -290,7 +290,7 @@ impl UdfFs {
|
||||
/// Handles files with many extents (e.g. 88 GB m2ts files have ~90 extents).
|
||||
fn read_icb_extents(
|
||||
&self,
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
meta_lba: u32,
|
||||
) -> Result<Vec<(u32, u32)>> {
|
||||
let mut icb = [0u8; 2048];
|
||||
@@ -367,7 +367,7 @@ impl UdfFs {
|
||||
/// Returns Vec of (absolute_lba, sector_count) covering the entire file.
|
||||
pub fn file_extents(
|
||||
&self,
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
path: &str,
|
||||
) -> Result<Vec<(u32, u32)>> {
|
||||
let parts: Vec<&str> = path.trim_matches('/').split('/').collect();
|
||||
@@ -417,7 +417,7 @@ impl UdfFs {
|
||||
/// 3. Metadata partition file → metadata content location
|
||||
/// 4. FSD → root directory ICB
|
||||
/// 5. Root directory → file tree
|
||||
pub fn read_filesystem(reader: &mut dyn SectorReader) -> Result<UdfFs> {
|
||||
pub fn read_filesystem(reader: &mut dyn SectorSource) -> Result<UdfFs> {
|
||||
// Step 1: Anchor Volume Descriptor Pointer at sector 256
|
||||
// ECMA-167 §10.2 — always at sector 256
|
||||
let mut avdp = [0u8; 2048];
|
||||
@@ -593,7 +593,7 @@ pub fn read_filesystem(reader: &mut dyn SectorReader) -> Result<UdfFs> {
|
||||
/// and points to its ICB.
|
||||
#[allow(clippy::only_used_in_recursion)]
|
||||
fn read_directory(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
part_start: u32,
|
||||
meta_start: u32,
|
||||
meta_lba: u32,
|
||||
@@ -758,7 +758,7 @@ fn read_directory(
|
||||
}
|
||||
|
||||
/// Read file size (info_length) from an Extended File Entry ICB.
|
||||
fn read_file_size(reader: &mut dyn SectorReader, meta_start: u32, meta_lba: u32) -> Result<u64> {
|
||||
fn read_file_size(reader: &mut dyn SectorSource, meta_start: u32, meta_lba: u32) -> Result<u64> {
|
||||
let mut icb = [0u8; 2048];
|
||||
read_sector(reader, meta_start + meta_lba, &mut icb)?;
|
||||
|
||||
@@ -874,7 +874,7 @@ fn parse_dstring(data: &[u8]) -> String {
|
||||
/// Each SCSI command has ~500ms overhead on USB drives, so reading 32 sectors
|
||||
/// at once (one command) is 32x faster than 32 individual reads.
|
||||
pub(crate) struct BufferedSectorReader<'a> {
|
||||
inner: &'a mut dyn SectorReader,
|
||||
inner: &'a mut dyn SectorSource,
|
||||
cache_start: u32,
|
||||
cache: Vec<u8>,
|
||||
cache_sectors: u32,
|
||||
@@ -884,7 +884,7 @@ pub(crate) struct BufferedSectorReader<'a> {
|
||||
}
|
||||
|
||||
impl<'a> BufferedSectorReader<'a> {
|
||||
pub(crate) fn new(inner: &'a mut dyn SectorReader, batch: u16) -> Self {
|
||||
pub(crate) fn new(inner: &'a mut dyn SectorSource, batch: u16) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
cache_start: u32::MAX,
|
||||
@@ -952,7 +952,7 @@ impl BufferedSectorReader<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for BufferedSectorReader<'_> {
|
||||
impl SectorSource for BufferedSectorReader<'_> {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
@@ -996,7 +996,7 @@ impl SectorReader for BufferedSectorReader<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
fn read_sector(reader: &mut dyn SectorReader, lba: u32, buf: &mut [u8]) -> Result<()> {
|
||||
fn read_sector(reader: &mut dyn SectorSource, lba: u32, buf: &mut [u8]) -> Result<()> {
|
||||
reader.read_sectors(lba, 1, buf, true)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+2
-2
@@ -2,7 +2,7 @@
|
||||
|
||||
use crate::disc::{Chapter, DiscTitle};
|
||||
use crate::progress::Progress;
|
||||
use crate::sector::SectorReader;
|
||||
use crate::sector::SectorSource;
|
||||
use std::time::Instant;
|
||||
|
||||
/// Health status of a single sector read.
|
||||
@@ -82,7 +82,7 @@ impl VerifyResult {
|
||||
/// Reads in batches for speed, falls back to single-sector on failure.
|
||||
/// The progress callback returns false to request early stop.
|
||||
pub fn verify_title(
|
||||
reader: &mut dyn SectorReader,
|
||||
reader: &mut dyn SectorSource,
|
||||
title: &DiscTitle,
|
||||
batch_sectors: u16,
|
||||
on_progress: Option<&dyn Progress>,
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
//! Disc scanning pipeline tests.
|
||||
|
||||
use libfreemkv::SectorReader;
|
||||
use libfreemkv::SectorSource;
|
||||
use libfreemkv::error::Result;
|
||||
use libfreemkv::{Disc, DiscTitle, ScanOptions};
|
||||
use std::collections::HashMap;
|
||||
@@ -20,7 +20,7 @@ impl MockSectorReader {
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for MockSectorReader {
|
||||
impl SectorSource for MockSectorReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
|
||||
@@ -1,17 +1,12 @@
|
||||
//! Integration tests for progress reporting, halt behavior, drop safety,
|
||||
//! and the file-backed sector reader round trip.
|
||||
|
||||
// 0.18 trait split: this suite still drives the deprecated `pes::Stream`
|
||||
// trait directly. It will be migrated to `FrameSource`/`FrameSink` in the
|
||||
// follow-up that ports concrete impls.
|
||||
#![allow(deprecated)]
|
||||
|
||||
use libfreemkv::disc::{CopyOptions, DiscRegion};
|
||||
use libfreemkv::error::Result;
|
||||
use libfreemkv::pes::Stream as PesStream;
|
||||
use libfreemkv::{
|
||||
ContentFormat, Disc, DiscFormat, DiscStream, DiscTitle, EventKind, Extent, FileSectorReader,
|
||||
SectorReader,
|
||||
ContentFormat, Disc, DiscFormat, DiscStream, DiscTitle, EventKind, Extent, FileSectorSource,
|
||||
SectorSource,
|
||||
};
|
||||
use std::io::Write;
|
||||
use std::sync::Arc;
|
||||
@@ -37,7 +32,7 @@ impl ZeroSectorReader {
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for ZeroSectorReader {
|
||||
impl SectorSource for ZeroSectorReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
_lba: u32,
|
||||
@@ -51,7 +46,7 @@ impl SectorReader for ZeroSectorReader {
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn capacity(&self) -> u32 {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
@@ -72,7 +67,7 @@ impl SlowZeroSectorReader {
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for SlowZeroSectorReader {
|
||||
impl SectorSource for SlowZeroSectorReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
_lba: u32,
|
||||
@@ -86,7 +81,7 @@ impl SectorReader for SlowZeroSectorReader {
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn capacity(&self) -> u32 {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
@@ -309,7 +304,7 @@ fn test_drop_impls_do_not_panic_or_block() {
|
||||
panic!("DiscStream drop did not complete within 100ms");
|
||||
}
|
||||
|
||||
// ── 5. FileSectorReader round trip ────────────────────────────────────────
|
||||
// ── 5. FileSectorSource round trip ────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_file_sector_reader_round_trip() {
|
||||
@@ -326,9 +321,13 @@ fn test_file_sector_reader_round_trip() {
|
||||
tmp.flush().expect("flush");
|
||||
|
||||
let path = tmp.path().to_path_buf();
|
||||
let mut fsr = FileSectorReader::open(&path).expect("open FileSectorReader");
|
||||
let mut fsr = FileSectorSource::open(&path).expect("open FileSectorSource");
|
||||
|
||||
assert_eq!(fsr.capacity(), N_SECTORS as u32, "capacity mismatch");
|
||||
assert_eq!(
|
||||
fsr.capacity_sectors(),
|
||||
N_SECTORS as u32,
|
||||
"capacity mismatch"
|
||||
);
|
||||
|
||||
// Read each sector individually and compare.
|
||||
let mut buf = vec![0u8; SECTOR_SIZE];
|
||||
@@ -396,7 +395,7 @@ impl FailingSectorReader {
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for FailingSectorReader {
|
||||
impl SectorSource for FailingSectorReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
_lba: u32,
|
||||
@@ -423,7 +422,7 @@ impl SectorReader for FailingSectorReader {
|
||||
})
|
||||
}
|
||||
|
||||
fn capacity(&self) -> u32 {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
@@ -572,7 +571,7 @@ struct BlockSizeFailingReader {
|
||||
capacity: u32,
|
||||
}
|
||||
|
||||
impl SectorReader for BlockSizeFailingReader {
|
||||
impl SectorSource for BlockSizeFailingReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
@@ -598,7 +597,7 @@ impl SectorReader for BlockSizeFailingReader {
|
||||
}
|
||||
}
|
||||
|
||||
fn capacity(&self) -> u32 {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ use libfreemkv::disc::CopyOptions;
|
||||
use libfreemkv::disc::DiscRegion;
|
||||
use libfreemkv::disc::mapfile::{Mapfile, SectorStatus};
|
||||
use libfreemkv::error::Result;
|
||||
use libfreemkv::{ContentFormat, Disc, DiscFormat, SectorReader};
|
||||
use libfreemkv::{ContentFormat, Disc, DiscFormat, SectorSource};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
@@ -46,7 +46,7 @@ impl PatternedSectorReader {
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for PatternedSectorReader {
|
||||
impl SectorSource for PatternedSectorReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
@@ -82,7 +82,7 @@ impl SectorReader for PatternedSectorReader {
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn capacity(&self) -> u32 {
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,807 @@
|
||||
//! Pass-N (`Disc::patch`) read-error handler — A/B golden fixture.
|
||||
//!
|
||||
//! Background (2026-05-13, v0.20.8 release bundle planning):
|
||||
//!
|
||||
//! `libfreemkv::disc::read_error::handle_read_error` is supposed to be
|
||||
//! the single source of truth for sector-read error → recovery action
|
||||
//! decisions. Pass 1 sweep routes through it. Pass N patch's
|
||||
//! `handle_read_failure` (in `disc/patch.rs`) does NOT — historically
|
||||
//! MEDIUM_ERROR / NOT_READY get inline handling with their own thresholds
|
||||
//! (`PASSN_DAMAGE_THRESHOLD_PCT=6` vs the sweep's `12`), their own
|
||||
//! damage_window (state.damage_window, separate from ReadCtx.damage_window),
|
||||
//! and their own skip logic (`compute_damage_skip`, which runs AFTER
|
||||
//! the failure handler and has a size-aware `range_remaining/4` cap
|
||||
//! that `handle_read_error::JumpAhead` does not know about).
|
||||
//!
|
||||
//! This file is the A/B fixture for that unification. It pins the
|
||||
//! CURRENT (pre-unification) end-to-end behavior of `Disc::patch` for
|
||||
//! eight canonical damage profiles against a synthetic
|
||||
//! `ScriptedSectorReader`. Each profile asserts the exact observable
|
||||
//! outcome — final mapfile byte counts and outer-loop counters — so any
|
||||
//! attempt to refactor the failure path either preserves the goldens or
|
||||
//! the test fails loudly.
|
||||
//!
|
||||
//! The prompt called for "exact sequence of `ReadAction` enums per
|
||||
//! LBA"; that framing doesn't fit the current architecture because
|
||||
//! `handle_read_failure` produces `FailureAction`, not `ReadAction`,
|
||||
//! and interleaves with `compute_damage_skip` + cursor management in
|
||||
//! the outer loop. The observable contract — what `Disc::patch` does
|
||||
//! to the mapfile and how many reads it performs — is the equivalent
|
||||
//! invariant, captured end-to-end.
|
||||
//!
|
||||
//! Why we expect divergence under naïve unification (see final report
|
||||
//! of the 0.20.8 unification attempt): the patch loop's skip semantics
|
||||
//! live in `compute_damage_skip` POST-failure-handler, with a size-aware
|
||||
//! cap that `handle_read_error` knows nothing about; routing through
|
||||
//! `handle_read_error` would invert that cursor flow. The fixture stays
|
||||
//! checked in regardless — it documents the contract for the next
|
||||
//! refactor attempt.
|
||||
|
||||
use libfreemkv::ContentFormat;
|
||||
use libfreemkv::Disc;
|
||||
use libfreemkv::DiscFormat;
|
||||
use libfreemkv::disc::CopyOptions;
|
||||
use libfreemkv::disc::DiscRegion;
|
||||
use libfreemkv::disc::mapfile::{Mapfile, SectorStatus};
|
||||
use libfreemkv::error::{Error, Result};
|
||||
use libfreemkv::scsi;
|
||||
use libfreemkv::{ScsiSense, SectorSource};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
const SECTOR_SIZE: usize = 2048;
|
||||
|
||||
/// Per-attempt result the script can emit. `Ok` returns a deterministic
|
||||
/// per-sector byte pattern (LBA mod 256 in each sector). `Err` returns
|
||||
/// the SCSI sense triple supplied — the patch failure path inspects
|
||||
/// `scsi_sense().sense_key` to classify (MEDIUM, NOT_READY,
|
||||
/// HARDWARE, ILLEGAL_REQUEST, ABORTED_COMMAND).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum ScriptStep {
|
||||
Ok,
|
||||
Err { sense_key: u8, asc: u8, ascq: u8 },
|
||||
}
|
||||
|
||||
/// A scripted reader. For each (lba, count) read attempt, picks the
|
||||
/// step at `attempt_idx[lba]`, advances the index. If no script entry
|
||||
/// exists for an LBA, defaults to `Ok` so we don't need to script
|
||||
/// every sector of large ranges.
|
||||
///
|
||||
/// "Batch fails if ANY sector in the batch is bad" — matches real
|
||||
/// drive behavior (`pass_n_size_aware_skip.rs` uses the same model).
|
||||
/// For batched reads we synthesize an Err with the FIRST scripted
|
||||
/// failure in the batch.
|
||||
struct ScriptedSectorReader {
|
||||
capacity: u32,
|
||||
/// Per-LBA script of (step, then next step on retry, …). When
|
||||
/// retries exhaust the script, the LAST step repeats forever.
|
||||
script: std::collections::HashMap<u32, Vec<ScriptStep>>,
|
||||
/// Per-LBA index into its script vec. Bumps on each read attempt
|
||||
/// at that LBA.
|
||||
attempt_idx: Mutex<std::collections::HashMap<u32, usize>>,
|
||||
/// Full read trace: every (lba, count, result_was_ok) tuple in
|
||||
/// call order. Lets the test assert that adaptive-batch dropped
|
||||
/// to count=1, bisection happened, etc.
|
||||
trace: Arc<Mutex<Vec<(u32, u16, bool)>>>,
|
||||
}
|
||||
|
||||
impl ScriptedSectorReader {
|
||||
fn new(capacity: u32) -> (Self, Arc<Mutex<Vec<(u32, u16, bool)>>>) {
|
||||
let trace = Arc::new(Mutex::new(Vec::new()));
|
||||
(
|
||||
Self {
|
||||
capacity,
|
||||
script: std::collections::HashMap::new(),
|
||||
attempt_idx: Mutex::new(std::collections::HashMap::new()),
|
||||
trace: trace.clone(),
|
||||
},
|
||||
trace,
|
||||
)
|
||||
}
|
||||
|
||||
/// Set a single-step script for `lba`: every attempt yields `step`.
|
||||
fn always(&mut self, lba: u32, step: ScriptStep) {
|
||||
self.script.insert(lba, vec![step]);
|
||||
}
|
||||
|
||||
/// Set a multi-step script for `lba`: first attempt yields
|
||||
/// `steps[0]`, second `steps[1]`, … on retry the last step repeats.
|
||||
#[allow(dead_code)]
|
||||
fn sequence(&mut self, lba: u32, steps: Vec<ScriptStep>) {
|
||||
self.script.insert(lba, steps);
|
||||
}
|
||||
|
||||
fn step_for(&self, lba: u32) -> ScriptStep {
|
||||
let v = match self.script.get(&lba) {
|
||||
Some(v) => v,
|
||||
None => return ScriptStep::Ok,
|
||||
};
|
||||
let mut idx = self.attempt_idx.lock().unwrap();
|
||||
let i = idx.entry(lba).or_insert(0);
|
||||
let step = v[(*i).min(v.len() - 1)];
|
||||
*i += 1;
|
||||
step
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorSource for ScriptedSectorReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
count: u16,
|
||||
buf: &mut [u8],
|
||||
_recovery: bool,
|
||||
) -> Result<usize> {
|
||||
// Look at every sector in the batch — first failure determines
|
||||
// the outcome.
|
||||
let mut failure: Option<(u8, u8, u8)> = None;
|
||||
for offset in 0..count as u32 {
|
||||
match self.step_for(lba + offset) {
|
||||
ScriptStep::Ok => {}
|
||||
ScriptStep::Err {
|
||||
sense_key,
|
||||
asc,
|
||||
ascq,
|
||||
} => {
|
||||
failure = Some((sense_key, asc, ascq));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let ok = failure.is_none();
|
||||
self.trace.lock().unwrap().push((lba, count, ok));
|
||||
if let Some((sense_key, asc, ascq)) = failure {
|
||||
return Err(Error::ScsiError {
|
||||
opcode: scsi::SCSI_READ_10,
|
||||
status: scsi::SCSI_STATUS_CHECK_CONDITION,
|
||||
sense: Some(ScsiSense {
|
||||
sense_key,
|
||||
asc,
|
||||
ascq,
|
||||
}),
|
||||
});
|
||||
}
|
||||
// Per-sector LBA byte pattern.
|
||||
for (i, chunk) in buf.chunks_mut(SECTOR_SIZE).enumerate() {
|
||||
chunk.fill(((lba + i as u32) & 0xff) as u8);
|
||||
}
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn capacity_sectors(&self) -> u32 {
|
||||
self.capacity
|
||||
}
|
||||
}
|
||||
|
||||
fn synthetic_disc(capacity_sectors: u32) -> Disc {
|
||||
Disc {
|
||||
volume_id: String::new(),
|
||||
meta_title: None,
|
||||
format: DiscFormat::BluRay,
|
||||
capacity_sectors,
|
||||
capacity_bytes: capacity_sectors as u64 * SECTOR_SIZE as u64,
|
||||
layers: 1,
|
||||
titles: Vec::new(),
|
||||
region: DiscRegion::Free,
|
||||
aacs: None,
|
||||
css: None,
|
||||
encrypted: false,
|
||||
aacs_error: None,
|
||||
content_format: ContentFormat::BdTs,
|
||||
}
|
||||
}
|
||||
|
||||
fn prep_iso_and_mapfile(
|
||||
iso_path: &std::path::Path,
|
||||
total_bytes: u64,
|
||||
finished_ranges: &[(u64, u64)],
|
||||
nontrimmed_ranges: &[(u64, u64)],
|
||||
) {
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
let mut f = OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(true)
|
||||
.open(iso_path)
|
||||
.unwrap();
|
||||
f.set_len(total_bytes).unwrap();
|
||||
f.seek(SeekFrom::Start(0)).unwrap();
|
||||
f.write_all(&[]).unwrap();
|
||||
|
||||
let map_path = libfreemkv::disc::mapfile_path_for(iso_path);
|
||||
let mut mf = Mapfile::create(&map_path, total_bytes, "test").unwrap();
|
||||
for &(pos, size) in finished_ranges {
|
||||
mf.record(pos, size, SectorStatus::Finished).unwrap();
|
||||
}
|
||||
for &(pos, size) in nontrimmed_ranges {
|
||||
mf.record(pos, size, SectorStatus::NonTrimmed).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
/// Observable outcome of a patch run. Goldens for each profile pin
|
||||
/// these exact values.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
struct Golden {
|
||||
/// `bytes_good` at end of patch.
|
||||
bytes_good: u64,
|
||||
/// `bytes_unreadable` at end.
|
||||
bytes_unreadable: u64,
|
||||
/// `bytes_pending` (NonTrimmed) at end.
|
||||
bytes_pending: u64,
|
||||
/// Did the pass exit via wedge-detection?
|
||||
wedged_exit: bool,
|
||||
/// Sanity bound on trace length — patch makes a finite number of
|
||||
/// reads bounded by `MAX_SKIPS_PER_RANGE * range_sectors` plus
|
||||
/// retries. Asserted as an UPPER bound only (so any reduction in
|
||||
/// retries via future tuning doesn't fail the test spuriously).
|
||||
max_reads: usize,
|
||||
}
|
||||
|
||||
/// Common helper: prep ISO + mapfile, run `disc.copy(multipass)`,
|
||||
/// return (PatchOutcome ↔ CopyResult, final-map stats, trace length).
|
||||
fn run_profile(
|
||||
profile_name: &str,
|
||||
capacity_sectors: u32,
|
||||
nontrimmed: &[(u64, u64)],
|
||||
finished: &[(u64, u64)],
|
||||
scripted: ScriptedSectorReader,
|
||||
trace: Arc<Mutex<Vec<(u32, u16, bool)>>>,
|
||||
) -> (
|
||||
libfreemkv::disc::CopyResult,
|
||||
libfreemkv::disc::mapfile::MapStats,
|
||||
usize,
|
||||
) {
|
||||
let total_bytes: u64 = capacity_sectors as u64 * SECTOR_SIZE as u64;
|
||||
let disc = synthetic_disc(capacity_sectors);
|
||||
|
||||
let tmp = tempfile::NamedTempFile::new().unwrap();
|
||||
let iso_path = tmp.path().to_path_buf();
|
||||
drop(tmp);
|
||||
|
||||
prep_iso_and_mapfile(&iso_path, total_bytes, finished, nontrimmed);
|
||||
|
||||
let opts = CopyOptions {
|
||||
decrypt: false,
|
||||
multipass: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut reader = scripted;
|
||||
let pr = disc
|
||||
.copy(&mut reader, &iso_path, &opts)
|
||||
.unwrap_or_else(|e| panic!("[{profile_name}] disc.copy returned Err: {e:?}"));
|
||||
|
||||
let map_path = libfreemkv::disc::mapfile_path_for(&iso_path);
|
||||
let map = Mapfile::load(&map_path).unwrap();
|
||||
let stats = map.stats();
|
||||
|
||||
let trace_len = trace.lock().unwrap().len();
|
||||
|
||||
let _ = std::fs::remove_file(&iso_path);
|
||||
let _ = std::fs::remove_file(&map_path);
|
||||
|
||||
(pr, stats, trace_len)
|
||||
}
|
||||
|
||||
// ─────────────────────────── Profile 1: CLEAN ────────────────────────────
|
||||
//
|
||||
// The NonTrimmed range has zero scripted failures — every read succeeds.
|
||||
// Patch should march through the range and mark it Finished. Validates
|
||||
// the happy-path side of the failure-handler dispatch (it shouldn't
|
||||
// fire at all).
|
||||
|
||||
#[test]
|
||||
fn profile_01_clean_all_recoverable() {
|
||||
let capacity_sectors: u32 = 256;
|
||||
let (reader, trace) = ScriptedSectorReader::new(capacity_sectors);
|
||||
// No scripted errors → all reads succeed.
|
||||
|
||||
let nontrimmed = [(100 * 2048, 16 * 2048)]; // 16-sector NonTrimmed range
|
||||
let finished = [
|
||||
(0, 100 * 2048),
|
||||
(116 * 2048, (capacity_sectors as u64 - 116) * 2048),
|
||||
];
|
||||
|
||||
let (pr, stats, trace_len) = run_profile(
|
||||
"01_clean",
|
||||
capacity_sectors,
|
||||
&nontrimmed,
|
||||
&finished,
|
||||
reader,
|
||||
trace,
|
||||
);
|
||||
|
||||
let expected = Golden {
|
||||
bytes_good: capacity_sectors as u64 * 2048,
|
||||
bytes_unreadable: 0,
|
||||
bytes_pending: 0,
|
||||
wedged_exit: false,
|
||||
max_reads: 8, // adaptive batch=32 reads finishes 16 sectors in 1 read; allow up to 8.
|
||||
};
|
||||
assert_eq!(stats.bytes_good, expected.bytes_good, "01_clean bytes_good");
|
||||
assert_eq!(
|
||||
stats.bytes_unreadable, expected.bytes_unreadable,
|
||||
"01_clean bytes_unreadable"
|
||||
);
|
||||
assert_eq!(
|
||||
stats.bytes_pending, expected.bytes_pending,
|
||||
"01_clean bytes_pending"
|
||||
);
|
||||
assert!(!pr.halted, "01_clean halted");
|
||||
assert!(
|
||||
trace_len <= expected.max_reads,
|
||||
"01_clean trace_len={trace_len} exceeds bound {}",
|
||||
expected.max_reads
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────── Profile 2: ALL MEDIUM ───────────────────────
|
||||
//
|
||||
// Every LBA in the NonTrimmed range returns MEDIUM_ERROR every attempt.
|
||||
// Adaptive-batch drops to count=1 on first batch failure, then each
|
||||
// single-sector read fails → consecutive_failures climbs, damage_window
|
||||
// fills, compute_damage_skip fires, MAX_SKIPS_PER_RANGE caps the work,
|
||||
// remaining bytes stay NonTrimmed (NEVER marked Unreadable inside a
|
||||
// single pass — 2026-05-11 design call).
|
||||
|
||||
#[test]
|
||||
fn profile_02_all_medium_error() {
|
||||
let capacity_sectors: u32 = 256;
|
||||
let (mut reader, trace) = ScriptedSectorReader::new(capacity_sectors);
|
||||
for lba in 100..116 {
|
||||
reader.always(
|
||||
lba,
|
||||
ScriptStep::Err {
|
||||
sense_key: scsi::SENSE_KEY_MEDIUM_ERROR,
|
||||
asc: 0x11,
|
||||
ascq: 0x00,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let nontrimmed = [(100 * 2048, 16 * 2048)];
|
||||
let finished = [
|
||||
(0, 100 * 2048),
|
||||
(116 * 2048, (capacity_sectors as u64 - 116) * 2048),
|
||||
];
|
||||
|
||||
let (pr, stats, trace_len) = run_profile(
|
||||
"02_all_medium",
|
||||
capacity_sectors,
|
||||
&nontrimmed,
|
||||
&finished,
|
||||
reader,
|
||||
trace,
|
||||
);
|
||||
|
||||
// GOLDEN: the 16-sector bad range stays NonTrimmed (bytes_pending).
|
||||
// Pre-2026-05-11 patch would mark Unreadable here; current code
|
||||
// preserves NonTrimmed so subsequent passes get another shot.
|
||||
assert_eq!(
|
||||
stats.bytes_good,
|
||||
(capacity_sectors as u64 - 16) * 2048,
|
||||
"02_all_medium bytes_good"
|
||||
);
|
||||
assert_eq!(
|
||||
stats.bytes_unreadable, 0,
|
||||
"02_all_medium bytes_unreadable (must NOT be marked terminal in one pass)"
|
||||
);
|
||||
assert_eq!(
|
||||
stats.bytes_pending,
|
||||
16 * 2048,
|
||||
"02_all_medium bytes_pending (NonTrimmed retained across passes)"
|
||||
);
|
||||
assert!(!pr.halted, "02_all_medium halted");
|
||||
// Upper bound: every sector probed individually + a few batch-drop
|
||||
// and skip-escalation attempts. 16 sectors × ~3 visits ≈ 50.
|
||||
assert!(
|
||||
trace_len <= 80,
|
||||
"02_all_medium trace_len={trace_len} exceeds 80"
|
||||
);
|
||||
}
|
||||
|
||||
// ───────────────────── Profile 3: ALTERNATING GOOD/BAD ───────────────────
|
||||
//
|
||||
// LBAs 100, 102, 104, ... bad; odd LBAs good. Validates that good
|
||||
// sectors interleaved with bad get recovered individually after the
|
||||
// adaptive split (batch-fail → count=1 → per-sector probe).
|
||||
|
||||
#[test]
|
||||
fn profile_03_alternating_good_bad() {
|
||||
let capacity_sectors: u32 = 256;
|
||||
let (mut reader, trace) = ScriptedSectorReader::new(capacity_sectors);
|
||||
for lba in (100..116).step_by(2) {
|
||||
reader.always(
|
||||
lba,
|
||||
ScriptStep::Err {
|
||||
sense_key: scsi::SENSE_KEY_MEDIUM_ERROR,
|
||||
asc: 0x11,
|
||||
ascq: 0x00,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let nontrimmed = [(100 * 2048, 16 * 2048)];
|
||||
let finished = [
|
||||
(0, 100 * 2048),
|
||||
(116 * 2048, (capacity_sectors as u64 - 116) * 2048),
|
||||
];
|
||||
|
||||
let (pr, stats, trace_len) = run_profile(
|
||||
"03_alternating",
|
||||
capacity_sectors,
|
||||
&nontrimmed,
|
||||
&finished,
|
||||
reader,
|
||||
trace,
|
||||
);
|
||||
|
||||
// GOLDEN: 8 good sectors interleaved should mostly be Finished;
|
||||
// 8 bad stay NonTrimmed. Allow 2 sectors of slop for the actual
|
||||
// bisect cursor advance — converging on alternating bad/good in
|
||||
// a single pass isn't always exact at boundaries with the
|
||||
// size-aware skip cap.
|
||||
let good_total = stats.bytes_good;
|
||||
let baseline_good = (capacity_sectors as u64 - 16) * 2048;
|
||||
let middle_recovered = good_total - baseline_good;
|
||||
assert!(
|
||||
middle_recovered >= 6 * 2048,
|
||||
"03_alternating recovered only {middle_recovered} bytes of 8 good sectors"
|
||||
);
|
||||
assert!(
|
||||
middle_recovered <= 9 * 2048,
|
||||
"03_alternating recovered MORE than scripted good sectors: {middle_recovered}"
|
||||
);
|
||||
assert_eq!(stats.bytes_unreadable, 0, "03_alternating bytes_unreadable");
|
||||
// Remaining must be NonTrimmed (pending), not lost.
|
||||
assert!(
|
||||
stats.bytes_pending > 0,
|
||||
"03_alternating expected NonTrimmed remainder, got bytes_pending=0"
|
||||
);
|
||||
assert!(!pr.halted, "03_alternating halted");
|
||||
assert!(
|
||||
trace_len <= 120,
|
||||
"03_alternating trace_len={trace_len} exceeds 120"
|
||||
);
|
||||
}
|
||||
|
||||
// ───────────────────── Profile 4: EDGE-BAD (size-aware-skip canon) ───────
|
||||
//
|
||||
// Bad at start (100..104), good middle (104..112), bad at end (112..116).
|
||||
// This is the size-aware-skip canonical case. The middle good sectors
|
||||
// MUST be recovered — pre-fix patch would skip-escalate across the
|
||||
// whole range and miss them.
|
||||
|
||||
#[test]
|
||||
fn profile_04_edge_bad_good_middle() {
|
||||
let capacity_sectors: u32 = 256;
|
||||
let (mut reader, trace) = ScriptedSectorReader::new(capacity_sectors);
|
||||
for lba in 100..104 {
|
||||
reader.always(
|
||||
lba,
|
||||
ScriptStep::Err {
|
||||
sense_key: scsi::SENSE_KEY_MEDIUM_ERROR,
|
||||
asc: 0x11,
|
||||
ascq: 0x00,
|
||||
},
|
||||
);
|
||||
}
|
||||
for lba in 112..116 {
|
||||
reader.always(
|
||||
lba,
|
||||
ScriptStep::Err {
|
||||
sense_key: scsi::SENSE_KEY_MEDIUM_ERROR,
|
||||
asc: 0x11,
|
||||
ascq: 0x00,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let nontrimmed = [(100 * 2048, 16 * 2048)];
|
||||
let finished = [
|
||||
(0, 100 * 2048),
|
||||
(116 * 2048, (capacity_sectors as u64 - 116) * 2048),
|
||||
];
|
||||
|
||||
let (pr, stats, trace_len) = run_profile(
|
||||
"04_edge_bad",
|
||||
capacity_sectors,
|
||||
&nontrimmed,
|
||||
&finished,
|
||||
reader,
|
||||
trace,
|
||||
);
|
||||
|
||||
// GOLDEN: the 8 good middle sectors should land Finished (allowing
|
||||
// 2 sectors of bisection slop at boundaries).
|
||||
let middle_recovered = stats.bytes_good - (capacity_sectors as u64 - 16) * 2048;
|
||||
assert!(
|
||||
middle_recovered >= 6 * 2048,
|
||||
"04_edge_bad recovered only {middle_recovered} bytes of 8 good middle sectors"
|
||||
);
|
||||
assert_eq!(stats.bytes_unreadable, 0, "04_edge_bad bytes_unreadable");
|
||||
assert!(
|
||||
stats.bytes_pending > 0,
|
||||
"04_edge_bad bytes_pending expected > 0"
|
||||
);
|
||||
assert!(!pr.halted, "04_edge_bad halted");
|
||||
assert!(
|
||||
trace_len <= 120,
|
||||
"04_edge_bad trace_len={trace_len} exceeds 120"
|
||||
);
|
||||
}
|
||||
|
||||
// ───────────────────── Profile 5: SINGLE BAD SECTOR ──────────────────────
|
||||
//
|
||||
// 1 bad sector in the middle of an otherwise good 16-sector NonTrimmed
|
||||
// range. Validates the common "stochastic miss in Pass 1, easily picked
|
||||
// up in Pass N" scenario.
|
||||
|
||||
#[test]
|
||||
fn profile_05_single_bad_sector() {
|
||||
let capacity_sectors: u32 = 256;
|
||||
let (mut reader, trace) = ScriptedSectorReader::new(capacity_sectors);
|
||||
reader.always(
|
||||
108,
|
||||
ScriptStep::Err {
|
||||
sense_key: scsi::SENSE_KEY_MEDIUM_ERROR,
|
||||
asc: 0x11,
|
||||
ascq: 0x00,
|
||||
},
|
||||
);
|
||||
|
||||
let nontrimmed = [(100 * 2048, 16 * 2048)];
|
||||
let finished = [
|
||||
(0, 100 * 2048),
|
||||
(116 * 2048, (capacity_sectors as u64 - 116) * 2048),
|
||||
];
|
||||
|
||||
let (pr, stats, trace_len) = run_profile(
|
||||
"05_single_bad",
|
||||
capacity_sectors,
|
||||
&nontrimmed,
|
||||
&finished,
|
||||
reader,
|
||||
trace,
|
||||
);
|
||||
|
||||
// GOLDEN: 15 of 16 sectors recovered. 1 sector stays NonTrimmed
|
||||
// (NOT Unreadable — same multi-pass tolerance principle).
|
||||
assert_eq!(
|
||||
stats.bytes_good,
|
||||
(capacity_sectors as u64 - 1) * 2048,
|
||||
"05_single_bad bytes_good"
|
||||
);
|
||||
assert_eq!(stats.bytes_unreadable, 0, "05_single_bad bytes_unreadable");
|
||||
assert_eq!(stats.bytes_pending, 2048, "05_single_bad bytes_pending");
|
||||
assert!(!pr.halted, "05_single_bad halted");
|
||||
assert!(
|
||||
trace_len <= 80,
|
||||
"05_single_bad trace_len={trace_len} exceeds 80"
|
||||
);
|
||||
}
|
||||
|
||||
// ───────────────────── Profile 6: DEEP PIT ───────────────────────────────
|
||||
//
|
||||
// A contiguous 8-sector bad pit in the middle of a wider 24-sector
|
||||
// NonTrimmed range. Tests the damage-window threshold + size-aware-skip
|
||||
// converging on the actual pit boundaries instead of bailing on
|
||||
// MAX_SKIPS_PER_RANGE.
|
||||
|
||||
#[test]
|
||||
fn profile_06_deep_pit() {
|
||||
let capacity_sectors: u32 = 256;
|
||||
let (mut reader, trace) = ScriptedSectorReader::new(capacity_sectors);
|
||||
for lba in 108..116 {
|
||||
reader.always(
|
||||
lba,
|
||||
ScriptStep::Err {
|
||||
sense_key: scsi::SENSE_KEY_MEDIUM_ERROR,
|
||||
asc: 0x11,
|
||||
ascq: 0x00,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// 24 sectors NonTrimmed: 100..108 good, 108..116 BAD, 116..124 good.
|
||||
let nontrimmed = [(100 * 2048, 24 * 2048)];
|
||||
let finished = [
|
||||
(0, 100 * 2048),
|
||||
(124 * 2048, (capacity_sectors as u64 - 124) * 2048),
|
||||
];
|
||||
|
||||
let (pr, stats, trace_len) = run_profile(
|
||||
"06_deep_pit",
|
||||
capacity_sectors,
|
||||
&nontrimmed,
|
||||
&finished,
|
||||
reader,
|
||||
trace,
|
||||
);
|
||||
|
||||
// GOLDEN: 16 good (8 on each side of the pit) recovered, 8 bad
|
||||
// stay NonTrimmed.
|
||||
let recovered_in_range = stats.bytes_good - (capacity_sectors as u64 - 24) * 2048;
|
||||
assert!(
|
||||
recovered_in_range >= 14 * 2048,
|
||||
"06_deep_pit recovered only {recovered_in_range} bytes of 16 good sectors"
|
||||
);
|
||||
assert_eq!(stats.bytes_unreadable, 0, "06_deep_pit bytes_unreadable");
|
||||
assert!(
|
||||
stats.bytes_pending > 0,
|
||||
"06_deep_pit bytes_pending expected > 0"
|
||||
);
|
||||
assert!(!pr.halted, "06_deep_pit halted");
|
||||
assert!(
|
||||
trace_len <= 120,
|
||||
"06_deep_pit trace_len={trace_len} exceeds 120"
|
||||
);
|
||||
}
|
||||
|
||||
// ───────────────────── Profile 7: MEDIUM-THEN-GOOD ───────────────────────
|
||||
//
|
||||
// First N attempts at each bad LBA fail with MEDIUM_ERROR, then succeed.
|
||||
// Tests whether patch's retry semantics revisit failed sectors. Current
|
||||
// patch dispatches NonTrimmed on first failure and ADVANCES the cursor
|
||||
// — it does NOT retry the same LBA inside one pass for MEDIUM_ERROR
|
||||
// (only NOT_READY retries in-place). So the goldens here are: bad
|
||||
// sectors stay NonTrimmed in this pass (the recovery would happen in a
|
||||
// subsequent pass, which this single-pass fixture does not run).
|
||||
|
||||
#[test]
|
||||
fn profile_07_medium_then_good() {
|
||||
let capacity_sectors: u32 = 256;
|
||||
let (mut reader, trace) = ScriptedSectorReader::new(capacity_sectors);
|
||||
// Sectors 105..110: fail twice, then succeed.
|
||||
for lba in 105..110 {
|
||||
reader.sequence(
|
||||
lba,
|
||||
vec![
|
||||
ScriptStep::Err {
|
||||
sense_key: scsi::SENSE_KEY_MEDIUM_ERROR,
|
||||
asc: 0x11,
|
||||
ascq: 0x00,
|
||||
},
|
||||
ScriptStep::Err {
|
||||
sense_key: scsi::SENSE_KEY_MEDIUM_ERROR,
|
||||
asc: 0x11,
|
||||
ascq: 0x00,
|
||||
},
|
||||
ScriptStep::Ok,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
let nontrimmed = [(100 * 2048, 16 * 2048)];
|
||||
let finished = [
|
||||
(0, 100 * 2048),
|
||||
(116 * 2048, (capacity_sectors as u64 - 116) * 2048),
|
||||
];
|
||||
|
||||
let (pr, stats, trace_len) = run_profile(
|
||||
"07_medium_then_good",
|
||||
capacity_sectors,
|
||||
&nontrimmed,
|
||||
&finished,
|
||||
reader,
|
||||
trace,
|
||||
);
|
||||
|
||||
// GOLDEN: patch's cache-priming (`prime_cache`) issues 3
|
||||
// throwaway single-sector reads at lba-3..lba before each count==1
|
||||
// recovery read. Those throwaway reads ADVANCE the per-LBA script
|
||||
// step counter even though their results are discarded. So a
|
||||
// 3-step script (fail, fail, ok) gets consumed by 2 prime calls
|
||||
// plus 1 real read → the real read sees `Ok` and the sector is
|
||||
// recovered. Net effect: patch fully recovers the range in one
|
||||
// pass thanks to priming, even though the script said "fails on
|
||||
// first two attempts."
|
||||
//
|
||||
// This is the documented cache-prime behavior (`disc/patch.rs`
|
||||
// ~line 398, "Proven 2026-05-07 with dd-as-oracle: 8/8 sectors
|
||||
// recoverable when primed vs 6/8 cold"). The golden pins it.
|
||||
assert_eq!(
|
||||
stats.bytes_good,
|
||||
capacity_sectors as u64 * 2048,
|
||||
"07_medium_then_good bytes_good — cache-prime should consume \
|
||||
the failing script steps so the real read sees Ok"
|
||||
);
|
||||
assert_eq!(
|
||||
stats.bytes_unreadable, 0,
|
||||
"07_medium_then_good bytes_unreadable"
|
||||
);
|
||||
assert_eq!(stats.bytes_pending, 0, "07_medium_then_good bytes_pending");
|
||||
assert!(!pr.halted, "07_medium_then_good halted");
|
||||
assert!(
|
||||
trace_len <= 100,
|
||||
"07_medium_then_good trace_len={trace_len} exceeds 100"
|
||||
);
|
||||
}
|
||||
|
||||
// ───────────────────── Profile 8: BATCHED-FAIL ONLY ──────────────────────
|
||||
//
|
||||
// LBA 108 fails on BATCH reads (any batch including it) but succeeds
|
||||
// individually. Models a marginal sector that the drive can ECC-recover
|
||||
// when read alone but not at multi-sector throughput. Validates that
|
||||
// adaptive batch's drop-to-count=1 retries the same starting position
|
||||
// and rescues the data.
|
||||
//
|
||||
// Implementation note: the scripted reader marks the entire batch failed
|
||||
// on any failed sector. We can't easily differentiate "single vs batch"
|
||||
// without bigger plumbing — so this profile uses a script that fails
|
||||
// once then succeeds on retry at the same LBA, simulating "drive
|
||||
// recovered after retry."
|
||||
|
||||
#[test]
|
||||
fn profile_08_batch_fail_singles_ok() {
|
||||
let capacity_sectors: u32 = 256;
|
||||
let (mut reader, trace) = ScriptedSectorReader::new(capacity_sectors);
|
||||
// Sector 108: fail on first call (which is the batch read), succeed
|
||||
// on second call (the drop-to-count=1 retry at the same position).
|
||||
reader.sequence(
|
||||
108,
|
||||
vec![
|
||||
ScriptStep::Err {
|
||||
sense_key: scsi::SENSE_KEY_MEDIUM_ERROR,
|
||||
asc: 0x11,
|
||||
ascq: 0x00,
|
||||
},
|
||||
ScriptStep::Ok,
|
||||
],
|
||||
);
|
||||
|
||||
let nontrimmed = [(100 * 2048, 16 * 2048)];
|
||||
let finished = [
|
||||
(0, 100 * 2048),
|
||||
(116 * 2048, (capacity_sectors as u64 - 116) * 2048),
|
||||
];
|
||||
|
||||
let (pr, stats, trace_len) = run_profile(
|
||||
"08_batch_fail",
|
||||
capacity_sectors,
|
||||
&nontrimmed,
|
||||
&finished,
|
||||
reader,
|
||||
trace,
|
||||
);
|
||||
|
||||
// GOLDEN: the second attempt succeeds → all 16 sectors recovered.
|
||||
assert_eq!(
|
||||
stats.bytes_good,
|
||||
capacity_sectors as u64 * 2048,
|
||||
"08_batch_fail bytes_good — second attempt should recover"
|
||||
);
|
||||
assert_eq!(stats.bytes_unreadable, 0, "08_batch_fail bytes_unreadable");
|
||||
assert_eq!(stats.bytes_pending, 0, "08_batch_fail bytes_pending");
|
||||
assert!(!pr.halted, "08_batch_fail halted");
|
||||
assert!(
|
||||
trace_len <= 80,
|
||||
"08_batch_fail trace_len={trace_len} exceeds 80"
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// Suppressed for now: NOT_READY-then-recover, HARDWARE_ERROR (wedge),
|
||||
// ILLEGAL_REQUEST (wedge), and ABORTED_COMMAND profiles. Each would
|
||||
// trigger long real-time sleeps inside `handle_read_failure`:
|
||||
//
|
||||
// - NOT_READY (sense_key=0x02, asc=0x02/0x03/0x04): 15 s pause per
|
||||
// occurrence (`patch_not_ready_pause`), and retries the same LBA
|
||||
// in-place. Even one NOT_READY costs the test 15 s wall-time.
|
||||
//
|
||||
// - HARDWARE_ERROR / ILLEGAL_REQUEST: 30 s per occurrence
|
||||
// (`WEDGE_FAMILY_COOLDOWN_SECS`), bounded by
|
||||
// `WEDGE_ABORT_THRESHOLD=16` before wedged-exit. Worst case ~8
|
||||
// minutes per profile.
|
||||
//
|
||||
// The sleeps are not injectable. Adding them would require either a
|
||||
// `now()` / `sleep()` trait injection (out of scope for the unification
|
||||
// task) or a "test mode" compile-time flag (architectural smell). The
|
||||
// behavioural contracts for those paths are captured in
|
||||
// `read_error.rs`'s in-module tests instead — they exercise the
|
||||
// classifier without invoking the patch loop's sleep side-effects.
|
||||
//
|
||||
// If the unification ever proceeds, the next step is to add a clock
|
||||
// injection point in `handle_read_failure` and extend this fixture
|
||||
// with the wedge/NOT_READY profiles too.
|
||||
@@ -1,10 +1,5 @@
|
||||
//! Integration tests for the PES stream pipeline.
|
||||
|
||||
// 0.18 trait split: this suite still drives the deprecated `pes::Stream`
|
||||
// trait directly. It will be migrated to `FrameSource`/`FrameSink` in the
|
||||
// follow-up that ports concrete impls.
|
||||
#![allow(deprecated)]
|
||||
|
||||
use libfreemkv::mux::meta::M2tsMeta;
|
||||
use libfreemkv::pes::Stream as PesStream;
|
||||
use libfreemkv::*;
|
||||
|
||||
+4
-4
@@ -1,7 +1,7 @@
|
||||
//! UDF parser tests using a MockSectorReader.
|
||||
|
||||
use libfreemkv::error::Result;
|
||||
use libfreemkv::{SectorReader, read_filesystem};
|
||||
use libfreemkv::{SectorSource, read_filesystem};
|
||||
use std::collections::HashMap;
|
||||
|
||||
const SECTOR_SIZE: usize = 2048;
|
||||
@@ -38,7 +38,7 @@ impl MockSectorReader {
|
||||
}
|
||||
}
|
||||
|
||||
impl SectorReader for MockSectorReader {
|
||||
impl SectorSource for MockSectorReader {
|
||||
fn read_sectors(
|
||||
&mut self,
|
||||
lba: u32,
|
||||
@@ -463,11 +463,11 @@ fn find_dir_case_insensitive() {
|
||||
|
||||
#[test]
|
||||
fn sector_reader_is_object_safe() {
|
||||
// Verify SectorReader can be used as a trait object
|
||||
// Verify SectorSource can be used as a trait object
|
||||
let mut reader = MockSectorReader::new();
|
||||
reader.set_sector(0, vec![42u8; SECTOR_SIZE]);
|
||||
|
||||
let dyn_reader: &mut dyn SectorReader = &mut reader;
|
||||
let dyn_reader: &mut dyn SectorSource = &mut reader;
|
||||
let mut buf = vec![0u8; SECTOR_SIZE];
|
||||
let n = dyn_reader.read_sectors(0, 1, &mut buf, true).unwrap();
|
||||
assert_eq!(n, SECTOR_SIZE);
|
||||
|
||||
Reference in New Issue
Block a user