pq/libfreemkv
1
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Build BlockGroups in memory so the hot path never seeks

Browse Source 
Every BlockGroup frame back-patched its element size via ebml::end_master, which
does two stream_position() calls and two real seeks. BufWriter does not override
Seek::stream_position, so each position query is seek(Current(0)) = flush_buf +
lseek — and each of those flushed the 4 MiB BufWriter while every position-moving
seek reset WritebackPipeline::last_flush_pos. The buffer never got to do its job.

An MPEG-2 title takes this path for EVERY frame (the parser stamps a per-frame
duration, so I, P and B all become BlockGroups) — roughly 350,000 per feature.

A BlockGroup's size is knowable before writing, so there is no need to back-patch
at all. New seek-free twins start_master_buf / end_master_buf patch a placeholder
by buffer INDEX instead of file offset, and build_block_group assembles the whole
element into a persistent buffer that write_block_group and its MVC sibling take,
fill, write once, and hand back — including on the error path — so the allocation
is made once rather than per frame.

Measured with a counting writer that, like BufWriter, does not override
stream_position, over 200 frames:

              seek calls   position-moving
  plain   before 912              451
  plain   after  112               51
  MVC     before 2516            1253
  MVC     after  116               53

Per-frame cost is now zero; the residual is the header, the per-cluster
back-patch and Cues. For 350k BlockGroups that is 1.4M seek calls removed on the
plain path, 4.2M on an MVC title.

end_master is deliberately NOT changed for its other callers. The Cluster master
genuinely streams — frames are appended to an open cluster over time, so its body
cannot be buffered without holding a whole cluster in memory — and the rest
(EBML header, Tracks, Info, Chapters, Cues) run once, not per frame.

Byte-identity is the safety property, and it is structural: end_master always
patches a FIXED-width 8-byte VINT, and the buffered pair writes and patches
exactly that same placeholder, so the encodings cannot differ. Verified by
capture-then-compare over 200 frames (17 keyframes / 183 non-keyframes, multiple
clusters, BlockDuration present and absent, both reference branches) — output
byte-for-byte identical.

Three tests now pin it permanently: buffered vs seeking output byte-for-byte for
empty/tiny/multi-byte bodies, the same for NESTED masters (the MVC path nests
BlockAdditions > BlockMore, where an index-arithmetic slip would surface), and
end_master_buf erroring rather than panicking on a position outside the buffer.
All three verified red against a deliberately divergent placeholder width.

A note on verifying this kind of change: comparing emitted MKV across two
worktrees at different commits shows a spurious 14-byte difference, because
MuxingApp/WritingApp embed the build's git SHA twice. Compare at the same base.
This commit is contained in:
Matthew Jackson
2026-07-29 19:33:31 -07:00
parent 807eb053ca
commit e3676e7cdf
2 changed files with 220 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mux/mkv.rs
+85 -41
View File
@@ -617,6 +617,20 @@ pub struct MkvMuxer<W: Write + Seek> {
cluster_pos: u64,
cluster_size_pos: u64,
cluster_ts_ticks: i64,
/// Reusable scratch buffer for assembling ONE BlockGroup before it is
/// written with a single `write_all`.
///
/// The BlockGroup path is the hot one — the MPEG-2 parser stamps a per-frame
/// duration, so every I/P/B frame lands there, plus AC-3 audio and PGS
/// subtitles (~350k BlockGroups for a 90-minute feature). Building the
/// element in memory means its size is known before it reaches the file, so
/// no `start_master`/`end_master` back-patch is needed: that removes the
/// per-frame `stream_position()` + two `seek()`s, each of which flushed the
/// 4 MiB `BufWriter` (`BufWriter` does not override `stream_position`, so a
/// position query is `seek(Current(0))` = flush + lseek) and reset the
/// writeback pipeline's `last_flush_pos`. Kept on the muxer so the
/// allocation is made once, not per frame.
block_group_buf: Vec<u8>,
base_pts_ticks: Option<i64>,
/// Last block timecode (TimestampScale ticks, relative to base_pts) written
/// PER TRACK, to enforce strictly-monotonic per-track timestamps —
@@ -1237,6 +1251,7 @@ impl<W: Write + Seek> MkvMuxer<W> {
cluster_open: false,
cluster_pos: 0,
cluster_size_pos: 0,
block_group_buf: Vec::new(),
cluster_ts_ticks: 0,
base_pts_ticks: None,
last_pts_ticks: std::collections::HashMap::new(),
@@ -1851,24 +1866,69 @@ impl<W: Write + Seek> MkvMuxer<W> {
data: &[u8],
reference: Option<i64>,
duration_ticks: u64,
) -> io::Result<()> {
let mut buf = std::mem::take(&mut self.block_group_buf);
buf.clear();
let res = Self::build_block_group(
&mut buf,
track_num,
relative_ts,
data,
reference,
Some(duration_ticks),
None,
)
.and_then(|()| self.writer.write_all(&buf));
// Hand the (now grown) scratch buffer back so the next frame reuses the
// allocation, on the error path too.
self.block_group_buf = buf;
res
}
/// Assemble a complete BlockGroup into `buf` — one `write_all` at the call
/// site, no `stream_position` and no `seek`.
///
/// `additional` is `Some(dependent AU)` for the MVC path, which appends
/// `BlockAdditions > BlockMore { BlockAddID=2, BlockAdditional }`.
fn build_block_group(
buf: &mut Vec<u8>,
track_num: usize,
relative_ts: i16,
data: &[u8],
reference: Option<i64>,
duration_ticks: Option<u64>,
additional: Option<&[u8]>,
) -> io::Result<()> {
let (tv, tv_len) = track_vint(track_num);
let track_vint = &tv[..tv_len];
// The 0x80 Keyframe flag is SimpleBlock-only; inside a BlockGroup Block
// it is reserved and MUST be 0 — keyframe-ness is signalled by the
// presence/absence of ReferenceBlock.
let flags: u8 = 0x00;
let block_size = track_vint.len() + 2 + 1 + data.len();
let bg_pos = ebml::start_master(&mut self.writer, ebml::BLOCK_GROUP)?;
ebml::write_id(&mut self.writer, ebml::BLOCK)?;
ebml::write_size(&mut self.writer, block_size as u64)?;
self.writer.write_all(track_vint)?;
self.writer.write_all(&relative_ts.to_be_bytes())?;
self.writer.write_all(&[flags])?;
self.writer.write_all(data)?;
ebml::write_uint(&mut self.writer, ebml::BLOCK_DURATION, duration_ticks)?;
if let Some(ref_off) = reference {
ebml::write_int(&mut self.writer, ebml::REFERENCE_BLOCK, ref_off)?;
let bg_pos = ebml::start_master_buf(buf, ebml::BLOCK_GROUP)?;
ebml::write_id(buf, ebml::BLOCK)?;
ebml::write_size(buf, block_size as u64)?;
buf.extend_from_slice(track_vint);
buf.extend_from_slice(&relative_ts.to_be_bytes());
buf.push(flags);
buf.extend_from_slice(data);
if let Some(dt) = duration_ticks {
ebml::write_uint(buf, ebml::BLOCK_DURATION, dt)?;
}
ebml::end_master(&mut self.writer, bg_pos)?;
if let Some(ref_off) = reference {
ebml::write_int(buf, ebml::REFERENCE_BLOCK, ref_off)?;
}
if let Some(additional) = additional {
let adds_pos = ebml::start_master_buf(buf, ebml::BLOCK_ADDITIONS)?;
let more_pos = ebml::start_master_buf(buf, ebml::BLOCK_MORE)?;
ebml::write_uint(buf, ebml::BLOCK_ADD_ID, BLOCK_ADD_ID_VALUE_MVC)?;
ebml::write_binary(buf, ebml::BLOCK_ADDITIONAL, additional)?;
ebml::end_master_buf(buf, more_pos)?;
ebml::end_master_buf(buf, adds_pos)?;
}
ebml::end_master_buf(buf, bg_pos)?;
Ok(())
}
@@ -1886,36 +1946,20 @@ impl<W: Write + Seek> MkvMuxer<W> {
reference: Option<i64>,
duration_ticks: Option<u64>,
) -> io::Result<()> {
let (tv, tv_len) = track_vint(track_num);
let track_vint = &tv[..tv_len];
// The 0x80 Keyframe flag is SimpleBlock-only; inside a BlockGroup Block
// it is reserved and MUST be 0 — keyframe-ness is signalled by the
// presence/absence of ReferenceBlock.
let flags: u8 = 0x00;
let block_size = track_vint.len() + 2 + 1 + data.len();
let bg_pos = ebml::start_master(&mut self.writer, ebml::BLOCK_GROUP)?;
ebml::write_id(&mut self.writer, ebml::BLOCK)?;
ebml::write_size(&mut self.writer, block_size as u64)?;
self.writer.write_all(track_vint)?;
self.writer.write_all(&relative_ts.to_be_bytes())?;
self.writer.write_all(&[flags])?;
self.writer.write_all(data)?;
if let Some(dt) = duration_ticks {
ebml::write_uint(&mut self.writer, ebml::BLOCK_DURATION, dt)?;
}
if let Some(ref_off) = reference {
ebml::write_int(&mut self.writer, ebml::REFERENCE_BLOCK, ref_off)?;
}
// BlockAdditions → BlockMore { BlockAddID=2, BlockAdditional=dependent AU }.
let adds_pos = ebml::start_master(&mut self.writer, ebml::BLOCK_ADDITIONS)?;
let more_pos = ebml::start_master(&mut self.writer, ebml::BLOCK_MORE)?;
ebml::write_uint(&mut self.writer, ebml::BLOCK_ADD_ID, BLOCK_ADD_ID_VALUE_MVC)?;
ebml::write_binary(&mut self.writer, ebml::BLOCK_ADDITIONAL, additional)?;
ebml::end_master(&mut self.writer, more_pos)?;
ebml::end_master(&mut self.writer, adds_pos)?;
ebml::end_master(&mut self.writer, bg_pos)?;
Ok(())
let mut buf = std::mem::take(&mut self.block_group_buf);
buf.clear();
let res = Self::build_block_group(
&mut buf,
track_num,
relative_ts,
data,
reference,
duration_ticks,
Some(additional),
)
.and_then(|()| self.writer.write_all(&buf));
self.block_group_buf = buf;
res
}
}