Bound the synthesized image, and stop it pinning the page cache
A DVD title set records where its VOBS begins as an offset inside its own IFO, and the planner honours that offset because honouring it is what makes a real backup readable. Nothing bounded it: a regenerated .BUP or a hand-assembled folder naming an offset far past the content grew the image to wherever it pointed — a u32 sector count reaches ~8.8 TB, and writing that to an iso:// destination fills a disk with zeros before anything notices. Capped at 128 GiB, which clears BD-100 with room. Metadata is materialized up front and held for the life of the image, at a 2 KiB File Entry per node, so the 100,000-entry cap alone permitted ~205 MB of it for content of no size at all — and the mux holds two images at once while probing. The module claimed a budget of a few MiB; that budget is now enforced rather than asserted. Host reads had no page-cache eviction. The ISO source documents what that costs, measured: an 85 GB read pins the whole file, starves the writer, and collapses the mux to 2.7 MB/s against 70 MB/s isolated. A folder source reads host files the same way, so it now uses the same eviction — the hints move from private-to-that-module to crate-internal rather than being reimplemented.
This commit is contained in:
@@ -59,6 +59,30 @@ const MAX_CS0_NAME_BYTES: usize = 254;
|
||||
/// value is `u16::MAX - 1`.
|
||||
const MAX_SUBDIRS: usize = (u16::MAX - 1) as usize;
|
||||
|
||||
/// Largest image this planner will synthesize, in sectors (128 GiB).
|
||||
///
|
||||
/// A DVD title set records where its VOBS begins as an offset in its own IFO,
|
||||
/// and that offset is read verbatim out of a file in the folder. A regenerated
|
||||
/// `.BUP`, a tool that rewrote an IFO, or a hand-assembled folder can therefore
|
||||
/// name an offset far beyond the content — and the planner honours it, because
|
||||
/// honouring it is what makes a real backup readable. Without a ceiling the
|
||||
/// image grows to wherever that offset points: a `u32` sector count reaches
|
||||
/// ~8.8 TB, and writing one to an `iso://` destination would fill a disk with
|
||||
/// zeros before anything noticed.
|
||||
///
|
||||
/// 128 GiB clears the largest real medium (BD-100) with room to spare, so a
|
||||
/// genuine disc folder never meets it.
|
||||
const MAX_IMAGE_SECTORS: u32 = (128u64 * 1024 * 1024 * 1024 / SECTOR as u64) as u32;
|
||||
|
||||
/// Ceiling on the in-memory metadata region (64 MiB).
|
||||
///
|
||||
/// Every node costs a 2 KiB File Entry sector held for the life of the image,
|
||||
/// so the entry cap alone permits ~205 MB of metadata for content of no size at
|
||||
/// all — and the mux holds two of these at once while probing. The module
|
||||
/// documents a budget of "a few MiB even for a large Blu-ray"; this is what
|
||||
/// enforces it rather than merely asserting it.
|
||||
const MAX_META_BYTES: u64 = 64 * 1024 * 1024;
|
||||
|
||||
/// The fan-out cap must bite before the global entry cap, or it never fires.
|
||||
const _: () = assert!(MAX_SUBDIRS < MAX_ENTRIES);
|
||||
|
||||
@@ -605,6 +629,15 @@ pub(super) fn plan(root: &Path) -> Result<Layout> {
|
||||
let part_sectors = cursor;
|
||||
let total = part_start as u64 + part_sectors as u64 + 1; // + trailing anchor
|
||||
let total_sectors = u32::try_from(total).map_err(|_| Error::DirImageTooLarge)?;
|
||||
if total_sectors > MAX_IMAGE_SECTORS {
|
||||
return Err(Error::DirImageTooLarge);
|
||||
}
|
||||
// Metadata is materialized up front and held for the life of the image, so
|
||||
// its size is bounded here rather than discovered when memory runs out.
|
||||
let meta_bytes = (dir_count as u64 + file_count as u64).saturating_mul(SECTOR as u64);
|
||||
if meta_bytes > MAX_META_BYTES {
|
||||
return Err(Error::DirImageTooLarge);
|
||||
}
|
||||
|
||||
let volume_id = root
|
||||
.file_name()
|
||||
|
||||
+39
-1
@@ -36,6 +36,14 @@ mod encode;
|
||||
mod layout;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
#[cfg(target_os = "linux")]
|
||||
use crate::io::file_sector_source::linux::drop_window;
|
||||
#[cfg(target_os = "macos")]
|
||||
use crate::io::file_sector_source::macos::drop_window;
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))]
|
||||
use crate::io::file_sector_source::other::drop_window;
|
||||
#[cfg(target_os = "windows")]
|
||||
use crate::io::file_sector_source::windows::drop_window;
|
||||
use crate::sector::SectorSource;
|
||||
use encode::{MetaSectors, SECTOR};
|
||||
use std::fs::File;
|
||||
@@ -78,6 +86,12 @@ struct FileRef {
|
||||
/// Owns everything it reads through (`PathBuf`s and its own file handles), so
|
||||
/// it is `Send + 'static` and can be moved into `build_iso_pipeline`, which
|
||||
/// hands it to `PrefetchedSectorSource`'s producer thread.
|
||||
/// Bytes read between page-cache eviction calls (32 MiB).
|
||||
///
|
||||
/// Large enough that the hint costs nothing measurable against a rip, small
|
||||
/// enough that resident pages stay bounded well below any machine's RAM.
|
||||
const DROP_CHUNK_BYTES: u64 = 32 * 1024 * 1024;
|
||||
|
||||
pub struct DirImage {
|
||||
meta: MetaSectors,
|
||||
/// Sorted by `start_lba`, non-overlapping.
|
||||
@@ -87,6 +101,15 @@ pub struct DirImage {
|
||||
total_sectors: u32,
|
||||
volume_id: String,
|
||||
data_bytes: u64,
|
||||
/// Bytes read from host files since the last page-cache eviction.
|
||||
///
|
||||
/// A rip streams every byte of the folder exactly once. Without eviction
|
||||
/// the kernel keeps all of it resident, which starves the concurrent writer
|
||||
/// — `io::file_sector_source` records the measured cost of exactly this
|
||||
/// omission on the ISO path (2.7 MB/s mux against 70 MB/s isolated reads).
|
||||
/// A folder source reads host files the same way and needs the same
|
||||
/// treatment.
|
||||
bytes_since_drop: u64,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for DirImage {
|
||||
@@ -160,6 +183,7 @@ impl DirImage {
|
||||
total_sectors: plan.total_sectors,
|
||||
volume_id: plan.volume_id,
|
||||
data_bytes,
|
||||
bytes_since_drop: 0,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -225,7 +249,21 @@ impl DirImage {
|
||||
let file = r.file;
|
||||
let h = self.handle(file)?;
|
||||
h.seek(SeekFrom::Start(at)).map_err(Error::from)?;
|
||||
match h.read_exact(&mut out[..want]) {
|
||||
let res = h.read_exact(&mut out[..want]);
|
||||
if res.is_ok() {
|
||||
// Evict what we have consumed, per file handle. The window is the
|
||||
// read just completed rather than a running offset, because reads
|
||||
// here jump between files and a single monotonic cursor would name
|
||||
// the wrong pages.
|
||||
self.bytes_since_drop = self.bytes_since_drop.saturating_add(want as u64);
|
||||
if self.bytes_since_drop >= DROP_CHUNK_BYTES {
|
||||
if let Some((_, fh)) = self.open.iter().find(|(i, _)| *i == file) {
|
||||
drop_window(fh, at, want as u64);
|
||||
}
|
||||
self.bytes_since_drop = 0;
|
||||
}
|
||||
}
|
||||
match res {
|
||||
Ok(()) => Ok(()),
|
||||
// The file shrank while the handle was open. Same verdict as the
|
||||
// size check in `handle`, reached the other way.
|
||||
|
||||
Reference in New Issue
Block a user