0.18: rename crate::io::Writer → WritebackFile
This commit is contained in:
+9
-11
@@ -1418,10 +1418,11 @@ impl Disc {
|
|||||||
f
|
f
|
||||||
};
|
};
|
||||||
|
|
||||||
// Wrap the raw `File` in our bounded-cache writer (drains
|
// Wrap the raw `File` in our bounded-cache `WritebackFile`
|
||||||
// dirty pages continuously instead of bursting; see
|
// (drains dirty pages continuously instead of bursting; see
|
||||||
// `crate::io`). The Writer moves into the consumer thread.
|
// `crate::io`). The `WritebackFile` moves into the consumer
|
||||||
let file = crate::io::Writer::new(file).map_err(|e| Error::IoError { source: e })?;
|
// thread.
|
||||||
|
let file = crate::io::WritebackFile::new(file).map_err(|e| Error::IoError { source: e })?;
|
||||||
let batch: u16 = match opts.batch_sectors {
|
let batch: u16 = match opts.batch_sectors {
|
||||||
Some(b) => b,
|
Some(b) => b,
|
||||||
None if opts.skip_on_error => ecc_sectors(self.format),
|
None if opts.skip_on_error => ecc_sectors(self.format),
|
||||||
@@ -1435,7 +1436,7 @@ impl Disc {
|
|||||||
// sweep finishes are the patch pass's job.
|
// sweep finishes are the patch pass's job.
|
||||||
let regions: Vec<(u64, u64)> = map.ranges_with(&[mapfile::SectorStatus::NonTried]);
|
let regions: Vec<(u64, u64)> = map.ranges_with(&[mapfile::SectorStatus::NonTried]);
|
||||||
|
|
||||||
// Spawn the consumer. It owns Writer + Mapfile; the producer
|
// Spawn the consumer. It owns WritebackFile + Mapfile; the producer
|
||||||
// (this thread) keeps `reader`, `read_ctx`, halt + set_speed.
|
// (this thread) keeps `reader`, `read_ctx`, halt + set_speed.
|
||||||
let (work_tx, prog_rx, consumer_handle) = spawn_consumer(ConsumerInputs {
|
let (work_tx, prog_rx, consumer_handle) = spawn_consumer(ConsumerInputs {
|
||||||
file,
|
file,
|
||||||
@@ -1978,14 +1979,11 @@ impl Disc {
|
|||||||
let is_regular = std::fs::metadata(path)
|
let is_regular = std::fs::metadata(path)
|
||||||
.map(|m| m.file_type().is_file())
|
.map(|m| m.file_type().is_file())
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
let file = std::fs::OpenOptions::new()
|
// Same bounded-cache `WritebackFile` sweep uses, so patch's
|
||||||
.write(true)
|
|
||||||
.open(path)
|
|
||||||
.map_err(|e| Error::IoError { source: e })?;
|
|
||||||
// Same bounded-cache writeback wrapper sweep uses, so patch's
|
|
||||||
// recovery writes (sparse but can be many across a damaged region)
|
// recovery writes (sparse but can be many across a damaged region)
|
||||||
// get the burst-flush protection on slow / NFS-backed staging.
|
// get the burst-flush protection on slow / NFS-backed staging.
|
||||||
let mut file = crate::io::Writer::new(file).map_err(|e| Error::IoError { source: e })?;
|
let mut file =
|
||||||
|
crate::io::WritebackFile::open(path).map_err(|e| Error::IoError { source: e })?;
|
||||||
|
|
||||||
// Log ISO file size at patch start for write monitoring
|
// Log ISO file size at patch start for write monitoring
|
||||||
if let Ok(metadata) = std::fs::metadata(path) {
|
if let Ok(metadata) = std::fs::metadata(path) {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
//! both costs.
|
//! both costs.
|
||||||
//!
|
//!
|
||||||
//! This module decouples them. A consumer thread owns the
|
//! This module decouples them. A consumer thread owns the
|
||||||
//! [`crate::io::Writer`] (the ISO file) and the
|
//! [`crate::io::WritebackFile`] (the ISO file) and the
|
||||||
//! [`super::mapfile::Mapfile`]. The producer thread (the caller of
|
//! [`super::mapfile::Mapfile`]. The producer thread (the caller of
|
||||||
//! `Disc::sweep`) keeps the [`crate::sector::SectorReader`], the
|
//! `Disc::sweep`) keeps the [`crate::sector::SectorReader`], the
|
||||||
//! [`super::read_error`] state machine, and decrypt — so what enters
|
//! [`super::read_error`] state machine, and decrypt — so what enters
|
||||||
@@ -112,7 +112,7 @@ pub(super) struct ConsumerSummary {
|
|||||||
/// happens on the producer side before send, so the consumer never
|
/// happens on the producer side before send, so the consumer never
|
||||||
/// sees keys.
|
/// sees keys.
|
||||||
pub(super) struct ConsumerInputs {
|
pub(super) struct ConsumerInputs {
|
||||||
pub file: crate::io::Writer,
|
pub file: crate::io::WritebackFile,
|
||||||
pub map: Mapfile,
|
pub map: Mapfile,
|
||||||
/// `sync_all`-on-failure-is-an-error iff the output is a regular
|
/// `sync_all`-on-failure-is-an-error iff the output is a regular
|
||||||
/// file. `/dev/null` and pipes always fail `sync_all`; that's not
|
/// file. `/dev/null` and pipes always fail `sync_all`; that's not
|
||||||
@@ -279,7 +279,7 @@ fn apply_item(
|
|||||||
.file
|
.file
|
||||||
.seek(SeekFrom::Start(pos))
|
.seek(SeekFrom::Start(pos))
|
||||||
.map_err(|e| Error::IoError { source: e })?;
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
// Subsequent writes are sequential; `crate::io::Writer`'s
|
// Subsequent writes are sequential; `crate::io::WritebackFile`'s
|
||||||
// seek-elision keeps them on the writeback pipeline path.
|
// seek-elision keeps them on the writeback pipeline path.
|
||||||
let mut filled = 0u64;
|
let mut filled = 0u64;
|
||||||
while filled < len {
|
while filled < len {
|
||||||
|
|||||||
+11
-11
@@ -1,16 +1,16 @@
|
|||||||
//! File I/O helpers that bound kernel cache pressure on big writes.
|
//! File I/O helpers that bound kernel cache pressure on big writes.
|
||||||
//!
|
//!
|
||||||
//! `Writer` is a drop-in wrapper around `std::fs::File` for any call
|
//! `WritebackFile` is a drop-in wrapper around `std::fs::File` for any
|
||||||
//! site that performs large sequential writes (sweep, mux, etc.). It
|
//! call site that performs large sequential writes (sweep, patch, mux,
|
||||||
//! implements `Write` and `Seek` so existing code paths can swap
|
//! etc.). It implements `Write` and `Seek` so existing code paths can
|
||||||
//! `File` for `Writer` with no body changes. Internally it drives a
|
//! swap `File` for `WritebackFile` with no body changes. Internally it
|
||||||
//! `WritebackPipeline` that, on Linux, drains dirty pages continuously
|
//! drives a `WritebackPipeline` that, on Linux, drains dirty pages
|
||||||
//! at 32 MB granularity to avoid the kernel's accumulate-then-burst
|
//! continuously at 32 MB granularity to avoid the kernel's
|
||||||
//! flush behaviour. macOS and Windows use a no-op pipeline — their
|
//! accumulate-then-burst flush behaviour. macOS and Windows use a
|
||||||
//! default cache policies have not been shown to exhibit the same
|
//! no-op pipeline — their default cache policies have not been shown
|
||||||
//! pathology for this access pattern.
|
//! to exhibit the same pathology for this access pattern.
|
||||||
|
|
||||||
mod writeback;
|
mod writeback;
|
||||||
mod writer;
|
mod writeback_file;
|
||||||
|
|
||||||
pub(crate) use writer::Writer;
|
pub(crate) use writeback_file::WritebackFile;
|
||||||
|
|||||||
@@ -0,0 +1,112 @@
|
|||||||
|
//! `WritebackFile` — a `File` wrapper whose reason for existing is the
|
||||||
|
//! bounded-cache writeback pipeline.
|
||||||
|
//!
|
||||||
|
//! Why: large sequential writes (sweep, patch, mux on UHD-scale output)
|
||||||
|
//! left to the kernel's default writeback policy accumulate hundreds of
|
||||||
|
//! megabytes of dirty pages and then burst-flush, stalling subsequent
|
||||||
|
//! writes for seconds at a time. `WritebackFile` drives a continuous
|
||||||
|
//! [`super::writeback::WritebackPipeline`] that on Linux issues
|
||||||
|
//! incremental `sync_file_range` + `posix_fadvise(DONTNEED)` calls at
|
||||||
|
//! 32 MB granularity so dirty pages drain at the same rate they're
|
||||||
|
//! produced. macOS and Windows fall through to a no-op pipeline — their
|
||||||
|
//! default cache policies have not been shown to exhibit the same
|
||||||
|
//! pathology for this access pattern.
|
||||||
|
//!
|
||||||
|
//! It implements `Write` and `Seek` so any call site that wrote to a
|
||||||
|
//! plain `File` through those traits (sweep, patch, mux) can swap in
|
||||||
|
//! `WritebackFile` without touching the body of the loop. The wrapper
|
||||||
|
//! also tracks the current file position to feed the pipeline with
|
||||||
|
//! progress + seek boundaries.
|
||||||
|
//!
|
||||||
|
//! See `super::writeback::linux` for the underlying pathology and the
|
||||||
|
//! strategy.
|
||||||
|
|
||||||
|
use std::fs::{File, OpenOptions};
|
||||||
|
use std::io::{self, Seek, SeekFrom, Write};
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use super::writeback::WritebackPipeline;
|
||||||
|
|
||||||
|
const CHUNK_BYTES: u64 = 32 * 1024 * 1024;
|
||||||
|
|
||||||
|
pub(crate) struct WritebackFile {
|
||||||
|
file: File,
|
||||||
|
pipeline: WritebackPipeline,
|
||||||
|
pos: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WritebackFile {
|
||||||
|
/// Wrap an open `File`. The current OS file position is queried
|
||||||
|
/// once so the pipeline starts tracking from wherever the file
|
||||||
|
/// already is (typically 0 for fresh files; non-zero for resumed
|
||||||
|
/// or appended files).
|
||||||
|
pub(crate) fn new(mut file: File) -> io::Result<Self> {
|
||||||
|
let pos = file.stream_position()?;
|
||||||
|
let pipeline = WritebackPipeline::new(&file, pos, CHUNK_BYTES);
|
||||||
|
Ok(Self {
|
||||||
|
file,
|
||||||
|
pipeline,
|
||||||
|
pos,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a new file at `path` (truncating any existing contents)
|
||||||
|
/// and wrap it. Convenience for the common
|
||||||
|
/// `File::create(path)` + `WritebackFile::new(file)` pair so callers
|
||||||
|
/// don't have to assemble a `File` first.
|
||||||
|
pub(crate) fn create(path: &Path) -> io::Result<Self> {
|
||||||
|
let file = File::create(path)?;
|
||||||
|
Self::new(file)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Open an existing file at `path` for writing (no truncation) and
|
||||||
|
/// wrap it. Mirrors `File::open` semantics for the writable case
|
||||||
|
/// — used by patch / resume paths that mutate an existing ISO in
|
||||||
|
/// place.
|
||||||
|
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
||||||
|
let file = OpenOptions::new().write(true).open(path)?;
|
||||||
|
Self::new(file)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drain in-flight writeback then issue a full fsync. Use this in
|
||||||
|
/// place of `File::sync_all`.
|
||||||
|
pub(crate) fn sync_all(&mut self) -> io::Result<()> {
|
||||||
|
self.pipeline.finalize();
|
||||||
|
self.file.sync_all()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Write for WritebackFile {
|
||||||
|
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||||
|
let n = self.file.write(buf)?;
|
||||||
|
self.pos += n as u64;
|
||||||
|
self.pipeline.note_progress(self.pos);
|
||||||
|
Ok(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||||
|
self.file.write_all(buf)?;
|
||||||
|
self.pos += buf.len() as u64;
|
||||||
|
self.pipeline.note_progress(self.pos);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush(&mut self) -> io::Result<()> {
|
||||||
|
self.file.flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Seek for WritebackFile {
|
||||||
|
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||||
|
let p = self.file.seek(from)?;
|
||||||
|
// Only treat seeks that actually move the position as
|
||||||
|
// boundaries — sweep does a redundant `seek(Current(pos))`
|
||||||
|
// before every write, and we don't want that to drain the
|
||||||
|
// pipeline on every iteration.
|
||||||
|
if p != self.pos {
|
||||||
|
self.pipeline.handle_seek(p);
|
||||||
|
self.pos = p;
|
||||||
|
}
|
||||||
|
Ok(p)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,84 +0,0 @@
|
|||||||
//! `Writer` — a drop-in `File` wrapper that keeps the kernel page
|
|
||||||
//! cache bounded during large sequential output.
|
|
||||||
//!
|
|
||||||
//! Implements `Write` and `Seek`, so any call site that uses `File`
|
|
||||||
//! through those traits (sweep, mux, patch) can swap to `Writer`
|
|
||||||
//! without touching the body of the loop. The wrapper tracks the
|
|
||||||
//! current file position and forwards each write to a
|
|
||||||
//! [`super::writeback::WritebackPipeline`], which on Linux schedules
|
|
||||||
//! incremental `sync_file_range` + `posix_fadvise(DONTNEED)` calls
|
|
||||||
//! to drain dirty pages continuously instead of letting the kernel
|
|
||||||
//! burst-flush hundreds of MB at a time.
|
|
||||||
//!
|
|
||||||
//! See `super::writeback::linux` for the pathology and the strategy.
|
|
||||||
|
|
||||||
use std::fs::File;
|
|
||||||
use std::io::{self, Seek, SeekFrom, Write};
|
|
||||||
|
|
||||||
use super::writeback::WritebackPipeline;
|
|
||||||
|
|
||||||
const CHUNK_BYTES: u64 = 32 * 1024 * 1024;
|
|
||||||
|
|
||||||
pub(crate) struct Writer {
|
|
||||||
file: File,
|
|
||||||
pipeline: WritebackPipeline,
|
|
||||||
pos: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Writer {
|
|
||||||
/// Wrap an open `File`. The current OS file position is queried
|
|
||||||
/// once so the pipeline starts tracking from wherever the file
|
|
||||||
/// already is (typically 0 for fresh files; non-zero for resumed
|
|
||||||
/// or appended files).
|
|
||||||
pub(crate) fn new(mut file: File) -> io::Result<Self> {
|
|
||||||
let pos = file.stream_position()?;
|
|
||||||
let pipeline = WritebackPipeline::new(&file, pos, CHUNK_BYTES);
|
|
||||||
Ok(Self {
|
|
||||||
file,
|
|
||||||
pipeline,
|
|
||||||
pos,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Drain in-flight writeback then issue a full fsync. Use this in
|
|
||||||
/// place of `File::sync_all`.
|
|
||||||
pub(crate) fn sync_all(&mut self) -> io::Result<()> {
|
|
||||||
self.pipeline.finalize();
|
|
||||||
self.file.sync_all()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Write for Writer {
|
|
||||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
|
||||||
let n = self.file.write(buf)?;
|
|
||||||
self.pos += n as u64;
|
|
||||||
self.pipeline.note_progress(self.pos);
|
|
||||||
Ok(n)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
|
||||||
self.file.write_all(buf)?;
|
|
||||||
self.pos += buf.len() as u64;
|
|
||||||
self.pipeline.note_progress(self.pos);
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn flush(&mut self) -> io::Result<()> {
|
|
||||||
self.file.flush()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Seek for Writer {
|
|
||||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
|
||||||
let p = self.file.seek(from)?;
|
|
||||||
// Only treat seeks that actually move the position as
|
|
||||||
// boundaries — sweep does a redundant `seek(Current(pos))`
|
|
||||||
// before every write, and we don't want that to drain the
|
|
||||||
// pipeline on every iteration.
|
|
||||||
if p != self.pos {
|
|
||||||
self.pipeline.handle_seek(p);
|
|
||||||
self.pos = p;
|
|
||||||
}
|
|
||||||
Ok(p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+6
-6
@@ -242,23 +242,23 @@ pub fn output(
|
|||||||
match parsed {
|
match parsed {
|
||||||
StreamUrl::Mkv { ref path } => {
|
StreamUrl::Mkv { ref path } => {
|
||||||
validate_file_path(path, "mkv")?;
|
validate_file_path(path, "mkv")?;
|
||||||
// Wrap the raw `File` in `crate::io::Writer` (bounded-cache
|
// Wrap the output in `crate::io::WritebackFile` (bounded-cache
|
||||||
// writeback) so a UHD-scale MKV mux to slow / network-attached
|
// writeback) so a UHD-scale MKV mux to slow / network-attached
|
||||||
// staging doesn't hit the dirty-page burst pathology that
|
// staging doesn't hit the dirty-page burst pathology that
|
||||||
// sweep already side-steps. BufWriter sits on top to coalesce
|
// sweep already side-steps. BufWriter sits on top to coalesce
|
||||||
// mux's many small EBML element writes.
|
// mux's many small EBML element writes.
|
||||||
let file = std::fs::File::create(path)?;
|
|
||||||
let writer: Box<dyn super::WriteSeek> = Box::new(std::io::BufWriter::with_capacity(
|
let writer: Box<dyn super::WriteSeek> = Box::new(std::io::BufWriter::with_capacity(
|
||||||
IO_BUF_SIZE,
|
IO_BUF_SIZE,
|
||||||
crate::io::Writer::new(file)?,
|
crate::io::WritebackFile::create(path)?,
|
||||||
));
|
));
|
||||||
Ok(Box::new(MkvStream::create(writer, title)?))
|
Ok(Box::new(MkvStream::create(writer, title)?))
|
||||||
}
|
}
|
||||||
StreamUrl::M2ts { ref path } => {
|
StreamUrl::M2ts { ref path } => {
|
||||||
validate_file_path(path, "m2ts")?;
|
validate_file_path(path, "m2ts")?;
|
||||||
let file = std::fs::File::create(path)?;
|
let writer = std::io::BufWriter::with_capacity(
|
||||||
let writer =
|
IO_BUF_SIZE,
|
||||||
std::io::BufWriter::with_capacity(IO_BUF_SIZE, crate::io::Writer::new(file)?);
|
crate::io::WritebackFile::create(path)?,
|
||||||
|
);
|
||||||
Ok(Box::new(M2tsStream::create(writer, title)?))
|
Ok(Box::new(M2tsStream::create(writer, title)?))
|
||||||
}
|
}
|
||||||
StreamUrl::Network { ref addr } => {
|
StreamUrl::Network { ref addr } => {
|
||||||
|
|||||||
Reference in New Issue
Block a user