v0.13.13 — telemetry: tracing instrumentation in SgIoTransport + Disc::copy

v0.13.12 shipped the async fd_recovery design but a live test on Dune 2
showed Pass 1 sat for 14 minutes with bytes_good=0 — the inner loop iterates
(throttled on_progress log fires every 78s) but each iteration evidently
takes ~60s instead of the microseconds the design promises on fast-fail.
Without trace-level telemetry at the SCSI + Disc::copy boundaries we
can't tell where the time goes.

This release is instrumentation only — no behavior change.

- New dep: tracing 0.1. Per CLAUDE.md, debug/trace logging is allowed in
  libfreemkv (the no-English rule applies to errors). Consumers wire a
  tracing subscriber.
- SgIoTransport::execute (Linux): trace at every state transition (entry,
  recovery_swap_ok, recovery_pending, write_ok / write_err, poll_done,
  timeout_spawn_recovery, scsi_err, read_err, ok). Each event includes
  opcode + elapsed timing. The bg recovery thread also traces close_ms +
  open_ms so we can see if the kernel really takes 60s to close+open on a
  wedged Initio bridge.
- Disc::copy: trace at copy_start, outer_loop, region_enter, every 100
  inner-loop iterations (iter_progress with pos/region_end/skip_size/
  bytes_good/read_ok_count/read_err_count/last_read_ms/copy_elapsed_ms),
  copy_done.
- All trace events use targets `freemkv::scsi` and `freemkv::disc` so
  consumers can filter by subsystem (e.g. autorip /api/debug?q=freemkv::scsi).

Next: run the live test on Dune 2 again, read the autorip JSONL log,
diagnose why each iter is slow, fix the actual bug.
This commit is contained in:
2026-04-25 18:32:05 -07:00
parent 4fd60389fa
commit ae18dc35b7
4 changed files with 215 additions and 6 deletions
+66
View File
@@ -1286,6 +1286,20 @@ impl Disc {
let mut buf = vec![0u8; batch as usize * 2048];
let mut bytes_done = 0u64;
let mut halt_requested = false;
let copy_t0 = std::time::Instant::now();
let mut iter_count: u64 = 0;
let mut read_ok_count: u64 = 0;
let mut read_err_count: u64 = 0;
let mut last_log_iter: u64 = 0;
tracing::trace!(
target: "freemkv::disc",
phase = "copy_start",
total_bytes,
batch,
skip_init,
skip_max,
"Disc::copy entered"
);
// Iterate over not-yet-finished regions from the mapfile. We re-read the
// mapfile after each block because record() mutates the region list.
@@ -1295,6 +1309,12 @@ impl Disc {
mapfile::SectorStatus::NonTrimmed,
mapfile::SectorStatus::NonScraped,
]);
tracing::trace!(
target: "freemkv::disc",
phase = "outer_loop",
regions_remaining = regions_to_do.len(),
"Disc::copy outer iter"
);
if regions_to_do.is_empty() {
break;
}
@@ -1307,6 +1327,14 @@ impl Disc {
};
let region_end = region_pos + region_size;
let mut pos = region_pos;
tracing::trace!(
target: "freemkv::disc",
phase = "region_enter",
region_pos,
region_size,
region_end,
"entering NonTried region"
);
while pos < region_end {
if let Some(ref h) = opts.halt {
@@ -1321,11 +1349,15 @@ impl Disc {
let bytes = count as usize * 2048;
let recovery = !opts.skip_on_error; // fast reads when skipping
iter_count += 1;
let read_t0 = std::time::Instant::now();
let read_ok = reader
.read_sectors(lba, count, &mut buf[..bytes], recovery)
.is_ok();
let read_elapsed_ms = read_t0.elapsed().as_millis() as u64;
if read_ok {
read_ok_count += 1;
if opts.decrypt {
crate::decrypt::decrypt_sectors(&mut buf[..bytes], &keys, 0)?;
}
@@ -1339,6 +1371,7 @@ impl Disc {
skip_size = skip_init; // reset after success
pos += block_bytes;
} else if opts.skip_on_error {
read_err_count += 1;
// Zero-fill this block, mark non-trimmed for later patch trim.
buf[..bytes].fill(0);
file.seek(SeekFrom::Start(pos))
@@ -1364,6 +1397,27 @@ impl Disc {
return Err(Error::DiscRead { sector: lba as u64 });
}
// Throttled iter telemetry — every 100 inner iterations.
if iter_count - last_log_iter >= 100 {
last_log_iter = iter_count;
let stats = map.stats();
tracing::trace!(
target: "freemkv::disc",
phase = "iter_progress",
iter_count,
read_ok_count,
read_err_count,
last_read_ms = read_elapsed_ms,
pos,
region_end,
skip_size,
bytes_good = stats.bytes_good,
bytes_pending = stats.bytes_pending,
copy_elapsed_ms = copy_t0.elapsed().as_millis() as u64,
"Disc::copy inner iter"
);
}
if let Some(cb) = opts.on_progress {
let stats = map.stats();
cb(stats.bytes_good, total_bytes);
@@ -1373,6 +1427,18 @@ impl Disc {
file.sync_all().map_err(|e| Error::IoError { source: e })?;
let stats = map.stats();
tracing::trace!(
target: "freemkv::disc",
phase = "copy_done",
iter_count,
read_ok_count,
read_err_count,
bytes_good = stats.bytes_good,
bytes_pending = stats.bytes_pending,
halted = halt_requested,
copy_elapsed_ms = copy_t0.elapsed().as_millis() as u64,
"Disc::copy returning"
);
Ok(CopyResult {
bytes_total: total_bytes,
bytes_good: stats.bytes_good,