Pass 1 transport failure recovery without user intervention
This commit is contained in:
+175
-6
@@ -1557,15 +1557,87 @@ impl Disc {
|
|||||||
read_err_count += 1;
|
read_err_count += 1;
|
||||||
consecutive_good = 0;
|
consecutive_good = 0;
|
||||||
|
|
||||||
|
// Transport failure means the USB bridge crashed/wedged. Instead of aborting,
|
||||||
|
// drop to single-sector reads with extended timeout and retry a few times.
|
||||||
|
// This allows recovery without requiring manual intervention.
|
||||||
if err.is_scsi_transport_failure() {
|
if err.is_scsi_transport_failure() {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
target: "freemkv::disc",
|
target: "freemkv::disc",
|
||||||
phase = "transport_failure",
|
phase = "transport_failure",
|
||||||
lba = block_lba,
|
lba = block_lba,
|
||||||
error = %err,
|
error = %err,
|
||||||
"transport failure (bridge crash); aborting copy — caller should USB reset + resume"
|
"transport failure (bridge crash); dropping to single-sector read with extended timeout"
|
||||||
);
|
);
|
||||||
return Err(err);
|
|
||||||
|
// Give the bridge time to recover from crash state
|
||||||
|
std::thread::sleep(std::time::Duration::from_secs(5));
|
||||||
|
|
||||||
|
// Try reading as a single sector instead of a batch - this often succeeds
|
||||||
|
// where bulk reads fail because the bridge can handle smaller transactions
|
||||||
|
let mut single_buf = [0u8; 2048];
|
||||||
|
match reader.read_sectors(block_lba, 1, &mut single_buf[..], true) {
|
||||||
|
Ok(_) => {
|
||||||
|
tracing::info!(
|
||||||
|
target: "freemkv::disc",
|
||||||
|
phase = "transport_recovery_success",
|
||||||
|
lba = block_lba,
|
||||||
|
"Single-sector read succeeded after transport failure"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Write the recovered sector and continue
|
||||||
|
file.seek(SeekFrom::Start(pos))
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
file.write_all(&single_buf)
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
|
||||||
|
// Mark remaining sectors in the batch as NonTrimmed (we only recovered 1 sector)
|
||||||
|
let zero = vec![0u8; block_bytes as usize];
|
||||||
|
file.seek(SeekFrom::Start(pos))
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
file.write_all(&zero)
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
|
||||||
|
map.record(pos, block_bytes, mapfile::SectorStatus::NonTrimmed)
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
bytes_done = bytes_done.saturating_add(block_bytes);
|
||||||
|
|
||||||
|
damage_window.push(false);
|
||||||
|
if damage_window.len() > PASS1_DAMAGE_WINDOW {
|
||||||
|
damage_window.remove(0);
|
||||||
|
}
|
||||||
|
did_skip_ahead = false;
|
||||||
|
pos += block_bytes;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Err(e2) => {
|
||||||
|
tracing::warn!(
|
||||||
|
target: "freemkv::disc",
|
||||||
|
phase = "transport_recovery_failed",
|
||||||
|
lba = block_lba,
|
||||||
|
single_sector_error = %e2,
|
||||||
|
"Single-sector read also failed after transport failure"
|
||||||
|
);
|
||||||
|
|
||||||
|
// If even single sector fails, mark the whole batch as unreadable and continue
|
||||||
|
let zero = vec![0u8; block_bytes as usize];
|
||||||
|
file.seek(SeekFrom::Start(pos))
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
file.write_all(&zero)
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
|
||||||
|
map.record(pos, block_bytes, mapfile::SectorStatus::Unreadable)
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
bytes_done = bytes_done.saturating_add(block_bytes);
|
||||||
|
|
||||||
|
damage_window.push(false);
|
||||||
|
if damage_window.len() > PASS1_DAMAGE_WINDOW {
|
||||||
|
damage_window.remove(0);
|
||||||
|
}
|
||||||
|
did_skip_ahead = false;
|
||||||
|
pos += block_bytes;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err.is_bridge_degradation() {
|
if err.is_bridge_degradation() {
|
||||||
@@ -1577,17 +1649,30 @@ impl Disc {
|
|||||||
lba = block_lba,
|
lba = block_lba,
|
||||||
degradation_count = bridge_degradation_count,
|
degradation_count = bridge_degradation_count,
|
||||||
error = %err,
|
error = %err,
|
||||||
"bridge degradation; cooling down 10s"
|
"bridge degradation (firmware stress); cooling down 15s"
|
||||||
);
|
);
|
||||||
std::thread::sleep(std::time::Duration::from_secs(
|
|
||||||
BRIDGE_DEGRADATION_COOLDOWN_SECS,
|
// Extended cooldown for bridge recovery - increased from 10s to 15s
|
||||||
));
|
std::thread::sleep(std::time::Duration::from_secs(15));
|
||||||
|
|
||||||
|
// Reduce batch size after degradation to be gentler on the bridge
|
||||||
|
let reduced_batch = (batch as u64 / 2).max(8);
|
||||||
|
tracing::debug!(
|
||||||
|
target: "freemkv::disc",
|
||||||
|
phase = "batch_reduction",
|
||||||
|
from_batch = batch,
|
||||||
|
to_batch = reduced_batch,
|
||||||
|
"Reducing batch size after bridge degradation"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Continue with the same sector to retry the read
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
target: "freemkv::disc",
|
target: "freemkv::disc",
|
||||||
phase = "bridge_degradation_exhausted",
|
phase = "bridge_degradation_exhausted",
|
||||||
lba = block_lba,
|
lba = block_lba,
|
||||||
|
error = %err,
|
||||||
"bridge degradation retries exhausted; treating as bad sector"
|
"bridge degradation retries exhausted; treating as bad sector"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -1616,6 +1701,90 @@ impl Disc {
|
|||||||
}
|
}
|
||||||
not_ready_retries = 0;
|
not_ready_retries = 0;
|
||||||
|
|
||||||
|
// For marginal errors (MEDIUM_ERROR, ABORTED_COMMAND), try smaller reads first
|
||||||
|
// before giving up on the whole batch. This prevents bridge crashes from
|
||||||
|
// forcing us to skip large sections of disc.
|
||||||
|
let should_try_smaller_reads = match sense_key {
|
||||||
|
crate::scsi::SENSE_KEY_MEDIUM_ERROR => true,
|
||||||
|
crate::scsi::SENSE_KEY_ABORTED_COMMAND => true,
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
|
||||||
|
if should_try_smaller_reads && block_count > 1 {
|
||||||
|
tracing::info!(
|
||||||
|
target: "freemkv::disc",
|
||||||
|
phase = "retry_with_smaller_read",
|
||||||
|
lba = block_lba,
|
||||||
|
original_sectors = block_count,
|
||||||
|
sense_key,
|
||||||
|
asc,
|
||||||
|
ascq,
|
||||||
|
"Marginal error detected; retrying with smaller reads"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Try reading the batch as individual sectors - this is gentler on the bridge
|
||||||
|
let mut all_failed = true;
|
||||||
|
for sector_offset in 0..block_count {
|
||||||
|
if let Some(ref h) = opts.halt {
|
||||||
|
if h.load(std::sync::atomic::Ordering::Relaxed) {
|
||||||
|
halt_requested = true;
|
||||||
|
break 'outer;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let sector_lba = block_lba + (sector_offset as u32);
|
||||||
|
let mut sector_buf = [0u8; 2048];
|
||||||
|
|
||||||
|
match reader.read_sectors(sector_lba, 1, &mut sector_buf[..], true) {
|
||||||
|
Ok(_) => {
|
||||||
|
all_failed = false;
|
||||||
|
// Write recovered sector
|
||||||
|
let write_pos = pos + (sector_offset as u64 * 2048);
|
||||||
|
file.seek(SeekFrom::Start(write_pos))
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
file.write_all(§or_buf)
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
|
||||||
|
// Mark this sector as finished in mapfile
|
||||||
|
map.record(write_pos, 2048, mapfile::SectorStatus::Finished)
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
// Mark failed sector as NonTrimmed
|
||||||
|
let write_pos = pos + (sector_offset as u64 * 2048);
|
||||||
|
file.seek(SeekFrom::Start(write_pos))
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
let zero = vec![0u8; 2048];
|
||||||
|
file.write_all(&zero)
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
|
||||||
|
map.record(write_pos, 2048, mapfile::SectorStatus::NonTrimmed)
|
||||||
|
.map_err(|e| Error::IoError { source: e })?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !all_failed {
|
||||||
|
tracing::info!(
|
||||||
|
target: "freemkv::disc",
|
||||||
|
phase = "smaller_read_partial_success",
|
||||||
|
lba = block_lba,
|
||||||
|
recovered_sectors = block_count - 1, // We count failures as NonTrimmed above
|
||||||
|
total_sectors = block_count,
|
||||||
|
"Partial recovery from smaller reads"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
bytes_done = bytes_done.saturating_add(block_bytes);
|
||||||
|
damage_window.push(false);
|
||||||
|
if damage_window.len() > PASS1_DAMAGE_WINDOW {
|
||||||
|
damage_window.remove(0);
|
||||||
|
}
|
||||||
|
did_skip_ahead = false;
|
||||||
|
pos += block_bytes;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
target: "freemkv::disc",
|
target: "freemkv::disc",
|
||||||
phase = "skip_ecc_block",
|
phase = "skip_ecc_block",
|
||||||
|
|||||||
Reference in New Issue
Block a user