From d610060dbbb949dc321ea52e8620aa2bb4a609d6 Mon Sep 17 00:00:00 2001
From: Claw <matthew@pq.io>
Date: Fri, 6 Mar 2026 10:16:06 +0000
Subject: [PATCH] feat: persistent run history logs for searcher and filter
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- search_runs.json: append-only history of every searcher run
  (started_at, finished, added, seen, platforms, lookback_days)
- search_progress_last.json: snapshot of final progress state after
  each completed run — answers 'what keywords/tracks were searched?'
- filter_runs.json: append-only history of every filter batch
  (batch_id, submitted/collected timestamps, model, passed/filtered/errors)
Fixes the 'did the 90-day run complete?' ambiguity going forward
---
 job_filter.mjs   | 19 ++++++++++++++++++-
 job_searcher.mjs | 21 +++++++++++++++++++--
 2 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/job_filter.mjs b/job_filter.mjs
index fc39fd9..bb18b53 100644
--- a/job_filter.mjs
+++ b/job_filter.mjs
@@ -133,6 +133,21 @@ async function collect(state, settings) {
 
   clearState();
 
+  // Append to filter run history
+  const runsPath = resolve(__dir, 'data/filter_runs.json');
+  const runs = existsSync(runsPath) ? JSON.parse(readFileSync(runsPath, 'utf8')) : [];
+  runs.push({
+    batch_id: state.batch_id,
+    submitted_at: state.submitted_at,
+    collected_at: new Date().toISOString(),
+    job_count: state.job_count,
+    model: state.model,
+    passed,
+    filtered,
+    errors,
+  });
+  writeFileSync(runsPath, JSON.stringify(runs, null, 2));
+
   const summary = `✅ Filter complete — ${passed} passed, ${filtered} filtered, ${errors} errors`;
   console.log(`\n${summary}`);
 
@@ -182,11 +197,13 @@ async function submit(settings, searchConfig, candidateProfile) {
 
   const batchId = await submitBatch(filterable, jobProfilesByTrack, searchConfig, candidateProfile, model, apiKey);
 
+  const submittedAt = new Date().toISOString();
   writeState({
     batch_id: batchId,
-    submitted_at: new Date().toISOString(),
+    submitted_at: submittedAt,
     job_count: filterable.length,
     model,
+    tracks: Object.keys(jobProfilesByTrack),
   });
 
   console.log(`  Batch submitted: ${batchId}`);
diff --git a/job_searcher.mjs b/job_searcher.mjs
index 0b18dd1..4559efa 100644
--- a/job_searcher.mjs
+++ b/job_searcher.mjs
@@ -34,7 +34,7 @@ async function main() {
   const startedAt = Date.now();
 
   const writeLastRun = (finished = false) => {
-    writeFileSync(resolve(__dir, 'data/searcher_last_run.json'), JSON.stringify({
+    const entry = {
       started_at: startedAt,
       finished_at: finished ? Date.now() : null,
       finished,
@@ -42,7 +42,19 @@ async function main() {
       seen: totalSeen,
       skipped_dupes: totalSeen - totalAdded,
       platforms: platformsRun,
-    }, null, 2));
+    };
+    // Always update last-run snapshot
+    writeFileSync(resolve(__dir, 'data/searcher_last_run.json'), JSON.stringify(entry, null, 2));
+    // Append to run history log
+    const runsPath = resolve(__dir, 'data/search_runs.json');
+    const runs = existsSync(runsPath) ? JSON.parse(readFileSync(runsPath, 'utf8')) : [];
+    // Update last entry if same run, otherwise append
+    if (runs.length > 0 && runs[runs.length - 1].started_at === startedAt) {
+      runs[runs.length - 1] = entry;
+    } else {
+      runs.push(entry);
+    }
+    writeFileSync(runsPath, JSON.stringify(runs, null, 2));
   };
 
   lock.onShutdown(() => {
@@ -195,6 +207,11 @@ async function main() {
   if (totalAdded > 0) await sendTelegram(settings, summary);
 
   writeLastRun(true);
+  // Archive final progress snapshot before clearing (for audit — answers "what was searched?")
+  const progressPath = resolve(__dir, 'data/search_progress.json');
+  if (existsSync(progressPath)) {
+    writeFileSync(resolve(__dir, 'data/search_progress_last.json'), readFileSync(progressPath, 'utf8'));
+  }
   clearProgress(); // run finished cleanly — next run starts fresh with new keywords
 
   console.log('\n✅ Search complete');