feat: search progress tracking — resume on restart, skip completed platform:track combinations

job_searcher.mjs

@@ -18,6 +18,7 @@ import { verifyLogin as wfLogin, searchWellfound } from './lib/wellfound.mjs';
 import { sendTelegram, formatSearchSummary } from './lib/notify.mjs';
 import { DEFAULT_FIRST_RUN_DAYS } from './lib/constants.mjs';
 import { generateKeywords } from './lib/keywords.mjs';
+import { initProgress, isCompleted, markComplete } from './lib/search_progress.mjs';
 
 async function main() {
   const lock = acquireLock('searcher', resolve(__dir, 'data'));
@@ -69,9 +70,12 @@ async function main() {
   }
 
   const isFirstRun = loadQueue().length === 0;
-  const lookbackDays = isFirstRun ? (searchConfig.first_run_days || DEFAULT_FIRST_RUN_DAYS) : null;
+  const lookbackDays = isFirstRun ? (searchConfig.first_run_days || DEFAULT_FIRST_RUN_DAYS) : (searchConfig.posted_within_days || 2);
   if (isFirstRun) console.log(`📅 First run — looking back ${lookbackDays} days\n`);
 
+  // Init progress tracking — enables resume on restart
+  initProgress(resolve(__dir, 'data'), lookbackDays);
+
   // Group searches by platform
   const liSearches = searchConfig.searches.filter(s => s.platforms?.includes('linkedin'));
   const wfSearches = searchConfig.searches.filter(s => s.platforms?.includes('wellfound'));
@@ -87,9 +91,11 @@ async function main() {
       console.log('  ✅ Logged in');
 
       for (const search of liSearches) {
-        const effectiveSearch = lookbackDays
-          ? { ...search, filters: { ...search.filters, posted_within_days: lookbackDays } }
-          : search;
+        if (isCompleted('linkedin', search.name)) {
+          console.log(`  [${search.name}] ✓ already done, skipping`);
+          continue;
+        }
+        const effectiveSearch = { ...search, filters: { ...search.filters, posted_within_days: lookbackDays } };
         let queryFound = 0, queryAdded = 0;
         await searchLinkedIn(liBrowser.page, effectiveSearch, {
           onPage: (pageJobs) => {
@@ -102,6 +108,7 @@ async function main() {
           }
         });
         console.log(`\r  [${search.name}] ${queryFound} found, ${queryAdded} new`);
+        markComplete('linkedin', search.name, { found: queryFound, added: queryAdded });
       }
 
       platformsRun.push('LinkedIn');
@@ -123,9 +130,11 @@ async function main() {
       else console.log('  ✅ Logged in');
 
       for (const search of wfSearches) {
-        const effectiveSearch = lookbackDays
-          ? { ...search, filters: { ...search.filters, posted_within_days: lookbackDays } }
-          : search;
+        if (isCompleted('wellfound', search.name)) {
+          console.log(`  [${search.name}] ✓ already done, skipping`);
+          continue;
+        }
+        const effectiveSearch = { ...search, filters: { ...search.filters, posted_within_days: lookbackDays } };
         let queryFound = 0, queryAdded = 0;
         await searchWellfound(wfBrowser.page, effectiveSearch, {
           onPage: (pageJobs) => {
@@ -138,6 +147,7 @@ async function main() {
           }
         });
         console.log(`\r  [${search.name}] ${queryFound} found, ${queryAdded} new`);
+        markComplete('wellfound', search.name, { found: queryFound, added: queryAdded });
       }
 
       platformsRun.push('Wellfound');

lib/search_progress.mjs

@@ -0,0 +1,61 @@
+/**
+ * search_progress.mjs — Track which searches have completed
+ * Enables resume on restart without re-running finished searches
+ */
+import { readFileSync, writeFileSync, existsSync, unlinkSync } from 'fs';
+
+let progressPath = null;
+let progress = null;
+
+export function initProgress(dataDir, lookbackDays) {
+  progressPath = `${dataDir}/search_progress.json`;
+
+  if (existsSync(progressPath)) {
+    const saved = JSON.parse(readFileSync(progressPath, 'utf8'));
+    // Only resume if same lookback window
+    if (saved.lookback_days === lookbackDays) {
+      progress = saved;
+      const done = progress.completed.length;
+      if (done > 0) {
+        console.log(`🔁 Resuming — skipping already-completed: ${progress.completed.join(', ')}\n`);
+      }
+      return progress;
+    }
+    console.log(`🆕 New lookback window (${lookbackDays}d), starting fresh\n`);
+  }
+
+  // Fresh start
+  progress = {
+    lookback_days: lookbackDays,
+    started_at: Date.now(),
+    completed: [],
+    pending: [],
+  };
+  save();
+  return progress;
+}
+
+export function isCompleted(platform, track) {
+  if (!progress) return false;
+  return progress.completed.includes(`${platform}:${track}`);
+}
+
+export function markComplete(platform, track, stats) {
+  if (!progress) return;
+  const key = `${platform}:${track}`;
+  progress.pending = progress.pending.filter(k => k !== key);
+  if (!progress.completed.includes(key)) progress.completed.push(key);
+  progress[`stats:${key}`] = { ...stats, completed_at: Date.now() };
+  save();
+}
+
+export function clearProgress() {
+  try { if (progressPath) unlinkSync(progressPath); } catch {}
+  progress = null;
+}
+
+function save() {
+  if (progressPath && progress) {
+    writeFileSync(progressPath, JSON.stringify(progress, null, 2));
+  }
+}