fix: resume lookback from progress file — don't reset to 2d when queue is non-empty
This commit is contained in:
7
data/search_progress.json
Normal file
7
data/search_progress.json
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"lookback_days": 90,
|
||||||
|
"started_at": 1741220400000,
|
||||||
|
"completed": ["linkedin:Founding GTM"],
|
||||||
|
"pending": ["linkedin:Enterprise AE", "wellfound:Founding GTM"],
|
||||||
|
"stats:linkedin:Founding GTM": { "found": 913, "added": 814, "completed_at": 1741224000000 }
|
||||||
|
}
|
||||||
9
data/searcher_last_run.json
Normal file
9
data/searcher_last_run.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"started_at": 1772757505890,
|
||||||
|
"finished_at": null,
|
||||||
|
"finished": false,
|
||||||
|
"added": 40,
|
||||||
|
"seen": 146,
|
||||||
|
"skipped_dupes": 106,
|
||||||
|
"platforms": []
|
||||||
|
}
|
||||||
@@ -10,7 +10,7 @@ import { fileURLToPath } from 'url';
|
|||||||
const __dir = dirname(fileURLToPath(import.meta.url));
|
const __dir = dirname(fileURLToPath(import.meta.url));
|
||||||
|
|
||||||
import { addJobs, loadQueue, loadConfig } from './lib/queue.mjs';
|
import { addJobs, loadQueue, loadConfig } from './lib/queue.mjs';
|
||||||
import { writeFileSync } from 'fs';
|
import { writeFileSync, readFileSync, existsSync } from 'fs';
|
||||||
import { acquireLock } from './lib/lock.mjs';
|
import { acquireLock } from './lib/lock.mjs';
|
||||||
import { createBrowser } from './lib/browser.mjs';
|
import { createBrowser } from './lib/browser.mjs';
|
||||||
import { verifyLogin as liLogin, searchLinkedIn } from './lib/linkedin.mjs';
|
import { verifyLogin as liLogin, searchLinkedIn } from './lib/linkedin.mjs';
|
||||||
@@ -69,9 +69,19 @@ async function main() {
|
|||||||
console.log('');
|
console.log('');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Determine lookback: check for an in-progress run first, then fall back to first-run/normal logic
|
||||||
|
const savedProgress = existsSync(resolve(__dir, 'data/search_progress.json'))
|
||||||
|
? JSON.parse(readFileSync(resolve(__dir, 'data/search_progress.json'), 'utf8'))
|
||||||
|
: null;
|
||||||
const isFirstRun = loadQueue().length === 0;
|
const isFirstRun = loadQueue().length === 0;
|
||||||
const lookbackDays = isFirstRun ? (searchConfig.first_run_days || DEFAULT_FIRST_RUN_DAYS) : (searchConfig.posted_within_days || 2);
|
const lookbackDays = savedProgress?.lookback_days
|
||||||
if (isFirstRun) console.log(`📅 First run — looking back ${lookbackDays} days\n`);
|
|| (isFirstRun ? (searchConfig.first_run_days || DEFAULT_FIRST_RUN_DAYS) : (searchConfig.posted_within_days || 2));
|
||||||
|
|
||||||
|
if (savedProgress?.lookback_days) {
|
||||||
|
console.log(`🔁 Resuming ${lookbackDays}-day search run\n`);
|
||||||
|
} else if (isFirstRun) {
|
||||||
|
console.log(`📅 First run — looking back ${lookbackDays} days\n`);
|
||||||
|
}
|
||||||
|
|
||||||
// Init progress tracking — enables resume on restart
|
// Init progress tracking — enables resume on restart
|
||||||
initProgress(resolve(__dir, 'data'), lookbackDays);
|
initProgress(resolve(__dir, 'data'), lookbackDays);
|
||||||
|
|||||||
Reference in New Issue
Block a user