From f9fa36b47c2267c10ae7a41361fbf46f52898ae3 Mon Sep 17 00:00:00 2001
From: Claw <matthew@pq.io>
Date: Fri, 6 Mar 2026 00:17:33 +0000
Subject: [PATCH] =?UTF-8?q?feat:=20flush=20jobs=20to=20queue=20per-page=20?=
 =?UTF-8?q?=E2=80=94=20no=20data=20loss=20on=20crash;=20live=20progress=20?=
 =?UTF-8?q?output?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .gitignore        |  1 +
 job_searcher.mjs  | 34 ++++++++++++++++++++++++----------
 lib/linkedin.mjs  |  3 ++-
 lib/wellfound.mjs |  3 ++-
 4 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/.gitignore b/.gitignore
index fe57f11..a43f994 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@ config/answers.json
 config/search_config.json
 
 # Templates are committed instead (see config/*.example.json)
+data/*.lock
diff --git a/job_searcher.mjs b/job_searcher.mjs
index 158dd09..00366ce 100644
--- a/job_searcher.mjs
+++ b/job_searcher.mjs
@@ -72,11 +72,18 @@ async function main() {
         const effectiveSearch = lookbackDays
           ? { ...search, filters: { ...search.filters, posted_within_days: lookbackDays } }
           : search;
-        const jobs = await searchLinkedIn(liBrowser.page, effectiveSearch);
-        const added = addJobs(jobs);
-        totalAdded += added;
-        totalSeen += jobs.length;
-        console.log(`  [${search.name}] ${jobs.length} found, ${added} new`);
+        let queryFound = 0, queryAdded = 0;
+        await searchLinkedIn(liBrowser.page, effectiveSearch, {
+          onPage: (pageJobs) => {
+            const added = addJobs(pageJobs);
+            totalAdded += added;
+            totalSeen += pageJobs.length;
+            queryFound += pageJobs.length;
+            queryAdded += added;
+            process.stdout.write(`\r  [${search.name}] ${queryFound} found, ${queryAdded} new...`);
+          }
+        });
+        console.log(`\r  [${search.name}] ${queryFound} found, ${queryAdded} new`);
       }
 
       platformsRun.push('LinkedIn');
@@ -101,11 +108,18 @@ async function main() {
         const effectiveSearch = lookbackDays
           ? { ...search, filters: { ...search.filters, posted_within_days: lookbackDays } }
           : search;
-        const jobs = await searchWellfound(wfBrowser.page, effectiveSearch);
-        const added = addJobs(jobs);
-        totalAdded += added;
-        totalSeen += jobs.length;
-        console.log(`  [${search.name}] ${jobs.length} found, ${added} new`);
+        let queryFound = 0, queryAdded = 0;
+        await searchWellfound(wfBrowser.page, effectiveSearch, {
+          onPage: (pageJobs) => {
+            const added = addJobs(pageJobs);
+            totalAdded += added;
+            totalSeen += pageJobs.length;
+            queryFound += pageJobs.length;
+            queryAdded += added;
+            process.stdout.write(`\r  [${search.name}] ${queryFound} found, ${queryAdded} new...`);
+          }
+        });
+        console.log(`\r  [${search.name}] ${queryFound} found, ${queryAdded} new`);
       }
 
       platformsRun.push('Wellfound');
diff --git a/lib/linkedin.mjs b/lib/linkedin.mjs
index 94000ef..1e72b13 100644
--- a/lib/linkedin.mjs
+++ b/lib/linkedin.mjs
@@ -19,7 +19,7 @@ export async function verifyLogin(page) {
   return page.url().includes('/feed');
 }
 
-export async function searchLinkedIn(page, search) {
+export async function searchLinkedIn(page, search, { onPage } = {}) {
   const jobs = [];
 
   for (const keyword of search.keywords) {
@@ -69,6 +69,7 @@ export async function searchLinkedIn(page, search) {
       }, { track: search.track, excludes: search.exclude_keywords || [] });
 
       jobs.push(...found);
+      if (found.length > 0 && onPage) onPage(found);
 
       // Click next page button
       const nextBtn = await page.$('button[aria-label="View next page"]');
diff --git a/lib/wellfound.mjs b/lib/wellfound.mjs
index 7e40bd0..27b3fb7 100644
--- a/lib/wellfound.mjs
+++ b/lib/wellfound.mjs
@@ -18,7 +18,7 @@ export async function verifyLogin(page) {
   return loggedIn;
 }
 
-export async function searchWellfound(page, search) {
+export async function searchWellfound(page, search, { onPage } = {}) {
   const jobs = [];
 
   for (const keyword of search.keywords) {
@@ -77,6 +77,7 @@ export async function searchWellfound(page, search) {
     }, { track: search.track, excludes: search.exclude_keywords || [], maxResults: SEARCH_RESULTS_MAX });
 
     jobs.push(...found);
+    if (found.length > 0 && onPage) onPage(found);
   }
 
   // Dedupe by URL