diff --git a/lib/linkedin.mjs b/lib/linkedin.mjs index 5fb8d5b..686ac47 100644 --- a/lib/linkedin.mjs +++ b/lib/linkedin.mjs @@ -26,8 +26,13 @@ export async function searchLinkedIn(page, search) { const url = `${BASE}/jobs/search/?${params.toString()}`; await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 25000 }); await page.waitForTimeout(3000); - await page.evaluate(() => window.scrollBy(0, 2000)); - await page.waitForTimeout(1500); + + // Paginate through all result pages + let pageNum = 0; + while (pageNum < 40) { // cap at 40 pages (1000 jobs) + // Scroll to load all cards on current page + await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)); + await page.waitForTimeout(1500); const found = await page.evaluate(({ track, excludes }) => { const ids = [...new Set( @@ -56,6 +61,14 @@ export async function searchLinkedIn(page, search) { }, { track: search.track, excludes: search.exclude_keywords || [] }); jobs.push(...found); + + // Click next page button + const nextBtn = await page.$('button[aria-label="View next page"]'); + if (!nextBtn) break; // no more pages + await nextBtn.click(); + await page.waitForTimeout(3000); + pageNum++; + } } // Dedupe by jobId diff --git a/lib/wellfound.mjs b/lib/wellfound.mjs index f67bdd8..e9f4a51 100644 --- a/lib/wellfound.mjs +++ b/lib/wellfound.mjs @@ -18,8 +18,16 @@ export async function searchWellfound(page, search) { const url = `https://wellfound.com/jobs?q=${encodeURIComponent(keyword)}&remote=true`; await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 }); await page.waitForTimeout(5000); - await page.evaluate(() => window.scrollBy(0, 3000)); - await page.waitForTimeout(2000); + + // Scroll to bottom repeatedly to trigger infinite scroll + let lastHeight = 0; + for (let i = 0; i < 10; i++) { + await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)); + await page.waitForTimeout(2000); + const newHeight = await page.evaluate(() => document.body.scrollHeight); + if (newHeight === lastHeight) break; + lastHeight = newHeight; + } const found = await page.evaluate(({ track, excludes }) => { const seen = new Set(); diff --git a/node_modules b/node_modules new file mode 120000 index 0000000..867c978 --- /dev/null +++ b/node_modules @@ -0,0 +1 @@ +/home/ubuntu/.openclaw/workspace/node_modules \ No newline at end of file