matthew/claw-apply
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: security/bug fixes, extract constants, remove magic values

Browse Source 
- Remove random suffix from Wellfound job IDs (broke dedup)
- Add null coalescing to all profile field returns in form_filler
- Fix honeypot case referencing nonexistent results.skipped counter
- Remove unused makeJobId import from linkedin.mjs
- Navigate directly to job URL instead of search+click in linkedin apply
- Add Telegram notification rate limiting (1.5s between sends)
- Replace Mode B blocking sleep with --preview flag
- Add max_applications_per_run enforcement
- Remove tracked search_config.json (keep .example.json only)
- Add search_config.json to .gitignore, fix duplicate node_modules entry
- Extract all magic numbers/strings to lib/constants.mjs

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Matthew Jackson
2026-03-05 15:54:23 -08:00
parent 8bc9af14c4
commit 47513e8cec
10 changed files with 214 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
lib/browser.mjs
View File

@@ -2,13 +2,15 @@
* browser.mjs — Browser factory
* Creates Kernel stealth browsers or falls back to local Playwright
*/
import { LOCAL_USER_AGENT, KERNEL_SDK_PATH, DEFAULT_PLAYWRIGHT_PATH } from './constants.mjs';
// Use configured playwright path or fall back to npm global
let _chromium;
async function getChromium(playwrightPath) {
if (_chromium) return _chromium;
const paths = [
playwrightPath,
'/home/ubuntu/.npm-global/lib/node_modules/playwright/index.mjs',
DEFAULT_PLAYWRIGHT_PATH,
'playwright'
].filter(Boolean);
for (const p of paths) {
@@ -39,7 +41,7 @@ export async function createBrowser(settings, profileKey) {
async function createKernelBrowser(kernelConfig, profileKey, playwrightPath) {
let Kernel;
try {
const mod = await import('/home/ubuntu/.openclaw/workspace/node_modules/@onkernel/sdk/index.js');
const mod = await import(KERNEL_SDK_PATH);
Kernel = mod.default;
} catch {
throw new Error('Kernel SDK not installed — run: npm install @onkernel/sdk');
@@ -66,9 +68,7 @@ async function createKernelBrowser(kernelConfig, profileKey, playwrightPath) {
async function createLocalBrowser(playwrightPath) {
const chromium = await getChromium(playwrightPath);
const browser = await chromium.launch({ headless: true });
const ctx = await browser.newContext({
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'
});
const ctx = await browser.newContext({ userAgent: LOCAL_USER_AGENT });
const page = await ctx.newPage();
return { browser, page, type: 'local' };
}

55
lib/constants.mjs Normal file
View File

@@ -0,0 +1,55 @@
/**
* constants.mjs — Shared constants for claw-apply
*/
// --- Timeouts (ms) ---
export const NAVIGATION_TIMEOUT = 25000;
export const SEARCH_NAVIGATION_TIMEOUT = 30000;
export const FEED_NAVIGATION_TIMEOUT = 20000;
export const PAGE_LOAD_WAIT = 3000;
export const SCROLL_WAIT = 1500;
export const SEARCH_LOAD_WAIT = 5000;
export const SEARCH_SCROLL_WAIT = 2000;
export const LOGIN_WAIT = 2000;
export const CLICK_WAIT = 1500;
export const MODAL_STEP_WAIT = 600;
export const SUBMIT_WAIT = 2500;
export const FORM_FILL_WAIT = 2500;
export const DISMISS_TIMEOUT = 3000;
export const APPLY_CLICK_TIMEOUT = 5000;
export const APPLY_BETWEEN_DELAY_BASE = 2000;
export const APPLY_BETWEEN_DELAY_WF_BASE = 1500;
export const APPLY_BETWEEN_DELAY_JITTER = 1000;
// --- LinkedIn ---
export const LINKEDIN_BASE = 'https://www.linkedin.com';
export const LINKEDIN_EASY_APPLY_MODAL_SELECTOR = '.jobs-easy-apply-modal';
export const LINKEDIN_APPLY_BUTTON_SELECTOR = 'button.jobs-apply-button';
export const LINKEDIN_SUBMIT_SELECTOR = 'button[aria-label="Submit application"]';
export const LINKEDIN_NEXT_SELECTOR = 'button[aria-label="Continue to next step"]';
export const LINKEDIN_REVIEW_SELECTOR = 'button[aria-label="Review your application"]';
export const LINKEDIN_DISMISS_SELECTOR = 'button[aria-label="Dismiss"]';
export const LINKEDIN_MAX_MODAL_STEPS = 12;
// --- Wellfound ---
export const WELLFOUND_BASE = 'https://wellfound.com';
// --- Browser ---
export const LOCAL_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36';
export const KERNEL_SDK_PATH = '/home/ubuntu/.openclaw/workspace/node_modules/@onkernel/sdk/index.js';
export const DEFAULT_PLAYWRIGHT_PATH = '/home/ubuntu/.npm-global/lib/node_modules/playwright/index.mjs';
// --- Form Filler Defaults ---
export const DEFAULT_YEARS_EXPERIENCE = 7;
export const DEFAULT_DESIRED_SALARY = 150000;
export const MINIMUM_SALARY_FACTOR = 0.85;
export const DEFAULT_SKILL_RATING = '8';
export const DEFAULT_FIRST_RUN_DAYS = 90;
export const SEARCH_RESULTS_MAX = 30;
// --- Notification ---
export const TELEGRAM_API_BASE = 'https://api.telegram.org/bot';
export const NOTIFY_RATE_LIMIT_MS = 1500;
// --- Queue ---
export const DEFAULT_REVIEW_WINDOW_MINUTES = 30;

39
lib/form_filler.mjs
View File

@@ -3,6 +3,11 @@
* Config-driven: answers loaded from answers.json
* Returns list of unknown required fields
*/
import {
DEFAULT_YEARS_EXPERIENCE, DEFAULT_DESIRED_SALARY,
MINIMUM_SALARY_FACTOR, DEFAULT_SKILL_RATING,
LINKEDIN_EASY_APPLY_MODAL_SELECTOR
} from './constants.mjs';
export class FormFiller {
constructor(profile, answers) {
@@ -29,17 +34,21 @@ export class FormFiller {
const p = this.profile;
// Contact
if (l.includes('first name') && !l.includes('last')) return p.name?.first;
if (l.includes('last name')) return p.name?.last;
if (l.includes('full name') || l === 'name') return `${p.name?.first} ${p.name?.last}`;
if (l.includes('email')) return p.email;
if (l.includes('phone') || l.includes('mobile')) return p.phone;
if (l.includes('city') && !l.includes('remote')) return p.location?.city;
if (l.includes('state') && !l.includes('statement')) return p.location?.state;
if (l.includes('zip') || l.includes('postal')) return p.location?.zip;
if (l.includes('country')) return p.location?.country;
if (l.includes('linkedin')) return p.linkedin_url;
if (l.includes('website') || l.includes('portfolio')) return p.linkedin_url;
if (l.includes('first name') && !l.includes('last')) return p.name?.first || null;
if (l.includes('last name')) return p.name?.last || null;
if (l.includes('full name') || l === 'name') {
const first = p.name?.first;
const last = p.name?.last;
return (first && last) ? `${first} ${last}` : null;
}
if (l.includes('email')) return p.email || null;
if (l.includes('phone') || l.includes('mobile')) return p.phone || null;
if (l.includes('city') && !l.includes('remote')) return p.location?.city || null;
if (l.includes('state') && !l.includes('statement')) return p.location?.state || null;
if (l.includes('zip') || l.includes('postal')) return p.location?.zip || null;
if (l.includes('country')) return p.location?.country || null;
if (l.includes('linkedin')) return p.linkedin_url || null;
if (l.includes('website') || l.includes('portfolio')) return p.linkedin_url || null;
// Work auth
if (l.includes('sponsor') || l.includes('visa')) return p.work_authorization?.requires_sponsorship ? 'Yes' : 'No';
@@ -57,7 +66,7 @@ export class FormFiller {
if (l.includes('sales') || l.includes('revenue') || l.includes('quota') || l.includes('account')) return '7';
if (l.includes('saas') || l.includes('software') || l.includes('tech')) return '7';
if (l.includes('manag') || l.includes('leadership')) return '3';
return String(p.years_experience || 7);
return String(p.years_experience || DEFAULT_YEARS_EXPERIENCE);
}
// 1-10 scale
@@ -67,12 +76,12 @@ export class FormFiller {
if (l.includes('enterprise') || l.includes('b2b')) return '9';
if (l.includes('technical') || l.includes('engineering')) return '7';
if (l.includes('crm') || l.includes('salesforce')) return '8';
return '8';
return DEFAULT_SKILL_RATING;
}
// Compensation
if (l.includes('salary') || l.includes('compensation') || l.includes('expected pay')) return String(p.desired_salary || '');
if (l.includes('minimum') && l.includes('salary')) return String(Math.round((p.desired_salary || 150000) * 0.85));
if (l.includes('minimum') && l.includes('salary')) return String(Math.round((p.desired_salary || DEFAULT_DESIRED_SALARY) * MINIMUM_SALARY_FACTOR));
// Dates
if (l.includes('start date') || l.includes('when can you start') || l.includes('available to start')) return 'Immediately';
@@ -111,7 +120,7 @@ export class FormFiller {
// Returns array of unknown required field labels
async fill(page, resumePath) {
const unknown = [];
const modal = await page.$('.jobs-easy-apply-modal') || page;
const modal = await page.$(LINKEDIN_EASY_APPLY_MODAL_SELECTOR) || page;
// Resume upload — only if no existing resume selected
const hasResumeSelected = await page.$('input[type="radio"][aria-label*="resume"], input[type="radio"][aria-label*="Resume"]').catch(() => null);

144
lib/linkedin.mjs
View File

@@ -1,13 +1,21 @@
/**
* linkedin.mjs — LinkedIn search and Easy Apply
*/
import { makeJobId } from './queue.mjs';
import {
LINKEDIN_BASE, NAVIGATION_TIMEOUT, FEED_NAVIGATION_TIMEOUT,
PAGE_LOAD_WAIT, SCROLL_WAIT, CLICK_WAIT, MODAL_STEP_WAIT,
SUBMIT_WAIT, DISMISS_TIMEOUT, APPLY_CLICK_TIMEOUT,
LINKEDIN_EASY_APPLY_MODAL_SELECTOR, LINKEDIN_APPLY_BUTTON_SELECTOR,
LINKEDIN_SUBMIT_SELECTOR, LINKEDIN_NEXT_SELECTOR,
LINKEDIN_REVIEW_SELECTOR, LINKEDIN_DISMISS_SELECTOR,
LINKEDIN_MAX_MODAL_STEPS
} from './constants.mjs';
const BASE = 'https://www.linkedin.com';
const MAX_SEARCH_PAGES = 40;
export async function verifyLogin(page) {
await page.goto(`${BASE}/feed/`, { waitUntil: 'domcontentloaded', timeout: 20000 });
await page.waitForTimeout(1500);
await page.goto(`${LINKEDIN_BASE}/feed/`, { waitUntil: 'domcontentloaded', timeout: FEED_NAVIGATION_TIMEOUT });
await page.waitForTimeout(CLICK_WAIT);
return page.url().includes('/feed');
}
@@ -23,50 +31,50 @@ export async function searchLinkedIn(page, search) {
params.set('f_TPR', `r${seconds}`);
}
const url = `${BASE}/jobs/search/?${params.toString()}`;
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 25000 });
await page.waitForTimeout(3000);
const url = `${LINKEDIN_BASE}/jobs/search/?${params.toString()}`;
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: NAVIGATION_TIMEOUT });
await page.waitForTimeout(PAGE_LOAD_WAIT);
// Paginate through all result pages
let pageNum = 0;
while (pageNum < 40) { // cap at 40 pages (1000 jobs)
while (pageNum < MAX_SEARCH_PAGES) {
// Scroll to load all cards on current page
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
await page.waitForTimeout(1500);
await page.waitForTimeout(SCROLL_WAIT);
const found = await page.evaluate(({ track, excludes }) => {
const ids = [...new Set(
Array.from(document.querySelectorAll('a[href*="/jobs/view/"]'))
.map(a => a.href.match(/\/jobs\/view\/(\d+)/)?.[1])
.filter(Boolean)
)];
const found = await page.evaluate(({ track, excludes }) => {
const ids = [...new Set(
Array.from(document.querySelectorAll('a[href*="/jobs/view/"]'))
.map(a => a.href.match(/\/jobs\/view\/(\d+)/)?.[1])
.filter(Boolean)
)];
return ids.map(id => {
const link = document.querySelector(`a[href*="/jobs/view/${id}"]`);
const container = link?.closest('li') || link?.parentElement;
const title = container?.querySelector('strong, [class*="title"], h3')?.textContent?.trim()
|| link?.textContent?.trim() || '';
const company = container?.querySelector('[class*="company"], [class*="subtitle"], h4')?.textContent?.trim() || '';
const location = container?.querySelector('[class*="location"]')?.textContent?.trim() || '';
return ids.map(id => {
const link = document.querySelector(`a[href*="/jobs/view/${id}"]`);
const container = link?.closest('li') || link?.parentElement;
const title = container?.querySelector('strong, [class*="title"], h3')?.textContent?.trim()
|| link?.textContent?.trim() || '';
const company = container?.querySelector('[class*="company"], [class*="subtitle"], h4')?.textContent?.trim() || '';
const location = container?.querySelector('[class*="location"]')?.textContent?.trim() || '';
const titleLower = title.toLowerCase();
const companyLower = company.toLowerCase();
for (const ex of excludes) {
if (titleLower.includes(ex.toLowerCase()) || companyLower.includes(ex.toLowerCase())) return null;
}
const titleLower = title.toLowerCase();
const companyLower = company.toLowerCase();
for (const ex of excludes) {
if (titleLower.includes(ex.toLowerCase()) || companyLower.includes(ex.toLowerCase())) return null;
}
return { id: `li_${id}`, platform: 'linkedin', track, title, company, location,
url: `https://www.linkedin.com/jobs/view/${id}/`, jobId: id };
}).filter(Boolean);
}, { track: search.track, excludes: search.exclude_keywords || [] });
return { id: `li_${id}`, platform: 'linkedin', track, title, company, location,
url: `https://www.linkedin.com/jobs/view/${id}/`, jobId: id };
}).filter(Boolean);
}, { track: search.track, excludes: search.exclude_keywords || [] });
jobs.push(...found);
jobs.push(...found);
// Click next page button
const nextBtn = await page.$('button[aria-label="View next page"]');
if (!nextBtn) break; // no more pages
if (!nextBtn) break;
await nextBtn.click();
await page.waitForTimeout(3000);
await page.waitForTimeout(PAGE_LOAD_WAIT);
pageNum++;
}
}
@@ -77,28 +85,9 @@ export async function searchLinkedIn(page, search) {
}
export async function applyLinkedIn(page, job, formFiller) {
// Navigate to search results with Easy Apply filter to get two-panel view
const params = new URLSearchParams({
keywords: job.title,
f_WT: '2',
f_LF: 'f_AL',
sortBy: 'DD'
});
await page.goto(`${BASE}/jobs/search/?${params.toString()}`, { waitUntil: 'domcontentloaded', timeout: 25000 });
await page.waitForTimeout(3000);
// Click the specific job by ID
const clicked = await page.evaluate((jobId) => {
const link = document.querySelector(`a[href*="/jobs/view/${jobId}"]`);
if (link) { link.click(); return true; }
return false;
}, job.jobId || job.url.match(/\/jobs\/view\/(\d+)/)?.[1]);
if (!clicked) {
// Direct navigation fallback
await page.goto(job.url, { waitUntil: 'domcontentloaded', timeout: 20000 });
}
await page.waitForTimeout(3000);
// Navigate directly to job page
await page.goto(job.url, { waitUntil: 'domcontentloaded', timeout: NAVIGATION_TIMEOUT });
await page.waitForTimeout(PAGE_LOAD_WAIT);
// Get title/company from detail panel
const meta = await page.evaluate(() => ({
@@ -107,8 +96,8 @@ export async function applyLinkedIn(page, job, formFiller) {
}));
// Detect apply type
const eaBtn = await page.$('button.jobs-apply-button[aria-label*="Easy Apply"]');
const externalBtn = await page.$('button.jobs-apply-button:not([aria-label*="Easy Apply"])');
const eaBtn = await page.$(`${LINKEDIN_APPLY_BUTTON_SELECTOR}[aria-label*="Easy Apply"]`);
const externalBtn = await page.$(`${LINKEDIN_APPLY_BUTTON_SELECTOR}:not([aria-label*="Easy Apply"])`);
const interestedBtn = await page.$('button[aria-label*="interested"], button:has-text("I\'m interested")');
if (!eaBtn && interestedBtn) return { status: 'skipped_recruiter_only', meta };
@@ -116,16 +105,16 @@ export async function applyLinkedIn(page, job, formFiller) {
if (!eaBtn) return { status: 'skipped_easy_apply_unsupported', meta };
// Click Easy Apply
await page.click('button.jobs-apply-button', { timeout: 5000 }).catch(() => {});
await page.waitForTimeout(1500);
await page.click(LINKEDIN_APPLY_BUTTON_SELECTOR, { timeout: APPLY_CLICK_TIMEOUT }).catch(() => {});
await page.waitForTimeout(CLICK_WAIT);
const modal = await page.$('.jobs-easy-apply-modal');
const modal = await page.$(LINKEDIN_EASY_APPLY_MODAL_SELECTOR);
if (!modal) return { status: 'no_modal', meta };
// Step through modal
let lastProgress = '-1';
for (let step = 0; step < 12; step++) {
const modalStillOpen = await page.$('.jobs-easy-apply-modal');
for (let step = 0; step < LINKEDIN_MAX_MODAL_STEPS; step++) {
const modalStillOpen = await page.$(LINKEDIN_EASY_APPLY_MODAL_SELECTOR);
if (!modalStillOpen) return { status: 'submitted', meta };
const progress = await page.$eval('[role="progressbar"]',
@@ -137,48 +126,47 @@ export async function applyLinkedIn(page, job, formFiller) {
// Honeypot?
if (unknowns[0]?.honeypot) {
await page.click('button[aria-label="Dismiss"]', { timeout: 3000 }).catch(() => {});
await page.click(LINKEDIN_DISMISS_SELECTOR, { timeout: DISMISS_TIMEOUT }).catch(() => {});
return { status: 'skipped_honeypot', meta };
}
// Has unknown required fields?
if (unknowns.length > 0) {
// Return first unknown question for user to answer
const question = unknowns[0];
await page.click('button[aria-label="Dismiss"]', { timeout: 3000 }).catch(() => {});
await page.click(LINKEDIN_DISMISS_SELECTOR, { timeout: DISMISS_TIMEOUT }).catch(() => {});
return { status: 'needs_answer', pending_question: question, meta };
}
await page.waitForTimeout(600);
await page.waitForTimeout(MODAL_STEP_WAIT);
// Submit?
const hasSubmit = await page.$('button[aria-label="Submit application"]');
const hasSubmit = await page.$(LINKEDIN_SUBMIT_SELECTOR);
if (hasSubmit) {
await page.click('button[aria-label="Submit application"]', { timeout: 5000 });
await page.waitForTimeout(2500);
await page.click(LINKEDIN_SUBMIT_SELECTOR, { timeout: APPLY_CLICK_TIMEOUT });
await page.waitForTimeout(SUBMIT_WAIT);
return { status: 'submitted', meta };
}
// Stuck?
if (progress === lastProgress && step > 2) {
await page.click('button[aria-label="Dismiss"]', { timeout: 3000 }).catch(() => {});
await page.click(LINKEDIN_DISMISS_SELECTOR, { timeout: DISMISS_TIMEOUT }).catch(() => {});
return { status: 'stuck', meta };
}
// Next/Continue?
const hasNext = await page.$('button[aria-label="Continue to next step"]');
const hasNext = await page.$(LINKEDIN_NEXT_SELECTOR);
if (hasNext) {
await page.click('button[aria-label="Continue to next step"]', { timeout: 5000 }).catch(() => {});
await page.waitForTimeout(1500);
await page.click(LINKEDIN_NEXT_SELECTOR, { timeout: APPLY_CLICK_TIMEOUT }).catch(() => {});
await page.waitForTimeout(CLICK_WAIT);
lastProgress = progress;
continue;
}
// Review?
const hasReview = await page.$('button[aria-label="Review your application"]');
const hasReview = await page.$(LINKEDIN_REVIEW_SELECTOR);
if (hasReview) {
await page.click('button[aria-label="Review your application"]', { timeout: 5000 }).catch(() => {});
await page.waitForTimeout(1500);
await page.click(LINKEDIN_REVIEW_SELECTOR, { timeout: APPLY_CLICK_TIMEOUT }).catch(() => {});
await page.waitForTimeout(CLICK_WAIT);
lastProgress = progress;
continue;
}
@@ -186,6 +174,6 @@ export async function applyLinkedIn(page, job, formFiller) {
break;
}
await page.click('button[aria-label="Dismiss"]', { timeout: 3000 }).catch(() => {});
await page.click(LINKEDIN_DISMISS_SELECTOR, { timeout: DISMISS_TIMEOUT }).catch(() => {});
return { status: 'incomplete', meta };
}

13
lib/notify.mjs
View File

@@ -2,6 +2,9 @@
* notify.mjs — Telegram notifications
* Sends messages directly via Telegram Bot API
*/
import { TELEGRAM_API_BASE, NOTIFY_RATE_LIMIT_MS } from './constants.mjs';
let lastSentAt = 0;
export async function sendTelegram(settings, message) {
const { bot_token, telegram_user_id } = settings.notifications;
@@ -10,7 +13,14 @@ export async function sendTelegram(settings, message) {
return;
}
const url = `https://api.telegram.org/bot${bot_token}/sendMessage`;
// Rate limit to avoid Telegram API throttling
const now = Date.now();
const elapsed = now - lastSentAt;
if (elapsed < NOTIFY_RATE_LIMIT_MS) {
await new Promise(r => setTimeout(r, NOTIFY_RATE_LIMIT_MS - elapsed));
}
const url = `${TELEGRAM_API_BASE}${bot_token}/sendMessage`;
try {
const res = await fetch(url, {
method: 'POST',
@@ -21,6 +31,7 @@ export async function sendTelegram(settings, message) {
parse_mode: 'Markdown',
}),
});
lastSentAt = Date.now();
const data = await res.json();
if (!data.ok) console.error('[notify] Telegram error:', data.description);
} catch (e) {

39
lib/wellfound.mjs
View File

@@ -1,10 +1,17 @@
/**
* wellfound.mjs — Wellfound search and apply
*/
import {
WELLFOUND_BASE, NAVIGATION_TIMEOUT, SEARCH_NAVIGATION_TIMEOUT,
SEARCH_LOAD_WAIT, SEARCH_SCROLL_WAIT, LOGIN_WAIT, PAGE_LOAD_WAIT,
FORM_FILL_WAIT, SUBMIT_WAIT, SEARCH_RESULTS_MAX
} from './constants.mjs';
const MAX_INFINITE_SCROLL = 10;
export async function verifyLogin(page) {
await page.goto('https://wellfound.com/', { waitUntil: 'domcontentloaded', timeout: 25000 });
await page.waitForTimeout(2000);
await page.goto(`${WELLFOUND_BASE}/`, { waitUntil: 'domcontentloaded', timeout: NAVIGATION_TIMEOUT });
await page.waitForTimeout(LOGIN_WAIT);
const loggedIn = await page.evaluate(() =>
document.body.innerText.includes('Applied') || document.body.innerText.includes('Open to offers')
);
@@ -15,21 +22,21 @@ export async function searchWellfound(page, search) {
const jobs = [];
for (const keyword of search.keywords) {
const url = `https://wellfound.com/jobs?q=${encodeURIComponent(keyword)}&remote=true`;
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
await page.waitForTimeout(5000);
const url = `${WELLFOUND_BASE}/jobs?q=${encodeURIComponent(keyword)}&remote=true`;
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: SEARCH_NAVIGATION_TIMEOUT });
await page.waitForTimeout(SEARCH_LOAD_WAIT);
// Scroll to bottom repeatedly to trigger infinite scroll
let lastHeight = 0;
for (let i = 0; i < 10; i++) {
for (let i = 0; i < MAX_INFINITE_SCROLL; i++) {
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
await page.waitForTimeout(2000);
await page.waitForTimeout(SEARCH_SCROLL_WAIT);
const newHeight = await page.evaluate(() => document.body.scrollHeight);
if (newHeight === lastHeight) break;
lastHeight = newHeight;
}
const found = await page.evaluate(({ track, excludes }) => {
const found = await page.evaluate(({ track, excludes, maxResults }) => {
const seen = new Set();
const results = [];
@@ -53,8 +60,10 @@ export async function searchWellfound(page, search) {
}
if (title.length > 3) {
// Deterministic ID from URL path
const slug = href.split('/').pop().split('?')[0];
results.push({
id: `wf_${href.split('/').pop().split('?')[0]}_${Math.random().toString(36).slice(2,6)}`,
id: `wf_${slug}`,
platform: 'wellfound',
track,
title,
@@ -64,8 +73,8 @@ export async function searchWellfound(page, search) {
}
});
return results.slice(0, 30);
}, { track: search.track, excludes: search.exclude_keywords || [] });
return results.slice(0, maxResults);
}, { track: search.track, excludes: search.exclude_keywords || [], maxResults: SEARCH_RESULTS_MAX });
jobs.push(...found);
}
@@ -76,8 +85,8 @@ export async function searchWellfound(page, search) {
}
export async function applyWellfound(page, job, formFiller) {
await page.goto(job.url, { waitUntil: 'domcontentloaded', timeout: 25000 });
await page.waitForTimeout(3000);
await page.goto(job.url, { waitUntil: 'domcontentloaded', timeout: NAVIGATION_TIMEOUT });
await page.waitForTimeout(PAGE_LOAD_WAIT);
const meta = await page.evaluate(() => ({
title: document.querySelector('h1')?.textContent?.trim(),
@@ -88,7 +97,7 @@ export async function applyWellfound(page, job, formFiller) {
if (!applyBtn) return { status: 'no_button', meta };
await applyBtn.click();
await page.waitForTimeout(2500);
await page.waitForTimeout(FORM_FILL_WAIT);
// Fill form
const unknowns = await formFiller.fill(page, formFiller.profile.resume_path);
@@ -100,7 +109,7 @@ export async function applyWellfound(page, job, formFiller) {
if (!submitBtn) return { status: 'no_submit', meta };
await submitBtn.click();
await page.waitForTimeout(2000);
await page.waitForTimeout(SUBMIT_WAIT);
return { status: 'submitted', meta };
}