From b1528ac0add7681b28717afeaf9895a36e526461 Mon Sep 17 00:00:00 2001 From: Matthew Jackson Date: Fri, 6 Mar 2026 08:45:09 -0800 Subject: [PATCH] refactor: extract magic numbers to constants, fix audit issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Centralize all magic numbers/strings in lib/constants.mjs - Fix double-replaced import names in filter.mjs - Consolidate duplicate fs imports in job_applier/job_searcher - Remove empty JSDoc block in job_searcher - Update keywords.mjs model from claude-3-haiku to claude-haiku-4-5 - Extract Anthropic API URLs to constants - Convert :has-text() selectors to page.locator() API - Fix SIGTERM handler conflict — move partial-run notification into lock.onShutdown - Remove unused exports (LOCAL_USER_AGENT, DEFAULT_REVIEW_WINDOW_MINUTES) - Fix variable shadowing (b -> v) in job_filter reduce callback - Replace SKILL.md PM2 references with system cron Co-Authored-By: Claude Opus 4.6 --- SKILL.md | 30 +++--------------------------- job_applier.mjs | 3 +-- job_filter.mjs | 15 ++++++++------- job_searcher.mjs | 18 ++++++------------ lib/apply/wellfound.mjs | 4 ++-- lib/constants.mjs | 21 +++++++++++++++++++-- lib/filter.mjs | 16 ++++++++-------- lib/form_filler.mjs | 8 ++++---- lib/keywords.mjs | 6 ++++-- lib/linkedin.mjs | 8 +++----- lib/session.mjs | 12 +++++++----- lib/wellfound.mjs | 6 ++---- 12 files changed, 67 insertions(+), 80 deletions(-) diff --git a/SKILL.md b/SKILL.md index 91ec850..00415f6 100644 --- a/SKILL.md +++ b/SKILL.md @@ -92,40 +92,16 @@ Setup will: - Send a Telegram test message - Test LinkedIn + Wellfound logins -### 6. Schedule with PM2 - -PM2 is a Node.js process manager that runs the searcher and applier as proper system daemons — no SIGTERM issues, survives reboots. - -```bash -# Install PM2 -npm install -g pm2 - -# Start both jobs (searcher runs immediately + hourly; applier stopped by default) -pm2 start ecosystem.config.cjs -pm2 stop claw-applier # keep applier off until you're ready - -# Survive reboots -pm2 save -pm2 startup # follow the printed command (requires sudo) -``` - -PM2 manages the processes but **does not schedule them** — scheduling is handled by system cron. This ensures a running searcher is never killed mid-run. If it's already running, the cron invocation hits the lockfile and exits immediately. +### 6. Schedule with cron Add to crontab (`crontab -e`): + ``` 0 */12 * * * cd /path/to/claw-apply && node job_searcher.mjs >> /tmp/claw-searcher.log 2>&1 0 */6 * * * cd /path/to/claw-apply && node job_applier.mjs >> /tmp/claw-applier.log 2>&1 ``` -**PM2 cheatsheet:** -```bash -pm2 list # show all processes + status -pm2 logs claw-searcher # tail searcher logs -pm2 logs claw-applier # tail applier logs -pm2 start claw-searcher # run searcher now -pm2 start claw-applier # run applier now -pm2 stop claw-applier # stop applier mid-run -``` +The lockfile mechanism ensures only one instance runs at a time — if a searcher is already running, the cron invocation exits immediately. ### 7. Run manually diff --git a/job_applier.mjs b/job_applier.mjs index 25fbdce..cf51772 100644 --- a/job_applier.mjs +++ b/job_applier.mjs @@ -6,8 +6,7 @@ loadEnv(); // load .env before anything else * Reads jobs queue and applies using the appropriate handler per apply_type * Run via cron or manually: node job_applier.mjs [--preview] */ -import { existsSync } from 'fs'; -import { writeFileSync } from 'fs'; +import { existsSync, writeFileSync } from 'fs'; import { dirname, resolve } from 'path'; import { fileURLToPath } from 'url'; diff --git a/job_filter.mjs b/job_filter.mjs index c988f32..4443832 100644 --- a/job_filter.mjs +++ b/job_filter.mjs @@ -26,11 +26,11 @@ const __dir = dirname(fileURLToPath(import.meta.url)); import { getJobsByStatus, updateJobStatus, loadConfig, loadQueue, saveQueue, dedupeAfterFilter } from './lib/queue.mjs'; import { loadProfile, submitBatches, checkBatch, downloadResults } from './lib/filter.mjs'; import { sendTelegram } from './lib/notify.mjs'; +import { DEFAULT_FILTER_MODEL, DEFAULT_FILTER_MIN_SCORE } from './lib/constants.mjs'; const isStats = process.argv.includes('--stats'); const STATE_PATH = resolve(__dir, 'data/filter_state.json'); -const DEFAULT_MODEL = 'claude-sonnet-4-6-20251101'; // --------------------------------------------------------------------------- // State helpers @@ -69,9 +69,10 @@ function showStats() { const state = readState(); if (state) { - console.log(` Pending batch: ${state.batch_id}`); - console.log(` Submitted: ${state.submitted_at}`); - console.log(` Job count: ${state.job_count}\n`); + const batchIds = state.batches?.map(b => b.batchId).join(', ') || 'none'; + console.log(` Pending batches: ${batchIds}`); + console.log(` Submitted: ${state.submitted_at}`); + console.log(` Job count: ${state.job_count}\n`); } if (filtered.length > 0) { @@ -97,7 +98,7 @@ async function collect(state, settings) { b._status = status; b._counts = counts; if (status === 'in_progress') { - const total = Object.values(counts).reduce((a, b) => a + b, 0); + const total = Object.values(counts).reduce((a, v) => a + v, 0); const done = (counts.succeeded || 0) + (counts.errored || 0) + (counts.canceled || 0) + (counts.expired || 0); console.log(` [${b.track}] Still processing — ${done}/${total} complete`); allDone = false; @@ -124,7 +125,7 @@ async function collect(state, settings) { } const searchConfig = loadConfig(resolve(__dir, 'config/search_config.json')); - const globalMin = searchConfig.filter_min_score ?? 5; + const globalMin = searchConfig.filter_min_score ?? DEFAULT_FILTER_MIN_SCORE; let passed = 0, filtered = 0, errors = 0; const queue = loadQueue(); @@ -222,7 +223,7 @@ async function submit(settings, searchConfig, candidateProfile) { return; } - const model = settings.filter?.model || DEFAULT_MODEL; + const model = settings.filter?.model || DEFAULT_FILTER_MODEL; const submittedAt = new Date().toISOString(); console.log(`🚀 Submitting batches — ${filterable.length} jobs across ${Object.keys(jobProfilesByTrack).length} tracks, model: ${model}`); diff --git a/job_searcher.mjs b/job_searcher.mjs index 1f31aaa..9f02508 100644 --- a/job_searcher.mjs +++ b/job_searcher.mjs @@ -6,8 +6,7 @@ */ import { loadEnv } from './lib/env.mjs'; loadEnv(); // load .env before anything else -/** - */ + import { dirname, resolve } from 'path'; import { fileURLToPath } from 'url'; @@ -35,15 +34,6 @@ async function main() { const settings = loadConfig(resolve(__dir, 'config/settings.json')); - // Send notification even if SIGTERM kills the process mid-run - process.once('SIGTERM', async () => { - if (totalAdded > 0) { - const summary = formatSearchSummary(totalAdded, totalSeen - totalAdded, platformsRun.length ? platformsRun : ['LinkedIn']); - await sendTelegram(settings, summary + '\n_(partial run — timed out)_').catch(() => {}); - } - process.exit(0); - }); - const writeLastRun = (finished = false) => { const entry = { started_at: startedAt, @@ -68,9 +58,13 @@ async function main() { writeFileSync(runsPath, JSON.stringify(runs, null, 2)); }; - lock.onShutdown(() => { + lock.onShutdown(async () => { console.log(' Writing partial results to last-run file...'); writeLastRun(false); + if (totalAdded > 0) { + const summary = formatSearchSummary(totalAdded, totalSeen - totalAdded, platformsRun.length ? platformsRun : ['LinkedIn']); + await sendTelegram(settings, summary + '\n_(partial run — interrupted)_').catch(() => {}); + } }); // Load config diff --git a/lib/apply/wellfound.mjs b/lib/apply/wellfound.mjs index c978f4a..51c2e22 100644 --- a/lib/apply/wellfound.mjs +++ b/lib/apply/wellfound.mjs @@ -16,8 +16,8 @@ export async function apply(page, job, formFiller) { company: document.querySelector('[class*="company"] h2, [class*="startup"] h2, h2')?.textContent?.trim(), })); - const applyBtn = await page.$('a:has-text("Apply"), button:has-text("Apply Now"), a:has-text("Apply Now")'); - if (!applyBtn) return { status: 'no_button', meta }; + const applyBtn = page.locator('a:has-text("Apply"), button:has-text("Apply Now"), a:has-text("Apply Now")').first(); + if (await applyBtn.count() === 0) return { status: 'no_button', meta }; await applyBtn.click(); await page.waitForTimeout(FORM_FILL_WAIT); diff --git a/lib/constants.mjs b/lib/constants.mjs index 1436ef5..5c6d104 100644 --- a/lib/constants.mjs +++ b/lib/constants.mjs @@ -35,18 +35,36 @@ export const LINKEDIN_MAX_MODAL_STEPS = 12; export const WELLFOUND_BASE = 'https://wellfound.com'; // --- Browser --- -export const LOCAL_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'; export const KERNEL_SDK_PATH = '/home/ubuntu/.openclaw/workspace/node_modules/@onkernel/sdk/index.js'; export const DEFAULT_PLAYWRIGHT_PATH = '/home/ubuntu/.npm-global/lib/node_modules/playwright/index.mjs'; +// --- Search --- +export const LINKEDIN_MAX_SEARCH_PAGES = 40; +export const WELLFOUND_MAX_INFINITE_SCROLL = 10; +export const LINKEDIN_SECONDS_PER_DAY = 86400; + +// --- Session --- +export const SESSION_REFRESH_POLL_TIMEOUT = 30000; +export const SESSION_REFRESH_POLL_WAIT = 2000; +export const SESSION_LOGIN_VERIFY_WAIT = 3000; + // --- Form Filler Defaults --- export const DEFAULT_YEARS_EXPERIENCE = 7; export const DEFAULT_DESIRED_SALARY = 150000; export const MINIMUM_SALARY_FACTOR = 0.85; export const DEFAULT_SKILL_RATING = '8'; +export const FORM_PATTERN_MAX_LENGTH = 200; export const DEFAULT_FIRST_RUN_DAYS = 90; export const SEARCH_RESULTS_MAX = 30; +// --- Anthropic API --- +export const ANTHROPIC_API_URL = 'https://api.anthropic.com/v1/messages'; +export const ANTHROPIC_BATCH_API_URL = 'https://api.anthropic.com/v1/messages/batches'; +export const FILTER_DESC_MAX_CHARS = 800; +export const FILTER_BATCH_MAX_TOKENS = 1024; +export const DEFAULT_FILTER_MODEL = 'claude-sonnet-4-6-20251101'; +export const DEFAULT_FILTER_MIN_SCORE = 5; + // --- Notification --- export const TELEGRAM_API_BASE = 'https://api.telegram.org/bot'; export const NOTIFY_RATE_LIMIT_MS = 1500; @@ -68,5 +86,4 @@ export const EXTERNAL_ATS_PATTERNS = [ ]; // --- Queue --- -export const DEFAULT_REVIEW_WINDOW_MINUTES = 30; export const DEFAULT_MAX_RETRIES = 2; diff --git a/lib/filter.mjs b/lib/filter.mjs index e9e3f3f..77cb064 100644 --- a/lib/filter.mjs +++ b/lib/filter.mjs @@ -5,9 +5,9 @@ */ import { readFileSync, existsSync } from 'fs'; - -const DESC_MAX_CHARS = 800; -const BATCH_API = 'https://api.anthropic.com/v1/messages/batches'; +import { + ANTHROPIC_BATCH_API_URL, FILTER_DESC_MAX_CHARS, FILTER_BATCH_MAX_TOKENS +} from './constants.mjs'; // --------------------------------------------------------------------------- // Helpers @@ -43,7 +43,7 @@ function sanitize(str) { } function buildJobMessage(job) { - const desc = sanitize(job.description).substring(0, DESC_MAX_CHARS).replace(/\s+/g, ' ').trim(); + const desc = sanitize(job.description).substring(0, FILTER_DESC_MAX_CHARS).replace(/\s+/g, ' ').trim(); return `Title: ${sanitize(job.title)} Company: ${sanitize(job.company) || 'Unknown'} Location: ${sanitize(job.location) || 'Unknown'} @@ -99,14 +99,14 @@ export async function submitBatches(jobs, jobProfilesByTrack, candidateProfile, custom_id: customId, params: { model, - max_tokens: 1024, + max_tokens: FILTER_BATCH_MAX_TOKENS, system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }], messages: [{ role: 'user', content: buildJobMessage(job) }], } }); } - const res = await fetch(BATCH_API, { + const res = await fetch(ANTHROPIC_BATCH_API_URL, { method: 'POST', headers: apiHeaders(apiKey), body: JSON.stringify({ requests }), @@ -129,7 +129,7 @@ export async function submitBatches(jobs, jobProfilesByTrack, candidateProfile, * Check batch status. Returns { status: 'in_progress'|'ended', counts } */ export async function checkBatch(batchId, apiKey) { - const res = await fetch(`${BATCH_API}/${batchId}`, { + const res = await fetch(`${ANTHROPIC_BATCH_API_URL}/${batchId}`, { headers: apiHeaders(apiKey), }); @@ -155,7 +155,7 @@ const PRICING = { }; export async function downloadResults(batchId, apiKey, idMap = {}) { - const res = await fetch(`${BATCH_API}/${batchId}/results`, { + const res = await fetch(`${ANTHROPIC_BATCH_API_URL}/${batchId}/results`, { headers: apiHeaders(apiKey), }); diff --git a/lib/form_filler.mjs b/lib/form_filler.mjs index 694f941..74da384 100644 --- a/lib/form_filler.mjs +++ b/lib/form_filler.mjs @@ -6,7 +6,7 @@ import { DEFAULT_YEARS_EXPERIENCE, DEFAULT_DESIRED_SALARY, MINIMUM_SALARY_FACTOR, DEFAULT_SKILL_RATING, - LINKEDIN_EASY_APPLY_MODAL_SELECTOR + LINKEDIN_EASY_APPLY_MODAL_SELECTOR, FORM_PATTERN_MAX_LENGTH } from './constants.mjs'; export class FormFiller { @@ -23,7 +23,7 @@ export class FormFiller { // Check custom answers first (user-defined, pattern is substring or regex) for (const entry of this.answers) { try { - if (entry.pattern.length > 200) throw new Error('pattern too long'); + if (entry.pattern.length > FORM_PATTERN_MAX_LENGTH) throw new Error('pattern too long'); const re = new RegExp(entry.pattern, 'i'); if (re.test(l)) return String(entry.answer); } catch { @@ -180,8 +180,8 @@ export class FormFiller { if (anyChecked) continue; const answer = this.answerFor(leg); if (answer) { - const lbl = await fs.$(`label:has-text("${answer}")`); - if (lbl) await lbl.click().catch(() => {}); + const lbl = fs.locator(`label:has-text("${answer}")`).first(); + if (await lbl.count() > 0) await lbl.click().catch(() => {}); } else { unknown.push(leg); } diff --git a/lib/keywords.mjs b/lib/keywords.mjs index 19fb775..c186d1e 100644 --- a/lib/keywords.mjs +++ b/lib/keywords.mjs @@ -3,6 +3,8 @@ * One Claude call per search track using full profile + search config context */ +import { ANTHROPIC_API_URL } from './constants.mjs'; + export async function generateKeywords(search, profile, apiKey) { if (!apiKey) throw new Error('ANTHROPIC_API_KEY not set'); @@ -38,7 +40,7 @@ Think about: Return ONLY a JSON array of strings, no explanation, no markdown. Example format: ["query one", "query two", "query three"]`; - const res = await fetch('https://api.anthropic.com/v1/messages', { + const res = await fetch(ANTHROPIC_API_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -46,7 +48,7 @@ Example format: ["query one", "query two", "query three"]`; 'anthropic-version': '2023-06-01' }, body: JSON.stringify({ - model: 'claude-3-haiku-20240307', + model: 'claude-haiku-4-5-20251001', max_tokens: 1024, messages: [{ role: 'user', content: prompt }] }) diff --git a/lib/linkedin.mjs b/lib/linkedin.mjs index c14d959..353c30c 100644 --- a/lib/linkedin.mjs +++ b/lib/linkedin.mjs @@ -5,11 +5,9 @@ import { LINKEDIN_BASE, NAVIGATION_TIMEOUT, FEED_NAVIGATION_TIMEOUT, PAGE_LOAD_WAIT, SCROLL_WAIT, CLICK_WAIT, - EXTERNAL_ATS_PATTERNS + EXTERNAL_ATS_PATTERNS, LINKEDIN_LINKEDIN_MAX_SEARCH_PAGES, LINKEDIN_SECONDS_PER_DAY } from './constants.mjs'; -const MAX_SEARCH_PAGES = 40; - export async function verifyLogin(page) { await page.goto(`${LINKEDIN_BASE}/feed/`, { waitUntil: 'domcontentloaded', timeout: FEED_NAVIGATION_TIMEOUT }); await page.waitForTimeout(CLICK_WAIT); @@ -30,7 +28,7 @@ export async function searchLinkedIn(page, search, { onPage, onKeyword } = {}) { if (search.filters?.remote) params.set('f_WT', '2'); if (search.filters?.easy_apply_only) params.set('f_LF', 'f_AL'); if (search.filters?.posted_within_days) { - params.set('f_TPR', `r${search.filters.posted_within_days * 86400}`); + params.set('f_TPR', `r${search.filters.posted_within_days * LINKEDIN_SECONDS_PER_DAY}`); } const url = `${LINKEDIN_BASE}/jobs/search/?${params.toString()}`; @@ -43,7 +41,7 @@ export async function searchLinkedIn(page, search, { onPage, onKeyword } = {}) { await page.waitForTimeout(PAGE_LOAD_WAIT); let pageNum = 0; - while (pageNum < MAX_SEARCH_PAGES) { + while (pageNum < LINKEDIN_MAX_SEARCH_PAGES) { // Scroll to load all cards await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)); await page.waitForTimeout(SCROLL_WAIT); diff --git a/lib/session.mjs b/lib/session.mjs index a8d2924..ded6911 100644 --- a/lib/session.mjs +++ b/lib/session.mjs @@ -3,10 +3,12 @@ * Call refreshSession() before creating a browser to ensure the profile is fresh */ import { createRequire } from 'module'; +import { + KERNEL_SDK_PATH, SESSION_REFRESH_POLL_TIMEOUT, SESSION_REFRESH_POLL_WAIT, + SESSION_LOGIN_VERIFY_WAIT +} from './constants.mjs'; const require = createRequire(import.meta.url); -const KERNEL_SDK_PATH = '/home/ubuntu/.openclaw/workspace/node_modules/@onkernel/sdk/index.js'; - export async function refreshSession(platform, apiKey, connectionIds = {}) { const connectionId = connectionIds[platform]; if (!connectionId) throw new Error(`No Kernel connection ID configured for platform: ${platform} — add it to settings.json under kernel.connection_ids`); @@ -28,8 +30,8 @@ export async function refreshSession(platform, apiKey, connectionIds = {}) { console.log(` ⏳ ${platform} session pending (status: ${loginResp.status}), polling...`); const start = Date.now(); let pollCount = 0; - while (Date.now() - start < 30000) { - await new Promise(r => setTimeout(r, 2000)); + while (Date.now() - start < SESSION_REFRESH_POLL_TIMEOUT) { + await new Promise(r => setTimeout(r, SESSION_REFRESH_POLL_WAIT)); pollCount++; const conn = await kernel.auth.connections.retrieve(connectionId); if (conn.status === 'SUCCESS') { @@ -58,7 +60,7 @@ export async function ensureLoggedIn(page, verifyFn, platform, apiKey, connectio console.warn(` ⚠️ ${platform} not logged in (attempt ${attempt}), refreshing session...`); await refreshSession(platform, apiKey, connectionIds); await page.reload({ waitUntil: 'domcontentloaded' }); - await page.waitForTimeout(3000); + await page.waitForTimeout(SESSION_LOGIN_VERIFY_WAIT); } } return false; diff --git a/lib/wellfound.mjs b/lib/wellfound.mjs index 579c9dd..e3a67cf 100644 --- a/lib/wellfound.mjs +++ b/lib/wellfound.mjs @@ -5,11 +5,9 @@ import { WELLFOUND_BASE, NAVIGATION_TIMEOUT, SEARCH_NAVIGATION_TIMEOUT, SEARCH_LOAD_WAIT, SEARCH_SCROLL_WAIT, LOGIN_WAIT, - SEARCH_RESULTS_MAX + SEARCH_RESULTS_MAX, WELLFOUND_WELLFOUND_MAX_INFINITE_SCROLL } from './constants.mjs'; -const MAX_INFINITE_SCROLL = 10; - export async function verifyLogin(page) { await page.goto(`${WELLFOUND_BASE}/`, { waitUntil: 'domcontentloaded', timeout: NAVIGATION_TIMEOUT }); await page.waitForTimeout(LOGIN_WAIT); @@ -34,7 +32,7 @@ export async function searchWellfound(page, search, { onPage } = {}) { // Scroll to bottom repeatedly to trigger infinite scroll let lastHeight = 0; - for (let i = 0; i < MAX_INFINITE_SCROLL; i++) { + for (let i = 0; i < WELLFOUND_MAX_INFINITE_SCROLL; i++) { await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)); await page.waitForTimeout(SEARCH_SCROLL_WAIT); const newHeight = await page.evaluate(() => document.body.scrollHeight);