refactor: extract magic numbers to constants, fix audit issues
- Centralize all magic numbers/strings in lib/constants.mjs - Fix double-replaced import names in filter.mjs - Consolidate duplicate fs imports in job_applier/job_searcher - Remove empty JSDoc block in job_searcher - Update keywords.mjs model from claude-3-haiku to claude-haiku-4-5 - Extract Anthropic API URLs to constants - Convert :has-text() selectors to page.locator() API - Fix SIGTERM handler conflict — move partial-run notification into lock.onShutdown - Remove unused exports (LOCAL_USER_AGENT, DEFAULT_REVIEW_WINDOW_MINUTES) - Fix variable shadowing (b -> v) in job_filter reduce callback - Replace SKILL.md PM2 references with system cron Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
30
SKILL.md
30
SKILL.md
@@ -92,40 +92,16 @@ Setup will:
|
|||||||
- Send a Telegram test message
|
- Send a Telegram test message
|
||||||
- Test LinkedIn + Wellfound logins
|
- Test LinkedIn + Wellfound logins
|
||||||
|
|
||||||
### 6. Schedule with PM2
|
### 6. Schedule with cron
|
||||||
|
|
||||||
PM2 is a Node.js process manager that runs the searcher and applier as proper system daemons — no SIGTERM issues, survives reboots.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Install PM2
|
|
||||||
npm install -g pm2
|
|
||||||
|
|
||||||
# Start both jobs (searcher runs immediately + hourly; applier stopped by default)
|
|
||||||
pm2 start ecosystem.config.cjs
|
|
||||||
pm2 stop claw-applier # keep applier off until you're ready
|
|
||||||
|
|
||||||
# Survive reboots
|
|
||||||
pm2 save
|
|
||||||
pm2 startup # follow the printed command (requires sudo)
|
|
||||||
```
|
|
||||||
|
|
||||||
PM2 manages the processes but **does not schedule them** — scheduling is handled by system cron. This ensures a running searcher is never killed mid-run. If it's already running, the cron invocation hits the lockfile and exits immediately.
|
|
||||||
|
|
||||||
Add to crontab (`crontab -e`):
|
Add to crontab (`crontab -e`):
|
||||||
|
|
||||||
```
|
```
|
||||||
0 */12 * * * cd /path/to/claw-apply && node job_searcher.mjs >> /tmp/claw-searcher.log 2>&1
|
0 */12 * * * cd /path/to/claw-apply && node job_searcher.mjs >> /tmp/claw-searcher.log 2>&1
|
||||||
0 */6 * * * cd /path/to/claw-apply && node job_applier.mjs >> /tmp/claw-applier.log 2>&1
|
0 */6 * * * cd /path/to/claw-apply && node job_applier.mjs >> /tmp/claw-applier.log 2>&1
|
||||||
```
|
```
|
||||||
|
|
||||||
**PM2 cheatsheet:**
|
The lockfile mechanism ensures only one instance runs at a time — if a searcher is already running, the cron invocation exits immediately.
|
||||||
```bash
|
|
||||||
pm2 list # show all processes + status
|
|
||||||
pm2 logs claw-searcher # tail searcher logs
|
|
||||||
pm2 logs claw-applier # tail applier logs
|
|
||||||
pm2 start claw-searcher # run searcher now
|
|
||||||
pm2 start claw-applier # run applier now
|
|
||||||
pm2 stop claw-applier # stop applier mid-run
|
|
||||||
```
|
|
||||||
|
|
||||||
### 7. Run manually
|
### 7. Run manually
|
||||||
|
|
||||||
|
|||||||
@@ -6,8 +6,7 @@ loadEnv(); // load .env before anything else
|
|||||||
* Reads jobs queue and applies using the appropriate handler per apply_type
|
* Reads jobs queue and applies using the appropriate handler per apply_type
|
||||||
* Run via cron or manually: node job_applier.mjs [--preview]
|
* Run via cron or manually: node job_applier.mjs [--preview]
|
||||||
*/
|
*/
|
||||||
import { existsSync } from 'fs';
|
import { existsSync, writeFileSync } from 'fs';
|
||||||
import { writeFileSync } from 'fs';
|
|
||||||
import { dirname, resolve } from 'path';
|
import { dirname, resolve } from 'path';
|
||||||
import { fileURLToPath } from 'url';
|
import { fileURLToPath } from 'url';
|
||||||
|
|
||||||
|
|||||||
@@ -26,11 +26,11 @@ const __dir = dirname(fileURLToPath(import.meta.url));
|
|||||||
import { getJobsByStatus, updateJobStatus, loadConfig, loadQueue, saveQueue, dedupeAfterFilter } from './lib/queue.mjs';
|
import { getJobsByStatus, updateJobStatus, loadConfig, loadQueue, saveQueue, dedupeAfterFilter } from './lib/queue.mjs';
|
||||||
import { loadProfile, submitBatches, checkBatch, downloadResults } from './lib/filter.mjs';
|
import { loadProfile, submitBatches, checkBatch, downloadResults } from './lib/filter.mjs';
|
||||||
import { sendTelegram } from './lib/notify.mjs';
|
import { sendTelegram } from './lib/notify.mjs';
|
||||||
|
import { DEFAULT_FILTER_MODEL, DEFAULT_FILTER_MIN_SCORE } from './lib/constants.mjs';
|
||||||
|
|
||||||
const isStats = process.argv.includes('--stats');
|
const isStats = process.argv.includes('--stats');
|
||||||
|
|
||||||
const STATE_PATH = resolve(__dir, 'data/filter_state.json');
|
const STATE_PATH = resolve(__dir, 'data/filter_state.json');
|
||||||
const DEFAULT_MODEL = 'claude-sonnet-4-6-20251101';
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// State helpers
|
// State helpers
|
||||||
@@ -69,9 +69,10 @@ function showStats() {
|
|||||||
|
|
||||||
const state = readState();
|
const state = readState();
|
||||||
if (state) {
|
if (state) {
|
||||||
console.log(` Pending batch: ${state.batch_id}`);
|
const batchIds = state.batches?.map(b => b.batchId).join(', ') || 'none';
|
||||||
console.log(` Submitted: ${state.submitted_at}`);
|
console.log(` Pending batches: ${batchIds}`);
|
||||||
console.log(` Job count: ${state.job_count}\n`);
|
console.log(` Submitted: ${state.submitted_at}`);
|
||||||
|
console.log(` Job count: ${state.job_count}\n`);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (filtered.length > 0) {
|
if (filtered.length > 0) {
|
||||||
@@ -97,7 +98,7 @@ async function collect(state, settings) {
|
|||||||
b._status = status;
|
b._status = status;
|
||||||
b._counts = counts;
|
b._counts = counts;
|
||||||
if (status === 'in_progress') {
|
if (status === 'in_progress') {
|
||||||
const total = Object.values(counts).reduce((a, b) => a + b, 0);
|
const total = Object.values(counts).reduce((a, v) => a + v, 0);
|
||||||
const done = (counts.succeeded || 0) + (counts.errored || 0) + (counts.canceled || 0) + (counts.expired || 0);
|
const done = (counts.succeeded || 0) + (counts.errored || 0) + (counts.canceled || 0) + (counts.expired || 0);
|
||||||
console.log(` [${b.track}] Still processing — ${done}/${total} complete`);
|
console.log(` [${b.track}] Still processing — ${done}/${total} complete`);
|
||||||
allDone = false;
|
allDone = false;
|
||||||
@@ -124,7 +125,7 @@ async function collect(state, settings) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const searchConfig = loadConfig(resolve(__dir, 'config/search_config.json'));
|
const searchConfig = loadConfig(resolve(__dir, 'config/search_config.json'));
|
||||||
const globalMin = searchConfig.filter_min_score ?? 5;
|
const globalMin = searchConfig.filter_min_score ?? DEFAULT_FILTER_MIN_SCORE;
|
||||||
|
|
||||||
let passed = 0, filtered = 0, errors = 0;
|
let passed = 0, filtered = 0, errors = 0;
|
||||||
const queue = loadQueue();
|
const queue = loadQueue();
|
||||||
@@ -222,7 +223,7 @@ async function submit(settings, searchConfig, candidateProfile) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const model = settings.filter?.model || DEFAULT_MODEL;
|
const model = settings.filter?.model || DEFAULT_FILTER_MODEL;
|
||||||
const submittedAt = new Date().toISOString();
|
const submittedAt = new Date().toISOString();
|
||||||
console.log(`🚀 Submitting batches — ${filterable.length} jobs across ${Object.keys(jobProfilesByTrack).length} tracks, model: ${model}`);
|
console.log(`🚀 Submitting batches — ${filterable.length} jobs across ${Object.keys(jobProfilesByTrack).length} tracks, model: ${model}`);
|
||||||
|
|
||||||
|
|||||||
@@ -6,8 +6,7 @@
|
|||||||
*/
|
*/
|
||||||
import { loadEnv } from './lib/env.mjs';
|
import { loadEnv } from './lib/env.mjs';
|
||||||
loadEnv(); // load .env before anything else
|
loadEnv(); // load .env before anything else
|
||||||
/**
|
|
||||||
*/
|
|
||||||
import { dirname, resolve } from 'path';
|
import { dirname, resolve } from 'path';
|
||||||
import { fileURLToPath } from 'url';
|
import { fileURLToPath } from 'url';
|
||||||
|
|
||||||
@@ -35,15 +34,6 @@ async function main() {
|
|||||||
|
|
||||||
const settings = loadConfig(resolve(__dir, 'config/settings.json'));
|
const settings = loadConfig(resolve(__dir, 'config/settings.json'));
|
||||||
|
|
||||||
// Send notification even if SIGTERM kills the process mid-run
|
|
||||||
process.once('SIGTERM', async () => {
|
|
||||||
if (totalAdded > 0) {
|
|
||||||
const summary = formatSearchSummary(totalAdded, totalSeen - totalAdded, platformsRun.length ? platformsRun : ['LinkedIn']);
|
|
||||||
await sendTelegram(settings, summary + '\n_(partial run — timed out)_').catch(() => {});
|
|
||||||
}
|
|
||||||
process.exit(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
const writeLastRun = (finished = false) => {
|
const writeLastRun = (finished = false) => {
|
||||||
const entry = {
|
const entry = {
|
||||||
started_at: startedAt,
|
started_at: startedAt,
|
||||||
@@ -68,9 +58,13 @@ async function main() {
|
|||||||
writeFileSync(runsPath, JSON.stringify(runs, null, 2));
|
writeFileSync(runsPath, JSON.stringify(runs, null, 2));
|
||||||
};
|
};
|
||||||
|
|
||||||
lock.onShutdown(() => {
|
lock.onShutdown(async () => {
|
||||||
console.log(' Writing partial results to last-run file...');
|
console.log(' Writing partial results to last-run file...');
|
||||||
writeLastRun(false);
|
writeLastRun(false);
|
||||||
|
if (totalAdded > 0) {
|
||||||
|
const summary = formatSearchSummary(totalAdded, totalSeen - totalAdded, platformsRun.length ? platformsRun : ['LinkedIn']);
|
||||||
|
await sendTelegram(settings, summary + '\n_(partial run — interrupted)_').catch(() => {});
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Load config
|
// Load config
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ export async function apply(page, job, formFiller) {
|
|||||||
company: document.querySelector('[class*="company"] h2, [class*="startup"] h2, h2')?.textContent?.trim(),
|
company: document.querySelector('[class*="company"] h2, [class*="startup"] h2, h2')?.textContent?.trim(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const applyBtn = await page.$('a:has-text("Apply"), button:has-text("Apply Now"), a:has-text("Apply Now")');
|
const applyBtn = page.locator('a:has-text("Apply"), button:has-text("Apply Now"), a:has-text("Apply Now")').first();
|
||||||
if (!applyBtn) return { status: 'no_button', meta };
|
if (await applyBtn.count() === 0) return { status: 'no_button', meta };
|
||||||
|
|
||||||
await applyBtn.click();
|
await applyBtn.click();
|
||||||
await page.waitForTimeout(FORM_FILL_WAIT);
|
await page.waitForTimeout(FORM_FILL_WAIT);
|
||||||
|
|||||||
@@ -35,18 +35,36 @@ export const LINKEDIN_MAX_MODAL_STEPS = 12;
|
|||||||
export const WELLFOUND_BASE = 'https://wellfound.com';
|
export const WELLFOUND_BASE = 'https://wellfound.com';
|
||||||
|
|
||||||
// --- Browser ---
|
// --- Browser ---
|
||||||
export const LOCAL_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36';
|
|
||||||
export const KERNEL_SDK_PATH = '/home/ubuntu/.openclaw/workspace/node_modules/@onkernel/sdk/index.js';
|
export const KERNEL_SDK_PATH = '/home/ubuntu/.openclaw/workspace/node_modules/@onkernel/sdk/index.js';
|
||||||
export const DEFAULT_PLAYWRIGHT_PATH = '/home/ubuntu/.npm-global/lib/node_modules/playwright/index.mjs';
|
export const DEFAULT_PLAYWRIGHT_PATH = '/home/ubuntu/.npm-global/lib/node_modules/playwright/index.mjs';
|
||||||
|
|
||||||
|
// --- Search ---
|
||||||
|
export const LINKEDIN_MAX_SEARCH_PAGES = 40;
|
||||||
|
export const WELLFOUND_MAX_INFINITE_SCROLL = 10;
|
||||||
|
export const LINKEDIN_SECONDS_PER_DAY = 86400;
|
||||||
|
|
||||||
|
// --- Session ---
|
||||||
|
export const SESSION_REFRESH_POLL_TIMEOUT = 30000;
|
||||||
|
export const SESSION_REFRESH_POLL_WAIT = 2000;
|
||||||
|
export const SESSION_LOGIN_VERIFY_WAIT = 3000;
|
||||||
|
|
||||||
// --- Form Filler Defaults ---
|
// --- Form Filler Defaults ---
|
||||||
export const DEFAULT_YEARS_EXPERIENCE = 7;
|
export const DEFAULT_YEARS_EXPERIENCE = 7;
|
||||||
export const DEFAULT_DESIRED_SALARY = 150000;
|
export const DEFAULT_DESIRED_SALARY = 150000;
|
||||||
export const MINIMUM_SALARY_FACTOR = 0.85;
|
export const MINIMUM_SALARY_FACTOR = 0.85;
|
||||||
export const DEFAULT_SKILL_RATING = '8';
|
export const DEFAULT_SKILL_RATING = '8';
|
||||||
|
export const FORM_PATTERN_MAX_LENGTH = 200;
|
||||||
export const DEFAULT_FIRST_RUN_DAYS = 90;
|
export const DEFAULT_FIRST_RUN_DAYS = 90;
|
||||||
export const SEARCH_RESULTS_MAX = 30;
|
export const SEARCH_RESULTS_MAX = 30;
|
||||||
|
|
||||||
|
// --- Anthropic API ---
|
||||||
|
export const ANTHROPIC_API_URL = 'https://api.anthropic.com/v1/messages';
|
||||||
|
export const ANTHROPIC_BATCH_API_URL = 'https://api.anthropic.com/v1/messages/batches';
|
||||||
|
export const FILTER_DESC_MAX_CHARS = 800;
|
||||||
|
export const FILTER_BATCH_MAX_TOKENS = 1024;
|
||||||
|
export const DEFAULT_FILTER_MODEL = 'claude-sonnet-4-6-20251101';
|
||||||
|
export const DEFAULT_FILTER_MIN_SCORE = 5;
|
||||||
|
|
||||||
// --- Notification ---
|
// --- Notification ---
|
||||||
export const TELEGRAM_API_BASE = 'https://api.telegram.org/bot';
|
export const TELEGRAM_API_BASE = 'https://api.telegram.org/bot';
|
||||||
export const NOTIFY_RATE_LIMIT_MS = 1500;
|
export const NOTIFY_RATE_LIMIT_MS = 1500;
|
||||||
@@ -68,5 +86,4 @@ export const EXTERNAL_ATS_PATTERNS = [
|
|||||||
];
|
];
|
||||||
|
|
||||||
// --- Queue ---
|
// --- Queue ---
|
||||||
export const DEFAULT_REVIEW_WINDOW_MINUTES = 30;
|
|
||||||
export const DEFAULT_MAX_RETRIES = 2;
|
export const DEFAULT_MAX_RETRIES = 2;
|
||||||
|
|||||||
@@ -5,9 +5,9 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { readFileSync, existsSync } from 'fs';
|
import { readFileSync, existsSync } from 'fs';
|
||||||
|
import {
|
||||||
const DESC_MAX_CHARS = 800;
|
ANTHROPIC_BATCH_API_URL, FILTER_DESC_MAX_CHARS, FILTER_BATCH_MAX_TOKENS
|
||||||
const BATCH_API = 'https://api.anthropic.com/v1/messages/batches';
|
} from './constants.mjs';
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Helpers
|
// Helpers
|
||||||
@@ -43,7 +43,7 @@ function sanitize(str) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildJobMessage(job) {
|
function buildJobMessage(job) {
|
||||||
const desc = sanitize(job.description).substring(0, DESC_MAX_CHARS).replace(/\s+/g, ' ').trim();
|
const desc = sanitize(job.description).substring(0, FILTER_DESC_MAX_CHARS).replace(/\s+/g, ' ').trim();
|
||||||
return `Title: ${sanitize(job.title)}
|
return `Title: ${sanitize(job.title)}
|
||||||
Company: ${sanitize(job.company) || 'Unknown'}
|
Company: ${sanitize(job.company) || 'Unknown'}
|
||||||
Location: ${sanitize(job.location) || 'Unknown'}
|
Location: ${sanitize(job.location) || 'Unknown'}
|
||||||
@@ -99,14 +99,14 @@ export async function submitBatches(jobs, jobProfilesByTrack, candidateProfile,
|
|||||||
custom_id: customId,
|
custom_id: customId,
|
||||||
params: {
|
params: {
|
||||||
model,
|
model,
|
||||||
max_tokens: 1024,
|
max_tokens: FILTER_BATCH_MAX_TOKENS,
|
||||||
system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }],
|
system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }],
|
||||||
messages: [{ role: 'user', content: buildJobMessage(job) }],
|
messages: [{ role: 'user', content: buildJobMessage(job) }],
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const res = await fetch(BATCH_API, {
|
const res = await fetch(ANTHROPIC_BATCH_API_URL, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: apiHeaders(apiKey),
|
headers: apiHeaders(apiKey),
|
||||||
body: JSON.stringify({ requests }),
|
body: JSON.stringify({ requests }),
|
||||||
@@ -129,7 +129,7 @@ export async function submitBatches(jobs, jobProfilesByTrack, candidateProfile,
|
|||||||
* Check batch status. Returns { status: 'in_progress'|'ended', counts }
|
* Check batch status. Returns { status: 'in_progress'|'ended', counts }
|
||||||
*/
|
*/
|
||||||
export async function checkBatch(batchId, apiKey) {
|
export async function checkBatch(batchId, apiKey) {
|
||||||
const res = await fetch(`${BATCH_API}/${batchId}`, {
|
const res = await fetch(`${ANTHROPIC_BATCH_API_URL}/${batchId}`, {
|
||||||
headers: apiHeaders(apiKey),
|
headers: apiHeaders(apiKey),
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -155,7 +155,7 @@ const PRICING = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
export async function downloadResults(batchId, apiKey, idMap = {}) {
|
export async function downloadResults(batchId, apiKey, idMap = {}) {
|
||||||
const res = await fetch(`${BATCH_API}/${batchId}/results`, {
|
const res = await fetch(`${ANTHROPIC_BATCH_API_URL}/${batchId}/results`, {
|
||||||
headers: apiHeaders(apiKey),
|
headers: apiHeaders(apiKey),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
import {
|
import {
|
||||||
DEFAULT_YEARS_EXPERIENCE, DEFAULT_DESIRED_SALARY,
|
DEFAULT_YEARS_EXPERIENCE, DEFAULT_DESIRED_SALARY,
|
||||||
MINIMUM_SALARY_FACTOR, DEFAULT_SKILL_RATING,
|
MINIMUM_SALARY_FACTOR, DEFAULT_SKILL_RATING,
|
||||||
LINKEDIN_EASY_APPLY_MODAL_SELECTOR
|
LINKEDIN_EASY_APPLY_MODAL_SELECTOR, FORM_PATTERN_MAX_LENGTH
|
||||||
} from './constants.mjs';
|
} from './constants.mjs';
|
||||||
|
|
||||||
export class FormFiller {
|
export class FormFiller {
|
||||||
@@ -23,7 +23,7 @@ export class FormFiller {
|
|||||||
// Check custom answers first (user-defined, pattern is substring or regex)
|
// Check custom answers first (user-defined, pattern is substring or regex)
|
||||||
for (const entry of this.answers) {
|
for (const entry of this.answers) {
|
||||||
try {
|
try {
|
||||||
if (entry.pattern.length > 200) throw new Error('pattern too long');
|
if (entry.pattern.length > FORM_PATTERN_MAX_LENGTH) throw new Error('pattern too long');
|
||||||
const re = new RegExp(entry.pattern, 'i');
|
const re = new RegExp(entry.pattern, 'i');
|
||||||
if (re.test(l)) return String(entry.answer);
|
if (re.test(l)) return String(entry.answer);
|
||||||
} catch {
|
} catch {
|
||||||
@@ -180,8 +180,8 @@ export class FormFiller {
|
|||||||
if (anyChecked) continue;
|
if (anyChecked) continue;
|
||||||
const answer = this.answerFor(leg);
|
const answer = this.answerFor(leg);
|
||||||
if (answer) {
|
if (answer) {
|
||||||
const lbl = await fs.$(`label:has-text("${answer}")`);
|
const lbl = fs.locator(`label:has-text("${answer}")`).first();
|
||||||
if (lbl) await lbl.click().catch(() => {});
|
if (await lbl.count() > 0) await lbl.click().catch(() => {});
|
||||||
} else {
|
} else {
|
||||||
unknown.push(leg);
|
unknown.push(leg);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,8 @@
|
|||||||
* One Claude call per search track using full profile + search config context
|
* One Claude call per search track using full profile + search config context
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { ANTHROPIC_API_URL } from './constants.mjs';
|
||||||
|
|
||||||
export async function generateKeywords(search, profile, apiKey) {
|
export async function generateKeywords(search, profile, apiKey) {
|
||||||
if (!apiKey) throw new Error('ANTHROPIC_API_KEY not set');
|
if (!apiKey) throw new Error('ANTHROPIC_API_KEY not set');
|
||||||
|
|
||||||
@@ -38,7 +40,7 @@ Think about:
|
|||||||
Return ONLY a JSON array of strings, no explanation, no markdown.
|
Return ONLY a JSON array of strings, no explanation, no markdown.
|
||||||
Example format: ["query one", "query two", "query three"]`;
|
Example format: ["query one", "query two", "query three"]`;
|
||||||
|
|
||||||
const res = await fetch('https://api.anthropic.com/v1/messages', {
|
const res = await fetch(ANTHROPIC_API_URL, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
@@ -46,7 +48,7 @@ Example format: ["query one", "query two", "query three"]`;
|
|||||||
'anthropic-version': '2023-06-01'
|
'anthropic-version': '2023-06-01'
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
model: 'claude-3-haiku-20240307',
|
model: 'claude-haiku-4-5-20251001',
|
||||||
max_tokens: 1024,
|
max_tokens: 1024,
|
||||||
messages: [{ role: 'user', content: prompt }]
|
messages: [{ role: 'user', content: prompt }]
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -5,11 +5,9 @@
|
|||||||
import {
|
import {
|
||||||
LINKEDIN_BASE, NAVIGATION_TIMEOUT, FEED_NAVIGATION_TIMEOUT,
|
LINKEDIN_BASE, NAVIGATION_TIMEOUT, FEED_NAVIGATION_TIMEOUT,
|
||||||
PAGE_LOAD_WAIT, SCROLL_WAIT, CLICK_WAIT,
|
PAGE_LOAD_WAIT, SCROLL_WAIT, CLICK_WAIT,
|
||||||
EXTERNAL_ATS_PATTERNS
|
EXTERNAL_ATS_PATTERNS, LINKEDIN_LINKEDIN_MAX_SEARCH_PAGES, LINKEDIN_SECONDS_PER_DAY
|
||||||
} from './constants.mjs';
|
} from './constants.mjs';
|
||||||
|
|
||||||
const MAX_SEARCH_PAGES = 40;
|
|
||||||
|
|
||||||
export async function verifyLogin(page) {
|
export async function verifyLogin(page) {
|
||||||
await page.goto(`${LINKEDIN_BASE}/feed/`, { waitUntil: 'domcontentloaded', timeout: FEED_NAVIGATION_TIMEOUT });
|
await page.goto(`${LINKEDIN_BASE}/feed/`, { waitUntil: 'domcontentloaded', timeout: FEED_NAVIGATION_TIMEOUT });
|
||||||
await page.waitForTimeout(CLICK_WAIT);
|
await page.waitForTimeout(CLICK_WAIT);
|
||||||
@@ -30,7 +28,7 @@ export async function searchLinkedIn(page, search, { onPage, onKeyword } = {}) {
|
|||||||
if (search.filters?.remote) params.set('f_WT', '2');
|
if (search.filters?.remote) params.set('f_WT', '2');
|
||||||
if (search.filters?.easy_apply_only) params.set('f_LF', 'f_AL');
|
if (search.filters?.easy_apply_only) params.set('f_LF', 'f_AL');
|
||||||
if (search.filters?.posted_within_days) {
|
if (search.filters?.posted_within_days) {
|
||||||
params.set('f_TPR', `r${search.filters.posted_within_days * 86400}`);
|
params.set('f_TPR', `r${search.filters.posted_within_days * LINKEDIN_SECONDS_PER_DAY}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const url = `${LINKEDIN_BASE}/jobs/search/?${params.toString()}`;
|
const url = `${LINKEDIN_BASE}/jobs/search/?${params.toString()}`;
|
||||||
@@ -43,7 +41,7 @@ export async function searchLinkedIn(page, search, { onPage, onKeyword } = {}) {
|
|||||||
await page.waitForTimeout(PAGE_LOAD_WAIT);
|
await page.waitForTimeout(PAGE_LOAD_WAIT);
|
||||||
|
|
||||||
let pageNum = 0;
|
let pageNum = 0;
|
||||||
while (pageNum < MAX_SEARCH_PAGES) {
|
while (pageNum < LINKEDIN_MAX_SEARCH_PAGES) {
|
||||||
// Scroll to load all cards
|
// Scroll to load all cards
|
||||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||||
await page.waitForTimeout(SCROLL_WAIT);
|
await page.waitForTimeout(SCROLL_WAIT);
|
||||||
|
|||||||
@@ -3,10 +3,12 @@
|
|||||||
* Call refreshSession() before creating a browser to ensure the profile is fresh
|
* Call refreshSession() before creating a browser to ensure the profile is fresh
|
||||||
*/
|
*/
|
||||||
import { createRequire } from 'module';
|
import { createRequire } from 'module';
|
||||||
|
import {
|
||||||
|
KERNEL_SDK_PATH, SESSION_REFRESH_POLL_TIMEOUT, SESSION_REFRESH_POLL_WAIT,
|
||||||
|
SESSION_LOGIN_VERIFY_WAIT
|
||||||
|
} from './constants.mjs';
|
||||||
const require = createRequire(import.meta.url);
|
const require = createRequire(import.meta.url);
|
||||||
|
|
||||||
const KERNEL_SDK_PATH = '/home/ubuntu/.openclaw/workspace/node_modules/@onkernel/sdk/index.js';
|
|
||||||
|
|
||||||
export async function refreshSession(platform, apiKey, connectionIds = {}) {
|
export async function refreshSession(platform, apiKey, connectionIds = {}) {
|
||||||
const connectionId = connectionIds[platform];
|
const connectionId = connectionIds[platform];
|
||||||
if (!connectionId) throw new Error(`No Kernel connection ID configured for platform: ${platform} — add it to settings.json under kernel.connection_ids`);
|
if (!connectionId) throw new Error(`No Kernel connection ID configured for platform: ${platform} — add it to settings.json under kernel.connection_ids`);
|
||||||
@@ -28,8 +30,8 @@ export async function refreshSession(platform, apiKey, connectionIds = {}) {
|
|||||||
console.log(` ⏳ ${platform} session pending (status: ${loginResp.status}), polling...`);
|
console.log(` ⏳ ${platform} session pending (status: ${loginResp.status}), polling...`);
|
||||||
const start = Date.now();
|
const start = Date.now();
|
||||||
let pollCount = 0;
|
let pollCount = 0;
|
||||||
while (Date.now() - start < 30000) {
|
while (Date.now() - start < SESSION_REFRESH_POLL_TIMEOUT) {
|
||||||
await new Promise(r => setTimeout(r, 2000));
|
await new Promise(r => setTimeout(r, SESSION_REFRESH_POLL_WAIT));
|
||||||
pollCount++;
|
pollCount++;
|
||||||
const conn = await kernel.auth.connections.retrieve(connectionId);
|
const conn = await kernel.auth.connections.retrieve(connectionId);
|
||||||
if (conn.status === 'SUCCESS') {
|
if (conn.status === 'SUCCESS') {
|
||||||
@@ -58,7 +60,7 @@ export async function ensureLoggedIn(page, verifyFn, platform, apiKey, connectio
|
|||||||
console.warn(` ⚠️ ${platform} not logged in (attempt ${attempt}), refreshing session...`);
|
console.warn(` ⚠️ ${platform} not logged in (attempt ${attempt}), refreshing session...`);
|
||||||
await refreshSession(platform, apiKey, connectionIds);
|
await refreshSession(platform, apiKey, connectionIds);
|
||||||
await page.reload({ waitUntil: 'domcontentloaded' });
|
await page.reload({ waitUntil: 'domcontentloaded' });
|
||||||
await page.waitForTimeout(3000);
|
await page.waitForTimeout(SESSION_LOGIN_VERIFY_WAIT);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -5,11 +5,9 @@
|
|||||||
import {
|
import {
|
||||||
WELLFOUND_BASE, NAVIGATION_TIMEOUT, SEARCH_NAVIGATION_TIMEOUT,
|
WELLFOUND_BASE, NAVIGATION_TIMEOUT, SEARCH_NAVIGATION_TIMEOUT,
|
||||||
SEARCH_LOAD_WAIT, SEARCH_SCROLL_WAIT, LOGIN_WAIT,
|
SEARCH_LOAD_WAIT, SEARCH_SCROLL_WAIT, LOGIN_WAIT,
|
||||||
SEARCH_RESULTS_MAX
|
SEARCH_RESULTS_MAX, WELLFOUND_WELLFOUND_MAX_INFINITE_SCROLL
|
||||||
} from './constants.mjs';
|
} from './constants.mjs';
|
||||||
|
|
||||||
const MAX_INFINITE_SCROLL = 10;
|
|
||||||
|
|
||||||
export async function verifyLogin(page) {
|
export async function verifyLogin(page) {
|
||||||
await page.goto(`${WELLFOUND_BASE}/`, { waitUntil: 'domcontentloaded', timeout: NAVIGATION_TIMEOUT });
|
await page.goto(`${WELLFOUND_BASE}/`, { waitUntil: 'domcontentloaded', timeout: NAVIGATION_TIMEOUT });
|
||||||
await page.waitForTimeout(LOGIN_WAIT);
|
await page.waitForTimeout(LOGIN_WAIT);
|
||||||
@@ -34,7 +32,7 @@ export async function searchWellfound(page, search, { onPage } = {}) {
|
|||||||
|
|
||||||
// Scroll to bottom repeatedly to trigger infinite scroll
|
// Scroll to bottom repeatedly to trigger infinite scroll
|
||||||
let lastHeight = 0;
|
let lastHeight = 0;
|
||||||
for (let i = 0; i < MAX_INFINITE_SCROLL; i++) {
|
for (let i = 0; i < WELLFOUND_MAX_INFINITE_SCROLL; i++) {
|
||||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||||
await page.waitForTimeout(SEARCH_SCROLL_WAIT);
|
await page.waitForTimeout(SEARCH_SCROLL_WAIT);
|
||||||
const newHeight = await page.evaluate(() => document.body.scrollHeight);
|
const newHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||||
|
|||||||
Reference in New Issue
Block a user