fix: add config validation and retry logic for failed jobs
- Add loadConfig() helper with clear errors for missing/malformed JSON - Replace raw JSON.parse(readFileSync(...)) in both entry points - Track retry_count on jobs; re-queue as 'new' up to max_retries (default 2) - Add max_retries and DEFAULT_MAX_RETRIES constant Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -4,14 +4,13 @@
|
||||
* Reads jobs queue and applies to each new/needs_answer job
|
||||
* Run via cron or manually: node job_applier.mjs [--preview]
|
||||
*/
|
||||
import { readFileSync, writeFileSync, existsSync } from 'fs';
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import { dirname, resolve } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dir = dirname(fileURLToPath(import.meta.url));
|
||||
const cfg = p => JSON.parse(readFileSync(resolve(__dir, p), 'utf8'));
|
||||
|
||||
import { getJobsByStatus, updateJobStatus, appendLog } from './lib/queue.mjs';
|
||||
import { getJobsByStatus, updateJobStatus, appendLog, loadConfig } from './lib/queue.mjs';
|
||||
import { createBrowser } from './lib/browser.mjs';
|
||||
import { FormFiller } from './lib/form_filler.mjs';
|
||||
import { verifyLogin as liLogin, applyLinkedIn } from './lib/linkedin.mjs';
|
||||
@@ -20,7 +19,7 @@ import { sendTelegram, formatApplySummary, formatUnknownQuestion } from './lib/n
|
||||
import {
|
||||
DEFAULT_REVIEW_WINDOW_MINUTES,
|
||||
APPLY_BETWEEN_DELAY_BASE, APPLY_BETWEEN_DELAY_WF_BASE,
|
||||
APPLY_BETWEEN_DELAY_JITTER
|
||||
APPLY_BETWEEN_DELAY_JITTER, DEFAULT_MAX_RETRIES
|
||||
} from './lib/constants.mjs';
|
||||
|
||||
const isPreview = process.argv.includes('--preview');
|
||||
@@ -28,14 +27,14 @@ const isPreview = process.argv.includes('--preview');
|
||||
async function main() {
|
||||
console.log('🚀 claw-apply: Job Applier starting\n');
|
||||
|
||||
const settings = cfg('config/settings.json');
|
||||
const profile = cfg('config/profile.json');
|
||||
const answers = existsSync(resolve(__dir, 'config/answers.json'))
|
||||
? JSON.parse(readFileSync(resolve(__dir, 'config/answers.json'), 'utf8'))
|
||||
: [];
|
||||
const settings = loadConfig(resolve(__dir, 'config/settings.json'));
|
||||
const profile = loadConfig(resolve(__dir, 'config/profile.json'));
|
||||
const answersPath = resolve(__dir, 'config/answers.json');
|
||||
const answers = existsSync(answersPath) ? loadConfig(answersPath) : [];
|
||||
|
||||
const formFiller = new FormFiller(profile, answers);
|
||||
const maxApps = settings.max_applications_per_run || Infinity;
|
||||
const maxRetries = settings.max_retries ?? DEFAULT_MAX_RETRIES;
|
||||
|
||||
// Preview mode: show queue and exit
|
||||
if (isPreview) {
|
||||
@@ -87,10 +86,7 @@ async function main() {
|
||||
const result = await applyLinkedIn(liBrowser.page, job, formFiller);
|
||||
await handleResult(job, result, results, settings);
|
||||
} catch (e) {
|
||||
console.log(` ❌ Error: ${e.message?.substring(0, 80)}`);
|
||||
updateJobStatus(job.id, 'failed', { notes: e.message?.substring(0, 80) });
|
||||
appendLog({ ...job, status: 'failed', error: e.message?.substring(0, 80) });
|
||||
results.failed++;
|
||||
handleError(job, e, results, maxRetries);
|
||||
}
|
||||
await liBrowser.page.waitForTimeout(APPLY_BETWEEN_DELAY_BASE + Math.random() * APPLY_BETWEEN_DELAY_JITTER);
|
||||
}
|
||||
@@ -117,10 +113,7 @@ async function main() {
|
||||
const result = await applyWellfound(wfBrowser.page, job, formFiller);
|
||||
await handleResult(job, result, results, settings);
|
||||
} catch (e) {
|
||||
console.log(` ❌ Error: ${e.message?.substring(0, 80)}`);
|
||||
updateJobStatus(job.id, 'failed', { notes: e.message?.substring(0, 80) });
|
||||
appendLog({ ...job, status: 'failed', error: e.message?.substring(0, 80) });
|
||||
results.failed++;
|
||||
handleError(job, e, results, maxRetries);
|
||||
}
|
||||
await wfBrowser.page.waitForTimeout(APPLY_BETWEEN_DELAY_WF_BASE + Math.random() * APPLY_BETWEEN_DELAY_JITTER);
|
||||
}
|
||||
@@ -199,6 +192,20 @@ async function handleResult(job, result, results, settings) {
|
||||
}
|
||||
}
|
||||
|
||||
function handleError(job, e, results, maxRetries) {
|
||||
const msg = e.message?.substring(0, 80);
|
||||
const retries = (job.retry_count || 0) + 1;
|
||||
if (retries <= maxRetries) {
|
||||
console.log(` ⚠️ Error (retry ${retries}/${maxRetries}): ${msg}`);
|
||||
updateJobStatus(job.id, 'new', { retry_count: retries, last_error: msg });
|
||||
} else {
|
||||
console.log(` ❌ Error (max retries reached): ${msg}`);
|
||||
updateJobStatus(job.id, 'failed', { retry_count: retries, notes: msg });
|
||||
appendLog({ ...job, status: 'failed', error: msg });
|
||||
results.failed++;
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(e => {
|
||||
console.error('Fatal:', e.message);
|
||||
process.exit(1);
|
||||
|
||||
@@ -4,14 +4,12 @@
|
||||
* Searches LinkedIn + Wellfound and populates the jobs queue
|
||||
* Run via cron or manually: node job_searcher.mjs
|
||||
*/
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import { dirname, resolve } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dir = dirname(fileURLToPath(import.meta.url));
|
||||
const cfg = p => JSON.parse(readFileSync(resolve(__dir, p), 'utf8'));
|
||||
|
||||
import { addJobs, loadQueue } from './lib/queue.mjs';
|
||||
import { addJobs, loadQueue, loadConfig } from './lib/queue.mjs';
|
||||
import { createBrowser } from './lib/browser.mjs';
|
||||
import { verifyLogin as liLogin, searchLinkedIn } from './lib/linkedin.mjs';
|
||||
import { verifyLogin as wfLogin, searchWellfound } from './lib/wellfound.mjs';
|
||||
@@ -22,8 +20,8 @@ async function main() {
|
||||
console.log('🔍 claw-apply: Job Searcher starting\n');
|
||||
|
||||
// Load config
|
||||
const settings = cfg('config/settings.json');
|
||||
const searchConfig = cfg('config/search_config.json');
|
||||
const settings = loadConfig(resolve(__dir, 'config/settings.json'));
|
||||
const searchConfig = loadConfig(resolve(__dir, 'config/search_config.json'));
|
||||
|
||||
// First run detection: if queue is empty, use first_run_days lookback
|
||||
const isFirstRun = loadQueue().length === 0;
|
||||
|
||||
@@ -53,3 +53,4 @@ export const NOTIFY_RATE_LIMIT_MS = 1500;
|
||||
|
||||
// --- Queue ---
|
||||
export const DEFAULT_REVIEW_WINDOW_MINUTES = 30;
|
||||
export const DEFAULT_MAX_RETRIES = 2;
|
||||
|
||||
@@ -3,13 +3,34 @@
|
||||
* Handles jobs_queue.json read/write/update
|
||||
*/
|
||||
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
|
||||
import { dirname } from 'path';
|
||||
import { dirname, resolve } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dir = dirname(fileURLToPath(import.meta.url));
|
||||
const QUEUE_PATH = `${__dir}/../data/jobs_queue.json`;
|
||||
const LOG_PATH = `${__dir}/../data/applications_log.json`;
|
||||
|
||||
/**
|
||||
* Load and validate a JSON config file. Throws with a clear message on failure.
|
||||
*/
|
||||
export function loadConfig(filePath) {
|
||||
const resolved = resolve(filePath);
|
||||
if (!existsSync(resolved)) {
|
||||
throw new Error(`Config file not found: ${resolved}\nCopy the matching .example.json and fill in your values.`);
|
||||
}
|
||||
let raw;
|
||||
try {
|
||||
raw = readFileSync(resolved, 'utf8');
|
||||
} catch (e) {
|
||||
throw new Error(`Cannot read config file ${resolved}: ${e.message}`);
|
||||
}
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch (e) {
|
||||
throw new Error(`Invalid JSON in ${resolved}: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function ensureDir(path) {
|
||||
const dir = dirname(path);
|
||||
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
||||
|
||||
Reference in New Issue
Block a user