Per-track lookback and per-track profiles

Searcher: each track independently tracks when it was last searched via data/track_history.json. New tracks get full lookback (90d), existing tracks look back since their last completion. Keyword-level crash resume preserved. Profiles: search tracks can specify profile_overrides (inline) or profile_path (external file) to customize resume, cover letter, and experience per track. Filter and applier both use the track-specific profile. Base profile.json provides shared info (name, contact, etc). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-07 11:20:07 -08:00
parent 9b0aa7bf77
commit c8357bb358
6 changed files with 148 additions and 46 deletions
--- a/job_applier.mjs
+++ b/job_applier.mjs
@@ -26,6 +26,7 @@ import { createBrowser } from './lib/browser.mjs';
 import { ensureAuth } from './lib/session.mjs';
 import { FormFiller } from './lib/form_filler.mjs';
 import { applyToJob, supportedTypes } from './lib/apply/index.mjs';
+import { buildTrackProfiles, getTrackProfile } from './lib/profile.mjs';
 import { sendTelegram, formatApplySummary } from './lib/notify.mjs';
 import { processTelegramReplies } from './lib/telegram_answers.mjs';
 import { generateAnswer } from './lib/ai_answer.mjs';
@@ -45,11 +46,16 @@ async function main() {

  const settings = await loadConfig(resolve(__dir, 'config/settings.json'));
  await initQueue(settings);
-  const profile = await loadConfig(resolve(__dir, 'config/profile.json'));
+  const baseProfile = await loadConfig(resolve(__dir, 'config/profile.json'));
+  const searchConfig = await loadConfig(resolve(__dir, 'config/search_config.json'));
+  const profilesByTrack = await buildTrackProfiles(baseProfile, searchConfig.searches || []);

-  // Ensure resume is available locally (downloads from S3 if needed)
-  if (profile.resume_path) {
-    profile.resume_path = await ensureLocalFile('config/Matthew_Jackson_Resume.pdf', profile.resume_path);
+  // Ensure resume is available locally for each track profile
+  for (const prof of Object.values(profilesByTrack)) {
+    if (prof.resume_path) {
+      const s3Key = prof.resume_path.startsWith('/') ? prof.resume_path.split('/').pop() : prof.resume_path;
+      prof.resume_path = await ensureLocalFile(`config/${s3Key}`, prof.resume_path);
+    }
  }

  const answersPath = resolve(__dir, 'config/answers.json');
@@ -58,7 +64,8 @@ async function main() {
  const maxRetries = settings.max_retries ?? DEFAULT_MAX_RETRIES;
  const enabledTypes = settings.enabled_apply_types || DEFAULT_ENABLED_APPLY_TYPES;
  const apiKey = process.env.ANTHROPIC_API_KEY || settings.anthropic_api_key;
-  const formFiller = new FormFiller(profile, answers, { apiKey, answersPath });
+  // Default FormFiller uses base profile — swapped per job below
+  const formFiller = new FormFiller(baseProfile, answers, { apiKey, answersPath });

  const startedAt = Date.now();
  const rateLimitPath = resolve(__dir, 'data/linkedin_rate_limited_at.json');
@@ -213,7 +220,9 @@ async function main() {
          break;
        }

-        // Set job context for AI answers
+        // Swap profile for this job's track (different resume/cover letter per track)
+        const jobProfile = getTrackProfile(profilesByTrack, job.track);
+        formFiller.profile = jobProfile;
        formFiller.jobContext = { title: job.title, company: job.company };

        // Reload answers.json before each job — picks up Telegram replies between jobs
@@ -243,7 +252,7 @@ async function main() {
            new Promise((_, reject) => setTimeout(() => reject(new Error('Job apply timed out')), PER_JOB_TIMEOUT_MS)),
          ]);
          result.applyStartedAt = applyStartedAt;
-          await handleResult(job, result, results, settings, profile, apiKey);
+          await handleResult(job, result, results, settings, jobProfile, apiKey);
          if (results.rate_limited) break;
        } catch (e) {
          console.error(`    ❌ Error: ${e.message}`);
--- a/job_filter.mjs
+++ b/job_filter.mjs
@@ -32,6 +32,7 @@ process.stderr.write = (chunk, ...args) => { logStream.write(chunk); return orig

 import { getJobsByStatus, updateJobStatus, loadConfig, loadQueue, saveQueue, dedupeAfterFilter, initQueue } from './lib/queue.mjs';
 import { submitBatches, checkBatch, downloadResults } from './lib/filter.mjs';
+import { buildTrackProfiles } from './lib/profile.mjs';
 import { sendTelegram, formatFilterSummary } from './lib/notify.mjs';
 import { DEFAULT_FILTER_MODEL, DEFAULT_FILTER_MIN_SCORE } from './lib/constants.mjs';

@@ -204,7 +205,7 @@ async function collect(state, settings) {
 // Phase 2 — Submit a new batch
 // ---------------------------------------------------------------------------

-async function submit(settings, searchConfig, candidateProfile) {
+async function submit(settings, searchConfig, profilesByTrack) {
  const apiKey = process.env.ANTHROPIC_API_KEY;

  // Clear stale batch markers — jobs marked as submitted but no filter_state.json means
@@ -255,7 +256,7 @@ async function submit(settings, searchConfig, candidateProfile) {
  const submittedAt = new Date().toISOString();
  console.log(`🚀 Submitting batches — ${filterable.length} jobs across ${Object.keys(searchesByTrack).length} tracks, model: ${model}`);

-  const submitted = await submitBatches(filterable, searchesByTrack, candidateProfile, model, apiKey);
+  const submitted = await submitBatches(filterable, searchesByTrack, profilesByTrack, model, apiKey);

  writeState({
    batches: submitted,
@@ -304,7 +305,8 @@ async function main() {
  const settings = await loadConfig(resolve(__dir, 'config/settings.json'));
  await initQueue(settings);
  const searchConfig = await loadConfig(resolve(__dir, 'config/search_config.json'));
-  const candidateProfile = await loadConfig(resolve(__dir, 'config/profile.json'));
+  const baseProfile = await loadConfig(resolve(__dir, 'config/profile.json'));
+  const profilesByTrack = await buildTrackProfiles(baseProfile, searchConfig.searches || []);

  console.log('🔍 claw-apply: AI Job Filter\n');

@@ -317,7 +319,7 @@ async function main() {

  // Phase 2: submit any remaining unscored jobs (runs after collect too)
  if (!readState()) {
-    await submit(settings, searchConfig, candidateProfile);
+    await submit(settings, searchConfig, profilesByTrack);
  }
 }

--- a/job_searcher.mjs
+++ b/job_searcher.mjs
@@ -29,7 +29,7 @@ import { verifyLogin as wfLogin, searchWellfound } from './lib/wellfound.mjs';
 import { sendTelegram, formatSearchSummary } from './lib/notify.mjs';
 import { DEFAULT_FIRST_RUN_DAYS } from './lib/constants.mjs';
 import { generateKeywords } from './lib/keywords.mjs';
-import { initProgress, isCompleted, markComplete, getKeywordStart, markKeywordComplete, saveKeywords, getSavedKeywords, clearProgress } from './lib/search_progress.mjs';
+import { initProgress, isCompleted, markComplete, getKeywordStart, markKeywordComplete, saveKeywords, getSavedKeywords, clearProgress, saveTrackLookback } from './lib/search_progress.mjs';
 import { ensureLoggedIn } from './lib/session.mjs';

 async function main() {
@@ -102,10 +102,13 @@ async function main() {
  const profile = await loadConfig(resolve(__dir, 'config/profile.json'));
  const anthropicKey = process.env.ANTHROPIC_API_KEY || settings.anthropic_api_key;

-  // Determine lookback:
-  // 1. data/next_run.json override (consumed after use)
-  // 2. Resuming in-progress run
-  // 3. Dynamic: time since last run × 1.25
+  // Per-track lookback: each track remembers when it was last searched.
+  // New tracks get first_run_days (default 90), existing tracks look back since last completion.
+  const trackHistoryPath = resolve(__dir, 'data/track_history.json');
+  const trackHistory = existsSync(trackHistoryPath)
+    ? JSON.parse(readFileSync(trackHistoryPath, 'utf8'))
+    : {};
+
  const savedProgress = existsSync(resolve(__dir, 'data/search_progress.json'))
    ? JSON.parse(readFileSync(resolve(__dir, 'data/search_progress.json'), 'utf8'))
    : null;
@@ -118,34 +121,41 @@ async function main() {
    } catch {}
  }

-  function dynamicLookbackDays() {
-    const lastRunPath = resolve(__dir, 'data/searcher_last_run.json');
-    if (!existsSync(lastRunPath)) return searchConfig.first_run_days || DEFAULT_FIRST_RUN_DAYS;
-    const lastRun = JSON.parse(readFileSync(lastRunPath, 'utf8'));
-    const lastRanAt = lastRun.started_at || lastRun.finished_at;
-    if (!lastRanAt) return searchConfig.first_run_days || DEFAULT_FIRST_RUN_DAYS;
-    const hoursSince = (Date.now() - new Date(lastRanAt).getTime()) / (1000 * 60 * 60);
+  const defaultFirstRunDays = searchConfig.first_run_days || DEFAULT_FIRST_RUN_DAYS;
+
+  function lookbackForTrack(trackName) {
+    // Override applies to all tracks
+    if (nextRunOverride?.lookback_days) return nextRunOverride.lookback_days;
+    // Resuming a crashed run — use the saved per-track lookback
+    if (savedProgress?.track_lookback?.[trackName]) return savedProgress.track_lookback[trackName];
+    // Per-track history: how long since this track last completed?
+    const lastSearched = trackHistory[trackName]?.last_searched_at;
+    if (!lastSearched) return defaultFirstRunDays; // new track — full lookback
+    const hoursSince = (Date.now() - new Date(lastSearched).getTime()) / (1000 * 60 * 60);
    const buffered = hoursSince * 1.25;
    const minHours = 4;
-    const maxDays = searchConfig.first_run_days || DEFAULT_FIRST_RUN_DAYS;
-    return Math.min(Math.max(buffered / 24, minHours / 24), maxDays);
+    return Math.min(Math.max(buffered / 24, minHours / 24), defaultFirstRunDays);
  }

-  let lookbackDays;
-  if (nextRunOverride?.lookback_days) {
-    lookbackDays = nextRunOverride.lookback_days;
-    console.log(`📋 Override from next_run.json — looking back ${lookbackDays} days\n`);
-  } else if (savedProgress?.lookback_days) {
-    lookbackDays = savedProgress.lookback_days;
-    console.log(`🔁 Resuming ${lookbackDays.toFixed(2)}-day search run\n`);
-  } else {
-    lookbackDays = dynamicLookbackDays();
-    const hours = (lookbackDays * 24).toFixed(1);
-    console.log(`⏱️  Lookback: ${hours}h (time since last run × 1.25)\n`);
+  function saveTrackCompletion(trackName) {
+    trackHistory[trackName] = { last_searched_at: new Date().toISOString() };
+    writeFileSync(trackHistoryPath, JSON.stringify(trackHistory, null, 2));
  }

+  // Log per-track lookback
+  console.log('📅 Per-track lookback:');
+  for (const search of searchConfig.searches) {
+    const days = lookbackForTrack(search.name);
+    const label = trackHistory[search.name]?.last_searched_at ? `${(days * 24).toFixed(1)}h since last run` : `${days}d (new track)`;
+    console.log(`  • ${search.name}: ${label}`);
+  }
+  if (nextRunOverride?.lookback_days) console.log(`  (override: ${nextRunOverride.lookback_days}d for all tracks)`);
+  console.log('');
+
  // Init progress tracking — enables resume on restart
-  initProgress(resolve(__dir, 'data'), lookbackDays);
+  // Use max lookback across tracks for progress file identity
+  const maxLookback = Math.max(...searchConfig.searches.map(s => lookbackForTrack(s.name)));
+  initProgress(resolve(__dir, 'data'), maxLookback);

  // Enhance keywords with AI — reuse saved keywords from progress if resuming, never regenerate mid-run
  for (const search of searchConfig.searches) {
@@ -194,7 +204,9 @@ async function main() {
        }
        const keywordStart = getKeywordStart('linkedin', search.name);
        if (keywordStart > 0) console.log(`  [${search.name}] resuming from keyword ${keywordStart + 1}/${search.keywords.length}`);
-        const effectiveSearch = { ...search, keywords: search.keywords.slice(keywordStart), keywordOffset: keywordStart, filters: { ...search.filters, posted_within_days: lookbackDays } };
+        const trackLookback = lookbackForTrack(search.name);
+        saveTrackLookback(search.name, trackLookback);
+        const effectiveSearch = { ...search, keywords: search.keywords.slice(keywordStart), keywordOffset: keywordStart, filters: { ...search.filters, posted_within_days: trackLookback } };
        let queryFound = 0, queryAdded = 0;
        try {
          await searchLinkedIn(liBrowser.page, effectiveSearch, {
@@ -225,6 +237,7 @@ async function main() {
        }
        console.log(`\r  [${search.name}] ${queryFound} found, ${queryAdded} new`);
        markComplete('linkedin', search.name, { found: queryFound, added: queryAdded });
+        saveTrackCompletion(search.name);
        const tc = trackCounts[search.name] || (trackCounts[search.name] = { found: 0, added: 0 });
        tc.found += queryFound; tc.added += queryAdded;
        // Save progress after each search track
@@ -280,7 +293,9 @@ async function main() {
          console.log(`  [${search.name}] ✓ already done, skipping`);
          continue;
        }
-        const effectiveSearch = { ...search, filters: { ...search.filters, posted_within_days: lookbackDays } };
+        const trackLookback = lookbackForTrack(search.name);
+        saveTrackLookback(search.name, trackLookback);
+        const effectiveSearch = { ...search, filters: { ...search.filters, posted_within_days: trackLookback } };
        let queryFound = 0, queryAdded = 0;
        try {
          await searchWellfound(wfBrowser.page, effectiveSearch, {
@@ -307,6 +322,7 @@ async function main() {
        }
        console.log(`\r  [${search.name}] ${queryFound} found, ${queryAdded} new`);
        markComplete('wellfound', search.name, { found: queryFound, added: queryAdded });
+        saveTrackCompletion(search.name);
        const tc = trackCounts[search.name] || (trackCounts[search.name] = { found: 0, added: 0 });
        tc.found += queryFound; tc.added += queryAdded;
        writeLastRun(false);
--- a/lib/filter.mjs
+++ b/lib/filter.mjs
@@ -70,7 +70,7 @@ function apiHeaders(apiKey) {
 * Each batch uses the system prompt for that track only — maximizes prompt caching.
 * Returns array of { track, batchId, idMap, jobCount }
 */
-export async function submitBatches(jobs, searchesByTrack, candidateProfile, model, apiKey) {
+export async function submitBatches(jobs, searchesByTrack, profilesByTrack, model, apiKey) {
  // Group jobs by track
  const byTrack = {};
  for (const job of jobs) {
@@ -86,7 +86,8 @@ export async function submitBatches(jobs, searchesByTrack, candidateProfile, mod

  for (const [track, trackJobs] of Object.entries(byTrack)) {
    const searchTrack = searchesByTrack[track];
-    const systemPrompt = buildSystemPrompt(searchTrack, candidateProfile);
+    const trackProfile = profilesByTrack[track] || profilesByTrack._base;
+    const systemPrompt = buildSystemPrompt(searchTrack, trackProfile);
    const idMap = {};
    const requests = [];

--- a/lib/profile.mjs
+++ b/lib/profile.mjs
@@ -0,0 +1,65 @@
+/**
+ * profile.mjs — Per-track profile management
+ *
+ * Each search track can override parts of the base profile.json:
+ *   - profile_overrides: { ... } inline in search_config.json
+ *   - profile_path: "config/profiles/se_profile.json" (loaded + merged)
+ *
+ * Base profile has shared info (name, phone, location, work auth).
+ * Track overrides customize resume, cover letter, experience highlights, etc.
+ */
+import { loadJSON } from './storage.mjs';
+
+/**
+ * Deep merge b into a (b wins on conflicts). Arrays are replaced, not concatenated.
+ */
+function deepMerge(a, b) {
+  const result = { ...a };
+  for (const [key, val] of Object.entries(b)) {
+    if (val && typeof val === 'object' && !Array.isArray(val) && typeof result[key] === 'object' && !Array.isArray(result[key])) {
+      result[key] = deepMerge(result[key], val);
+    } else {
+      result[key] = val;
+    }
+  }
+  return result;
+}
+
+/**
+ * Build per-track profile map from base profile + search config.
+ * Returns { _base: baseProfile, trackKey: mergedProfile, ... }
+ */
+export async function buildTrackProfiles(baseProfile, searches) {
+  const profiles = { _base: baseProfile };
+
+  for (const search of searches) {
+    const track = search.track;
+    let trackProfile = baseProfile;
+
+    // Load external profile file if specified
+    if (search.profile_path) {
+      try {
+        const overrides = await loadJSON(search.profile_path, null);
+        if (overrides) trackProfile = deepMerge(trackProfile, overrides);
+      } catch (e) {
+        console.warn(`  ⚠️  [${track}] Could not load profile ${search.profile_path}: ${e.message}`);
+      }
+    }
+
+    // Apply inline overrides (takes precedence over profile_path)
+    if (search.profile_overrides) {
+      trackProfile = deepMerge(trackProfile, search.profile_overrides);
+    }
+
+    profiles[track] = trackProfile;
+  }
+
+  return profiles;
+}
+
+/**
+ * Get the profile for a specific track from a profiles map.
+ */
+export function getTrackProfile(profilesByTrack, track) {
+  return profilesByTrack[track] || profilesByTrack._base;
+}
--- a/lib/search_progress.mjs
+++ b/lib/search_progress.mjs
@@ -7,12 +7,13 @@ import { readFileSync, writeFileSync, existsSync, unlinkSync } from 'fs';
 let progressPath = null;
 let progress = null;

-export function initProgress(dataDir, lookbackDays) {
+export function initProgress(dataDir, maxLookbackDays) {
  progressPath = `${dataDir}/search_progress.json`;

  if (existsSync(progressPath)) {
    const saved = JSON.parse(readFileSync(progressPath, 'utf8'));
-    if (saved.lookback_days === lookbackDays) {
+    // Resume if an in-progress run exists (has uncompleted tracks)
+    if (saved.started_at && !saved.finished) {
      progress = saved;
      const done = progress.completed?.length ?? 0;
      if (done > 0) {
@@ -20,19 +21,27 @@ export function initProgress(dataDir, lookbackDays) {
      }
      return progress;
    }
-    console.log(`🆕 New lookback window (${lookbackDays}d), starting fresh\n`);
  }

  progress = {
-    lookback_days: lookbackDays,
+    lookback_days: maxLookbackDays,
+    track_lookback: {},  // per-track lookback days, saved on init for crash resume
    started_at: Date.now(),
    completed: [],
-    keyword_progress: {},  // key: "platform:track" → last completed keyword index (0-based)
+    keyword_progress: {},
  };
  save();
  return progress;
 }

+/** Save per-track lookback so crash resume uses the right value */
+export function saveTrackLookback(trackName, days) {
+  if (!progress) return;
+  if (!progress.track_lookback) progress.track_lookback = {};
+  progress.track_lookback[trackName] = days;
+  save();
+}
+
 /** Save generated keywords for a track — reused on resume, never regenerated mid-run */
 export function saveKeywords(platform, track, keywords) {
  if (!progress) return;