divapply 0.4.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1404 @@
1
+ """Apply orchestration: acquire jobs, spawn Claude Code sessions, track results.
2
+
3
+ This is the main entry point for the apply pipeline. It pulls jobs from
4
+ the database, launches Chrome + Claude Code for each one, parses the
5
+ result, and updates the database. Supports parallel workers via --workers.
6
+ """
7
+
8
+ import atexit
9
+ import json
10
+ import logging
11
+ import os
12
+ import platform
13
+ import re
14
+ import shlex
15
+ import signal
16
+ import subprocess
17
+ import sys
18
+ import threading
19
+ import time
20
+ from concurrent.futures import ThreadPoolExecutor, as_completed
21
+ from datetime import datetime, timezone
22
+ from pathlib import Path
23
+
24
+ from rich.console import Console
25
+ from rich.live import Live
26
+
27
+ from divapply import config
28
+ from divapply.database import get_connection
29
+ from divapply.apply import chrome, dashboard, prompt as prompt_mod
30
+ from divapply.apply.chrome import (
31
+ launch_chrome, cleanup_worker, kill_all_chrome,
32
+ reset_worker_dir, cleanup_on_exit, _kill_process_tree,
33
+ setup_worker_profile,
34
+ BASE_CDP_PORT,
35
+ )
36
+ from divapply.apply.dashboard import (
37
+ init_worker, update_state, add_event, get_state,
38
+ render_full, get_totals,
39
+ )
40
+
41
+ logger = logging.getLogger(__name__)
42
+
43
+ # Blocked sites loaded from config/sites.yaml
44
+ def _load_blocked():
45
+ from divapply.config import load_blocked_sites
46
+ return load_blocked_sites()
47
+
48
+ # How often to poll the DB when the queue is empty (seconds)
49
+ POLL_INTERVAL = config.DEFAULTS["poll_interval"]
50
+
51
+ # Thread-safe shutdown coordination
52
+ _stop_event = threading.Event()
53
+
54
+ # Track active Claude Code processes for skip (Ctrl+C) handling
55
+ _claude_procs: dict[int, subprocess.Popen] = {}
56
+ _claude_lock = threading.Lock()
57
+
58
+ # Register cleanup on exit
59
+ atexit.register(cleanup_on_exit)
60
+ if platform.system() != "Windows":
61
+ signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
62
+
63
+
64
+ # ---------------------------------------------------------------------------
65
+ # MCP config
66
+ # ---------------------------------------------------------------------------
67
+
68
+ def _make_mcp_config(
69
+ cdp_port: int,
70
+ browser: str = "firefox",
71
+ worker_profile_dir: Path | None = None,
72
+ headless: bool = False,
73
+ ) -> dict:
74
+ """Build MCP config dict for a specific browser configuration."""
75
+ playwright_args = ["@playwright/mcp@0.0.70"]
76
+ if browser == "chrome":
77
+ playwright_args.extend([
78
+ f"--cdp-endpoint=http://localhost:{cdp_port}",
79
+ f"--viewport-size={config.DEFAULTS['viewport']}",
80
+ ])
81
+ else:
82
+ playwright_args.extend([
83
+ f"--browser={browser}",
84
+ f"--viewport-size={config.DEFAULTS['viewport']}",
85
+ ])
86
+ if worker_profile_dir is not None:
87
+ playwright_args.append(f"--user-data-dir={worker_profile_dir}")
88
+ if headless:
89
+ playwright_args.append("--headless")
90
+
91
+ return {
92
+ "mcpServers": {
93
+ "playwright": {
94
+ "command": "npx",
95
+ "args": playwright_args,
96
+ },
97
+ "gmail": {
98
+ "command": "npx",
99
+ "args": ["-y", "@gongrzhe/server-gmail-autoauth-mcp"],
100
+ },
101
+ }
102
+ }
103
+
104
+
105
+ # ---------------------------------------------------------------------------
106
+ # Database operations
107
+ # ---------------------------------------------------------------------------
108
+
109
+ def acquire_job(target_url: str | None = None, min_score: int = 7,
110
+ worker_id: int = 0) -> dict | None:
111
+ """Atomically acquire the next job to apply to.
112
+
113
+ Args:
114
+ target_url: Apply to a specific URL instead of picking from queue.
115
+ min_score: Minimum fit_score threshold.
116
+ worker_id: Worker claiming this job (for tracking).
117
+
118
+ Returns:
119
+ Job dict or None if the queue is empty.
120
+ """
121
+ conn = get_connection()
122
+ # Load blocked sites BEFORE acquiring the DB lock to avoid file I/O inside transaction
123
+ blocked_sites, blocked_patterns = _load_blocked()
124
+ try:
125
+ conn.execute("BEGIN IMMEDIATE")
126
+
127
+ if target_url:
128
+ like = f"%{target_url.split('?')[0].rstrip('/')}%"
129
+ row = conn.execute("""
130
+ SELECT url, title, site, application_url, tailored_resume_path,
131
+ fit_score, location, full_description, cover_letter_path
132
+ FROM jobs
133
+ WHERE (url = ? OR application_url = ? OR application_url LIKE ? OR url LIKE ?)
134
+ AND tailored_resume_path IS NOT NULL
135
+ AND (apply_status IS NULL OR apply_status != 'in_progress')
136
+ LIMIT 1
137
+ """, (target_url, target_url, like, like)).fetchone()
138
+ else:
139
+ # Build parameterized filters to avoid SQL injection
140
+ params: list = [min_score]
141
+ site_clause = ""
142
+ if blocked_sites:
143
+ placeholders = ",".join("?" * len(blocked_sites))
144
+ site_clause = f"AND site NOT IN ({placeholders})"
145
+ params.extend(blocked_sites)
146
+ url_clauses = ""
147
+ if blocked_patterns:
148
+ url_clauses = " ".join(f"AND url NOT LIKE ?" for _ in blocked_patterns)
149
+ params.extend(blocked_patterns)
150
+ row = conn.execute(f"""
151
+ SELECT url, title, site, application_url, tailored_resume_path,
152
+ fit_score, location, full_description, cover_letter_path
153
+ FROM jobs
154
+ WHERE tailored_resume_path IS NOT NULL
155
+ AND (apply_status IS NULL OR apply_status = 'failed')
156
+ AND (apply_attempts IS NULL OR apply_attempts < ?)
157
+ AND fit_score >= ?
158
+ {site_clause}
159
+ {url_clauses}
160
+ ORDER BY fit_score DESC, url
161
+ LIMIT 1
162
+ """, [config.DEFAULTS["max_apply_attempts"]] + params).fetchone()
163
+
164
+ if not row:
165
+ conn.rollback()
166
+ return None
167
+
168
+ # Skip manual ATS sites (unsolvable CAPTCHAs)
169
+ from divapply.config import is_manual_ats
170
+ apply_url = row["application_url"] or row["url"]
171
+ if is_manual_ats(apply_url):
172
+ conn.execute(
173
+ "UPDATE jobs SET apply_status = 'manual', apply_error = 'manual ATS' WHERE url = ?",
174
+ (row["url"],),
175
+ )
176
+ conn.commit()
177
+ logger.info("Skipping manual ATS: %s", row["url"][:80])
178
+ return None
179
+
180
+ now = datetime.now(timezone.utc).isoformat()
181
+ conn.execute("""
182
+ UPDATE jobs SET apply_status = 'in_progress',
183
+ agent_id = ?,
184
+ last_attempted_at = ?
185
+ WHERE url = ?
186
+ """, (f"worker-{worker_id}", now, row["url"]))
187
+ conn.commit()
188
+
189
+ return dict(row)
190
+ except Exception:
191
+ conn.rollback()
192
+ raise
193
+
194
+
195
+ def mark_result(url: str, status: str, error: str | None = None,
196
+ permanent: bool = False, duration_ms: int | None = None,
197
+ task_id: str | None = None) -> None:
198
+ """Update a job's apply status in the database."""
199
+ conn = get_connection()
200
+ now = datetime.now(timezone.utc).isoformat()
201
+ if status == "applied":
202
+ conn.execute("""
203
+ UPDATE jobs SET apply_status = 'applied', applied_at = ?,
204
+ apply_error = NULL, agent_id = NULL,
205
+ apply_duration_ms = ?, apply_task_id = ?
206
+ WHERE url = ?
207
+ """, (now, duration_ms, task_id, url))
208
+ else:
209
+ if permanent:
210
+ conn.execute("""
211
+ UPDATE jobs SET apply_status = ?, apply_error = ?,
212
+ apply_attempts = 99, agent_id = NULL,
213
+ apply_duration_ms = ?, apply_task_id = ?
214
+ WHERE url = ?
215
+ """, (status, error or "unknown", duration_ms, task_id, url))
216
+ else:
217
+ conn.execute("""
218
+ UPDATE jobs SET apply_status = ?, apply_error = ?,
219
+ apply_attempts = COALESCE(apply_attempts, 0) + 1,
220
+ agent_id = NULL,
221
+ apply_duration_ms = ?, apply_task_id = ?
222
+ WHERE url = ?
223
+ """, (status, error or "unknown", duration_ms, task_id, url))
224
+ conn.commit()
225
+
226
+
227
+ def release_lock(url: str) -> None:
228
+ """Release the in_progress lock without changing status."""
229
+ conn = get_connection()
230
+ conn.execute(
231
+ "UPDATE jobs SET apply_status = NULL, agent_id = NULL WHERE url = ? AND apply_status = 'in_progress'",
232
+ (url,),
233
+ )
234
+ conn.commit()
235
+
236
+
237
+ # ---------------------------------------------------------------------------
238
+ # Utility modes (--gen, --mark-applied, --mark-failed, --reset-failed)
239
+ # ---------------------------------------------------------------------------
240
+
241
+ def gen_prompt(target_url: str, min_score: int = 7,
242
+ model: str = "sonnet", worker_id: int = 0) -> Path | None:
243
+ """Generate a prompt file and print the Claude CLI command for manual debugging.
244
+
245
+ Returns:
246
+ Path to the generated prompt file, or None if no job found.
247
+ """
248
+ job = acquire_job(target_url=target_url, min_score=min_score, worker_id=worker_id)
249
+ if not job:
250
+ return None
251
+
252
+ # Read resume text
253
+ resume_path = job.get("tailored_resume_path")
254
+ txt_path = Path(resume_path).with_suffix(".txt") if resume_path else None
255
+ resume_text = ""
256
+ if txt_path and txt_path.exists():
257
+ resume_text = txt_path.read_text(encoding="utf-8")
258
+
259
+ prompt = prompt_mod.build_prompt(job=job, tailored_resume=resume_text)
260
+
261
+ # Release the lock so the job stays available
262
+ release_lock(job["url"])
263
+
264
+ # Write prompt file
265
+ config.ensure_dirs()
266
+ site_slug = (job.get("site") or "unknown")[:20].replace(" ", "_")
267
+ prompt_file = config.LOG_DIR / f"prompt_{site_slug}_{job['title'][:30].replace(' ', '_')}.txt"
268
+ prompt_file.write_text(prompt, encoding="utf-8")
269
+
270
+ # Write MCP config for reference
271
+ port = BASE_CDP_PORT + worker_id
272
+ mcp_path = config.APP_DIR / f".mcp-apply-{worker_id}.json"
273
+ mcp_path.write_text(json.dumps(_make_mcp_config(port)), encoding="utf-8")
274
+
275
+ return prompt_file
276
+
277
+
278
+ def mark_job(url: str, status: str, reason: str | None = None) -> None:
279
+ """Manually mark a job's apply status in the database.
280
+
281
+ Args:
282
+ url: Job URL to mark.
283
+ status: Either 'applied' or 'failed'.
284
+ reason: Failure reason (only for status='failed').
285
+ """
286
+ conn = get_connection()
287
+ now = datetime.now(timezone.utc).isoformat()
288
+ if status == "applied":
289
+ conn.execute("""
290
+ UPDATE jobs SET apply_status = 'applied', applied_at = ?,
291
+ apply_error = NULL, agent_id = NULL
292
+ WHERE url = ?
293
+ """, (now, url))
294
+ else:
295
+ conn.execute("""
296
+ UPDATE jobs SET apply_status = 'failed', apply_error = ?,
297
+ apply_attempts = 99, agent_id = NULL
298
+ WHERE url = ?
299
+ """, (reason or "manual", url))
300
+ conn.commit()
301
+
302
+
303
+ def reset_failed() -> int:
304
+ """Reset all failed jobs so they can be retried.
305
+
306
+ Returns:
307
+ Number of jobs reset.
308
+ """
309
+ conn = get_connection()
310
+ cursor = conn.execute("""
311
+ UPDATE jobs SET apply_status = NULL, apply_error = NULL,
312
+ apply_attempts = 0, agent_id = NULL
313
+ WHERE apply_status = 'failed'
314
+ OR (apply_status IS NOT NULL AND apply_status != 'applied'
315
+ AND apply_status != 'in_progress')
316
+ """)
317
+ conn.commit()
318
+ return cursor.rowcount
319
+
320
+
321
+ # ---------------------------------------------------------------------------
322
+ # Per-job execution
323
+ # ---------------------------------------------------------------------------
324
+
325
+ def run_job(job: dict, port: int, worker_id: int = 0,
326
+ model: str = "sonnet", dry_run: bool = False) -> tuple[str, int]:
327
+ """Spawn a Claude Code session for one job application.
328
+
329
+ Returns:
330
+ Tuple of (status_string, duration_ms). Status is one of:
331
+ 'applied', 'expired', 'captcha', 'login_issue',
332
+ 'failed:reason', or 'skipped'.
333
+ """
334
+ # Read tailored resume text
335
+ resume_path = job.get("tailored_resume_path")
336
+ txt_path = Path(resume_path).with_suffix(".txt") if resume_path else None
337
+ resume_text = ""
338
+ if txt_path and txt_path.exists():
339
+ resume_text = txt_path.read_text(encoding="utf-8")
340
+
341
+ # Build the prompt
342
+ agent_prompt = prompt_mod.build_prompt(
343
+ job=job,
344
+ tailored_resume=resume_text,
345
+ dry_run=dry_run,
346
+ )
347
+
348
+ # Write per-worker MCP config
349
+ mcp_config_path = config.APP_DIR / f".mcp-apply-{worker_id}.json"
350
+ mcp_config_path.write_text(json.dumps(_make_mcp_config(port)), encoding="utf-8")
351
+
352
+ # Build claude command
353
+ cmd = [
354
+ "claude",
355
+ "--model", model,
356
+ "-p",
357
+ "--max-turns", "150",
358
+ "--mcp-config", str(mcp_config_path),
359
+ "--permission-mode", "bypassPermissions",
360
+ "--no-session-persistence",
361
+ "--disallowedTools", (
362
+ "mcp__gmail__draft_email,mcp__gmail__modify_email,"
363
+ "mcp__gmail__delete_email,mcp__gmail__download_attachment,"
364
+ "mcp__gmail__batch_modify_emails,mcp__gmail__batch_delete_emails,"
365
+ "mcp__gmail__create_label,mcp__gmail__update_label,"
366
+ "mcp__gmail__delete_label,mcp__gmail__get_or_create_label,"
367
+ "mcp__gmail__list_email_labels,mcp__gmail__create_filter,"
368
+ "mcp__gmail__list_filters,mcp__gmail__get_filter,"
369
+ "mcp__gmail__delete_filter"
370
+ ),
371
+ "--output-format", "stream-json",
372
+ "--verbose", "-",
373
+ ]
374
+
375
+ env = os.environ.copy()
376
+ env.pop("CLAUDECODE", None)
377
+ env.pop("CLAUDE_CODE_ENTRYPOINT", None)
378
+
379
+ worker_dir = reset_worker_dir(worker_id)
380
+
381
+ update_state(worker_id, status="applying", job_title=job["title"],
382
+ company=job.get("site", ""), score=job.get("fit_score", 0),
383
+ start_time=time.time(), actions=0, last_action="starting")
384
+ add_event(f"[W{worker_id}] Starting: {job['title'][:40]} @ {job.get('site', '')}")
385
+
386
+ worker_log = config.LOG_DIR / f"worker-{worker_id}.log"
387
+ ts_header = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
388
+ log_header = (
389
+ f"\n{'=' * 60}\n"
390
+ f"[{ts_header}] {job['title']} @ {job.get('site', '')}\n"
391
+ f"URL: {job.get('application_url') or job['url']}\n"
392
+ f"Score: {job.get('fit_score', 'N/A')}/10\n"
393
+ f"{'=' * 60}\n"
394
+ )
395
+
396
+ start = time.time()
397
+ stats: dict = {}
398
+ proc = None
399
+
400
+ try:
401
+ proc = subprocess.Popen(
402
+ cmd,
403
+ stdin=subprocess.PIPE,
404
+ stdout=subprocess.PIPE,
405
+ stderr=subprocess.STDOUT,
406
+ text=True,
407
+ encoding="utf-8",
408
+ errors="replace",
409
+ env=env,
410
+ cwd=str(worker_dir),
411
+ )
412
+ with _claude_lock:
413
+ _claude_procs[worker_id] = proc
414
+
415
+ proc.stdin.write(agent_prompt)
416
+ proc.stdin.close()
417
+
418
+ text_parts: list[str] = []
419
+ with open(worker_log, "a", encoding="utf-8") as lf:
420
+ lf.write(log_header)
421
+
422
+ for line in proc.stdout:
423
+ line = line.strip()
424
+ if not line:
425
+ continue
426
+ try:
427
+ msg = json.loads(line)
428
+ msg_type = msg.get("type")
429
+ if msg_type == "assistant":
430
+ for block in msg.get("message", {}).get("content", []):
431
+ bt = block.get("type")
432
+ if bt == "text":
433
+ text_parts.append(block["text"])
434
+ lf.write(block["text"] + "\n")
435
+ elif bt == "tool_use":
436
+ name = (
437
+ block.get("name", "")
438
+ .replace("mcp__playwright__", "")
439
+ .replace("mcp__gmail__", "gmail:")
440
+ )
441
+ inp = block.get("input", {})
442
+ if "url" in inp:
443
+ desc = f"{name} {inp['url'][:60]}"
444
+ elif "ref" in inp:
445
+ desc = f"{name} {inp.get('element', inp.get('text', ''))}"[:50]
446
+ elif "fields" in inp:
447
+ desc = f"{name} ({len(inp['fields'])} fields)"
448
+ elif "paths" in inp:
449
+ desc = f"{name} upload"
450
+ else:
451
+ desc = name
452
+
453
+ lf.write(f" >> {desc}\n")
454
+ ws = get_state(worker_id)
455
+ cur_actions = ws.actions if ws else 0
456
+ update_state(worker_id,
457
+ actions=cur_actions + 1,
458
+ last_action=desc[:35])
459
+ elif msg_type == "result":
460
+ stats = {
461
+ "input_tokens": msg.get("usage", {}).get("input_tokens", 0),
462
+ "output_tokens": msg.get("usage", {}).get("output_tokens", 0),
463
+ "cache_read": msg.get("usage", {}).get("cache_read_input_tokens", 0),
464
+ "cache_create": msg.get("usage", {}).get("cache_creation_input_tokens", 0),
465
+ "cost_usd": msg.get("total_cost_usd", 0),
466
+ "turns": msg.get("num_turns", 0),
467
+ }
468
+ text_parts.append(msg.get("result", ""))
469
+ except json.JSONDecodeError:
470
+ text_parts.append(line)
471
+ lf.write(line + "\n")
472
+
473
+ proc.wait(timeout=300)
474
+ returncode = proc.returncode
475
+ proc = None
476
+
477
+ if returncode and returncode < 0:
478
+ return "skipped", int((time.time() - start) * 1000)
479
+
480
+ output = "\n".join(text_parts)
481
+ elapsed = int(time.time() - start)
482
+ duration_ms = int((time.time() - start) * 1000)
483
+
484
+ ts = datetime.now().strftime("%Y%m%d_%H%M%S")
485
+ job_log = config.LOG_DIR / f"claude_{ts}_w{worker_id}_{job.get('site', 'unknown')[:20]}.txt"
486
+ job_log.write_text(output, encoding="utf-8")
487
+
488
+ if stats:
489
+ cost = stats.get("cost_usd", 0)
490
+ ws = get_state(worker_id)
491
+ prev_cost = ws.total_cost if ws else 0.0
492
+ update_state(worker_id, total_cost=prev_cost + cost)
493
+
494
+ def _clean_reason(s: str) -> str:
495
+ return re.sub(r'[*`"]+$', '', s).strip()
496
+
497
+ for result_status in ["APPLIED", "EXPIRED", "CAPTCHA", "LOGIN_ISSUE"]:
498
+ if f"RESULT:{result_status}" in output:
499
+ add_event(f"[W{worker_id}] {result_status} ({elapsed}s): {job['title'][:30]}")
500
+ update_state(worker_id, status=result_status.lower(),
501
+ last_action=f"{result_status} ({elapsed}s)")
502
+ return result_status.lower(), duration_ms
503
+
504
+ # Fuzzy success detection: agent confirmed submission in narrative but omitted RESULT:APPLIED
505
+ # Require strong confirmation phrases (not just "confirmation" which is too vague)
506
+ _output_lower = output.lower()
507
+ _fuzzy_success = any(phrase in _output_lower for phrase in [
508
+ "application submitted", "successfully submitted", "application has been submitted",
509
+ "your application was submitted", "application was received",
510
+ "application is submitted", "submitted successfully",
511
+ "application number", "reference number",
512
+ ])
513
+ _fuzzy_fail_context = any(phrase in _output_lower for phrase in [
514
+ "result:failed", "not_eligible", "login_issue", "captcha",
515
+ "could not submit", "unable to submit", "failed to submit",
516
+ "was not submitted", "not able to submit", "cannot submit",
517
+ "i was unable", "i could not",
518
+ ])
519
+ if _fuzzy_success and not _fuzzy_fail_context:
520
+ add_event(f"[W{worker_id}] APPLIED (fuzzy) ({elapsed}s): {job['title'][:30]}")
521
+ update_state(worker_id, status="applied", last_action=f"APPLIED-fuzzy ({elapsed}s)")
522
+ return "applied", duration_ms
523
+
524
+ if "RESULT:FAILED" in output:
525
+ for out_line in output.split("\n"):
526
+ if "RESULT:FAILED" in out_line:
527
+ parts = out_line.split("RESULT:FAILED:", 1)
528
+ reason = parts[1].strip() if len(parts) > 1 and parts[1].strip() else "unknown"
529
+ reason = _clean_reason(reason)
530
+ PROMOTE_TO_STATUS = {"captcha", "expired", "login_issue"}
531
+ if reason in PROMOTE_TO_STATUS:
532
+ add_event(f"[W{worker_id}] {reason.upper()} ({elapsed}s): {job['title'][:30]}")
533
+ update_state(worker_id, status=reason,
534
+ last_action=f"{reason.upper()} ({elapsed}s)")
535
+ return reason, duration_ms
536
+ add_event(f"[W{worker_id}] FAILED ({elapsed}s): {reason[:30]}")
537
+ update_state(worker_id, status="failed",
538
+ last_action=f"FAILED: {reason[:25]}")
539
+ return f"failed:{reason}", duration_ms
540
+ return "failed:unknown", duration_ms
541
+
542
+ add_event(f"[W{worker_id}] NO RESULT ({elapsed}s)")
543
+ update_state(worker_id, status="failed", last_action=f"no result ({elapsed}s)")
544
+ return "failed:no_result_line", duration_ms
545
+
546
+ except subprocess.TimeoutExpired:
547
+ duration_ms = int((time.time() - start) * 1000)
548
+ elapsed = int(time.time() - start)
549
+ add_event(f"[W{worker_id}] TIMEOUT ({elapsed}s)")
550
+ update_state(worker_id, status="failed", last_action=f"TIMEOUT ({elapsed}s)")
551
+ return "failed:timeout", duration_ms
552
+ except Exception as e:
553
+ duration_ms = int((time.time() - start) * 1000)
554
+ add_event(f"[W{worker_id}] ERROR: {str(e)[:40]}")
555
+ update_state(worker_id, status="failed", last_action=f"ERROR: {str(e)[:25]}")
556
+ return f"failed:{str(e)[:100]}", duration_ms
557
+ finally:
558
+ with _claude_lock:
559
+ _claude_procs.pop(worker_id, None)
560
+ if proc is not None and proc.poll() is None:
561
+ _kill_process_tree(proc.pid)
562
+
563
+
564
+ # ---------------------------------------------------------------------------
565
+ # Permanent failure classification
566
+ # ---------------------------------------------------------------------------
567
+
568
+ PERMANENT_FAILURES: set[str] = {
569
+ "expired", "captcha", "login_issue",
570
+ "not_eligible_location", "not_eligible_salary",
571
+ "already_applied", "account_required",
572
+ "not_a_job_application", "unsafe_permissions",
573
+ "unsafe_verification", "sso_required",
574
+ "site_blocked", "cloudflare_blocked", "blocked_by_cloudflare",
575
+ }
576
+
577
+ PERMANENT_PREFIXES: tuple[str, ...] = ("site_blocked", "cloudflare", "blocked_by")
578
+
579
+
580
+ def _is_permanent_failure(result: str) -> bool:
581
+ """Determine if a failure should never be retried."""
582
+ reason = result.split(":", 1)[-1] if ":" in result else result
583
+ return (
584
+ result in PERMANENT_FAILURES
585
+ or reason in PERMANENT_FAILURES
586
+ or any(reason.startswith(p) for p in PERMANENT_PREFIXES)
587
+ )
588
+
589
+
590
+ # ---------------------------------------------------------------------------
591
+ # Worker loop
592
+ # ---------------------------------------------------------------------------
593
+
594
+ def worker_loop(worker_id: int = 0, limit: int = 1,
595
+ target_url: str | None = None,
596
+ min_score: int = 7, headless: bool = False,
597
+ model: str = "sonnet", dry_run: bool = False) -> tuple[int, int]:
598
+ """Run jobs sequentially until limit is reached or queue is empty.
599
+
600
+ Args:
601
+ worker_id: Numeric worker identifier.
602
+ limit: Max jobs to process (0 = continuous).
603
+ target_url: Apply to a specific URL.
604
+ min_score: Minimum fit_score threshold.
605
+ headless: Run Chrome headless.
606
+ model: Claude model name.
607
+ dry_run: Don't click Submit.
608
+
609
+ Returns:
610
+ Tuple of (applied_count, failed_count).
611
+ """
612
+ applied = 0
613
+ failed = 0
614
+ continuous = limit == 0
615
+ jobs_done = 0
616
+ empty_polls = 0
617
+ port = BASE_CDP_PORT + worker_id
618
+
619
+ while not _stop_event.is_set():
620
+ if not continuous and jobs_done >= limit:
621
+ break
622
+
623
+ update_state(worker_id, status="idle", job_title="", company="",
624
+ last_action="waiting for job", actions=0)
625
+
626
+ job = acquire_job(target_url=target_url, min_score=min_score,
627
+ worker_id=worker_id)
628
+ if not job:
629
+ if not continuous:
630
+ add_event(f"[W{worker_id}] Queue empty")
631
+ update_state(worker_id, status="done", last_action="queue empty")
632
+ break
633
+ empty_polls += 1
634
+ update_state(worker_id, status="idle",
635
+ last_action=f"polling ({empty_polls})")
636
+ if empty_polls == 1:
637
+ add_event(f"[W{worker_id}] Queue empty, polling every {POLL_INTERVAL}s...")
638
+ # Use Event.wait for interruptible sleep
639
+ if _stop_event.wait(timeout=POLL_INTERVAL):
640
+ break # Stop was requested during wait
641
+ continue
642
+
643
+ empty_polls = 0
644
+
645
+ chrome_proc = None
646
+ try:
647
+ add_event(f"[W{worker_id}] Launching Chrome...")
648
+ chrome_proc = launch_chrome(worker_id, port=port, headless=headless)
649
+
650
+ result, duration_ms = run_job(job, port=port, worker_id=worker_id,
651
+ model=model, dry_run=dry_run)
652
+
653
+ if result == "skipped":
654
+ release_lock(job["url"])
655
+ add_event(f"[W{worker_id}] Skipped: {job['title'][:30]}")
656
+ continue
657
+ elif result == "applied":
658
+ mark_result(job["url"], "applied", duration_ms=duration_ms)
659
+ applied += 1
660
+ update_state(worker_id, jobs_applied=applied,
661
+ jobs_done=applied + failed)
662
+ else:
663
+ reason = result.split(":", 1)[-1] if ":" in result else result
664
+ mark_result(job["url"], "failed", reason,
665
+ permanent=_is_permanent_failure(result),
666
+ duration_ms=duration_ms)
667
+ failed += 1
668
+ update_state(worker_id, jobs_failed=failed,
669
+ jobs_done=applied + failed)
670
+
671
+ except KeyboardInterrupt:
672
+ release_lock(job["url"])
673
+ if _stop_event.is_set():
674
+ break
675
+ add_event(f"[W{worker_id}] Job skipped (Ctrl+C)")
676
+ continue
677
+ except Exception as e:
678
+ logger.exception("Worker %d launcher error", worker_id)
679
+ add_event(f"[W{worker_id}] Launcher error: {str(e)[:40]}")
680
+ release_lock(job["url"])
681
+ failed += 1
682
+ update_state(worker_id, jobs_failed=failed)
683
+ finally:
684
+ if chrome_proc:
685
+ cleanup_worker(worker_id, chrome_proc)
686
+
687
+ jobs_done += 1
688
+ if target_url:
689
+ break
690
+
691
+ update_state(worker_id, status="done", last_action="finished")
692
+ return applied, failed
693
+
694
+
695
+ # ---------------------------------------------------------------------------
696
+ # Main entry point (called from cli.py)
697
+ # ---------------------------------------------------------------------------
698
+
699
+ def main(limit: int = 1, target_url: str | None = None,
700
+ min_score: int = 7, headless: bool = False, model: str = "sonnet",
701
+ dry_run: bool = False, continuous: bool = False,
702
+ poll_interval: int = 60, workers: int = 1) -> None:
703
+ """Launch the apply pipeline.
704
+
705
+ Args:
706
+ limit: Max jobs to apply to (0 or with continuous=True means run forever).
707
+ target_url: Apply to a specific URL.
708
+ min_score: Minimum fit_score threshold.
709
+ headless: Run Chrome in headless mode.
710
+ model: Claude model name.
711
+ dry_run: Don't click Submit.
712
+ continuous: Run forever, polling for new jobs.
713
+ poll_interval: Seconds between DB polls when queue is empty.
714
+ workers: Number of parallel workers (default 1).
715
+ """
716
+ global POLL_INTERVAL
717
+ POLL_INTERVAL = poll_interval
718
+ _stop_event.clear()
719
+
720
+ config.ensure_dirs()
721
+ console = Console()
722
+
723
+ if continuous:
724
+ effective_limit = 0
725
+ mode_label = "continuous"
726
+ else:
727
+ effective_limit = limit
728
+ mode_label = f"{limit} jobs"
729
+
730
+ # Initialize dashboard for all workers
731
+ for i in range(workers):
732
+ init_worker(i)
733
+
734
+ worker_label = f"{workers} worker{'s' if workers > 1 else ''}"
735
+ console.print(f"Launching apply pipeline ({mode_label}, {worker_label}, poll every {POLL_INTERVAL}s)...")
736
+ console.print("[dim]Ctrl+C = skip current job(s) | Ctrl+C x2 = stop[/dim]")
737
+
738
+ # Double Ctrl+C handler
739
+ _ctrl_c_count = 0
740
+
741
+ def _sigint_handler(sig, frame):
742
+ nonlocal _ctrl_c_count
743
+ _ctrl_c_count += 1
744
+ if _ctrl_c_count == 1:
745
+ console.print("\n[yellow]Skipping current job(s)... (Ctrl+C again to STOP)[/yellow]")
746
+ # Kill all active Claude processes to skip current jobs
747
+ with _claude_lock:
748
+ for wid, cproc in list(_claude_procs.items()):
749
+ if cproc.poll() is None:
750
+ _kill_process_tree(cproc.pid)
751
+ else:
752
+ console.print("\n[red bold]STOPPING[/red bold]")
753
+ _stop_event.set()
754
+ with _claude_lock:
755
+ for wid, cproc in list(_claude_procs.items()):
756
+ if cproc.poll() is None:
757
+ _kill_process_tree(cproc.pid)
758
+ kill_all_chrome()
759
+ raise KeyboardInterrupt
760
+
761
+ signal.signal(signal.SIGINT, _sigint_handler)
762
+
763
+ try:
764
+ with Live(render_full(), console=console, refresh_per_second=2) as live:
765
+ # Daemon thread for display refresh only (no business logic)
766
+ _dashboard_stop = threading.Event()
767
+
768
+ def _refresh():
769
+ while not _dashboard_stop.is_set():
770
+ live.update(render_full())
771
+ time.sleep(0.5)
772
+
773
+ refresh_thread = threading.Thread(target=_refresh, daemon=True)
774
+ refresh_thread.start()
775
+
776
+ if workers == 1:
777
+ # Single worker — run directly in main thread
778
+ total_applied, total_failed = worker_loop(
779
+ worker_id=0,
780
+ limit=effective_limit,
781
+ target_url=target_url,
782
+ min_score=min_score,
783
+ headless=headless,
784
+ model=model,
785
+ dry_run=dry_run,
786
+ )
787
+ else:
788
+ # Multi-worker — distribute limit across workers
789
+ if effective_limit:
790
+ base = effective_limit // workers
791
+ extra = effective_limit % workers
792
+ limits = [base + (1 if i < extra else 0)
793
+ for i in range(workers)]
794
+ else:
795
+ limits = [0] * workers # continuous mode
796
+
797
+ with ThreadPoolExecutor(max_workers=workers,
798
+ thread_name_prefix="apply-worker") as executor:
799
+ futures = {
800
+ executor.submit(
801
+ worker_loop,
802
+ worker_id=i,
803
+ limit=limits[i],
804
+ target_url=target_url,
805
+ min_score=min_score,
806
+ headless=headless,
807
+ model=model,
808
+ dry_run=dry_run,
809
+ ): i
810
+ for i in range(workers)
811
+ }
812
+
813
+ results: list[tuple[int, int]] = []
814
+ for future in as_completed(futures):
815
+ wid = futures[future]
816
+ try:
817
+ results.append(future.result())
818
+ except Exception:
819
+ logger.exception("Worker %d crashed", wid)
820
+ results.append((0, 0))
821
+
822
+ total_applied = sum(r[0] for r in results)
823
+ total_failed = sum(r[1] for r in results)
824
+
825
+ _dashboard_stop.set()
826
+ refresh_thread.join(timeout=2)
827
+ live.update(render_full())
828
+
829
+ totals = get_totals()
830
+ console.print(
831
+ f"\n[bold]Done: {total_applied} applied, {total_failed} failed "
832
+ f"(${totals['cost']:.3f})[/bold]"
833
+ )
834
+ console.print(f"Logs: {config.LOG_DIR}")
835
+
836
+ except KeyboardInterrupt:
837
+ pass
838
+ finally:
839
+ _stop_event.set()
840
+ kill_all_chrome()
841
+
842
+
843
+ def _clean_result_reason(text: str) -> str:
844
+ return re.sub(r'[*`"]+$', "", text).strip()
845
+
846
+
847
+ def _extract_result(output: str, worker_id: int, job: dict, duration_ms: int) -> tuple[str, int]:
848
+ elapsed = max(1, duration_ms // 1000)
849
+
850
+ for result_status in ["APPLIED", "EXPIRED", "CAPTCHA", "LOGIN_ISSUE"]:
851
+ if f"RESULT:{result_status}" in output:
852
+ add_event(f"[W{worker_id}] {result_status} ({elapsed}s): {job['title'][:30]}")
853
+ update_state(worker_id, status=result_status.lower(),
854
+ last_action=f"{result_status} ({elapsed}s)")
855
+ return result_status.lower(), duration_ms
856
+
857
+ output_lower = output.lower()
858
+ fuzzy_success = any(phrase in output_lower for phrase in [
859
+ "application submitted", "successfully submitted", "application has been submitted",
860
+ "your application was submitted", "application was received",
861
+ "application is submitted", "submitted successfully",
862
+ "application number", "reference number",
863
+ ])
864
+ fuzzy_fail_context = any(phrase in output_lower for phrase in [
865
+ "result:failed", "not_eligible", "login_issue", "captcha",
866
+ "could not submit", "unable to submit", "failed to submit",
867
+ "was not submitted", "not able to submit", "cannot submit",
868
+ "i was unable", "i could not",
869
+ ])
870
+ if fuzzy_success and not fuzzy_fail_context:
871
+ add_event(f"[W{worker_id}] APPLIED (fuzzy) ({elapsed}s): {job['title'][:30]}")
872
+ update_state(worker_id, status="applied", last_action=f"APPLIED-fuzzy ({elapsed}s)")
873
+ return "applied", duration_ms
874
+
875
+ if "RESULT:FAILED" in output:
876
+ for out_line in output.splitlines():
877
+ if "RESULT:FAILED" not in out_line:
878
+ continue
879
+ parts = out_line.split("RESULT:FAILED:", 1)
880
+ reason = parts[1].strip() if len(parts) > 1 and parts[1].strip() else "unknown"
881
+ reason = _clean_result_reason(reason)
882
+ if reason in {"captcha", "expired", "login_issue"}:
883
+ add_event(f"[W{worker_id}] {reason.upper()} ({elapsed}s): {job['title'][:30]}")
884
+ update_state(worker_id, status=reason, last_action=f"{reason.upper()} ({elapsed}s)")
885
+ return reason, duration_ms
886
+ add_event(f"[W{worker_id}] FAILED ({elapsed}s): {reason[:30]}")
887
+ update_state(worker_id, status="failed", last_action=f"FAILED: {reason[:25]}")
888
+ return f"failed:{reason}", duration_ms
889
+ return "failed:unknown", duration_ms
890
+
891
+ add_event(f"[W{worker_id}] NO RESULT ({elapsed}s)")
892
+ update_state(worker_id, status="failed", last_action=f"no result ({elapsed}s)")
893
+ return "failed:no_result_line", duration_ms
894
+
895
+
896
+ def _build_agent_command(
897
+ backend: str,
898
+ model: str,
899
+ mcp_config_path: Path,
900
+ prompt_file: Path,
901
+ ) -> list[str]:
902
+ import shlex
903
+
904
+ if backend == "claude":
905
+ return [
906
+ "claude",
907
+ "--model", model,
908
+ "-p",
909
+ "--max-turns", "150",
910
+ "--mcp-config", str(mcp_config_path),
911
+ "--permission-mode", "bypassPermissions",
912
+ "--no-session-persistence",
913
+ "--disallowedTools", (
914
+ "mcp__gmail__draft_email,mcp__gmail__modify_email,"
915
+ "mcp__gmail__delete_email,mcp__gmail__download_attachment,"
916
+ "mcp__gmail__batch_modify_emails,mcp__gmail__batch_delete_emails,"
917
+ "mcp__gmail__create_label,mcp__gmail__update_label,"
918
+ "mcp__gmail__delete_label,mcp__gmail__get_or_create_label,"
919
+ "mcp__gmail__list_email_labels,mcp__gmail__create_filter,"
920
+ "mcp__gmail__list_filters,mcp__gmail__get_filter,"
921
+ "mcp__gmail__delete_filter"
922
+ ),
923
+ "--output-format", "stream-json",
924
+ "--verbose", "-",
925
+ ]
926
+
927
+ raw_config = json.loads(mcp_config_path.read_text(encoding="utf-8"))
928
+ servers = raw_config.get("mcpServers", {})
929
+ codex_args = [
930
+ "codex",
931
+ "exec",
932
+ "--model",
933
+ model,
934
+ "--full-auto",
935
+ "--skip-git-repo-check",
936
+ ]
937
+
938
+ if (
939
+ os.environ.get("DIVAPPLY_CODEX_OSS", "").strip().lower() in {"1", "true", "yes", "on"}
940
+ or ":" in model
941
+ ):
942
+ codex_args.append("--oss")
943
+
944
+ for server_name, server_cfg in servers.items():
945
+ prefix = f"mcp_servers.{server_name}"
946
+ if "command" in server_cfg:
947
+ codex_args.extend(["-c", f'{prefix}.command={json.dumps(server_cfg["command"])}'])
948
+ if "args" in server_cfg:
949
+ codex_args.extend(["-c", f"{prefix}.args={json.dumps(server_cfg['args'])}"])
950
+ if "env" in server_cfg:
951
+ codex_args.extend(["-c", f"{prefix}.env={json.dumps(server_cfg['env'])}"])
952
+ codex_args.extend(["-c", f"{prefix}.enabled=true"])
953
+ if server_name == "playwright":
954
+ codex_args.extend(["-c", f"{prefix}.required=true"])
955
+
956
+ template = os.environ.get("DIVAPPLY_CODEX_CMD", "").strip()
957
+ if template:
958
+ rendered = template.format(
959
+ model=model,
960
+ mcp_config=mcp_config_path,
961
+ prompt_file=prompt_file,
962
+ )
963
+ return shlex.split(rendered, posix=False)
964
+
965
+ return codex_args
966
+
967
+
968
+ def get_manual_command(backend: str, model: str, prompt_file: Path, mcp_path: Path) -> str:
969
+ """Return a copy-pasteable manual debug command."""
970
+ if backend == "claude":
971
+ return (
972
+ f"claude --model {model} -p --mcp-config {mcp_path} "
973
+ f"--permission-mode bypassPermissions < {prompt_file}"
974
+ )
975
+ cmd = _build_agent_command(backend, model, mcp_path, prompt_file)
976
+ return " ".join(shlex.quote(part) for part in cmd) + f" < {shlex.quote(str(prompt_file))}"
977
+
978
+
979
+ def gen_prompt(target_url: str, min_score: int = 7,
980
+ model: str = "gpt-5.4-mini", worker_id: int = 0,
981
+ backend: str | None = None, browser: str = "firefox",
982
+ headless: bool = False) -> Path | None:
983
+ """Generate a prompt file for manual debugging."""
984
+ job = acquire_job(target_url=target_url, min_score=min_score, worker_id=worker_id)
985
+ if not job:
986
+ return None
987
+
988
+ resume_path = job.get("tailored_resume_path")
989
+ txt_path = Path(resume_path).with_suffix(".txt") if resume_path else None
990
+ resume_text = ""
991
+ if txt_path and txt_path.exists():
992
+ resume_text = txt_path.read_text(encoding="utf-8")
993
+
994
+ prompt = prompt_mod.build_prompt(job=job, tailored_resume=resume_text)
995
+ release_lock(job["url"])
996
+
997
+ config.ensure_dirs()
998
+ site_slug = (job.get("site") or "unknown")[:20].replace(" ", "_")
999
+ prompt_file = config.LOG_DIR / f"prompt_{site_slug}_{job['title'][:30].replace(' ', '_')}.txt"
1000
+ prompt_file.write_text(prompt, encoding="utf-8")
1001
+
1002
+ port = BASE_CDP_PORT + worker_id
1003
+ worker_profile_dir = setup_worker_profile(worker_id, browser)
1004
+ mcp_path = config.APP_DIR / f".mcp-apply-{worker_id}.json"
1005
+ mcp_path.write_text(
1006
+ json.dumps(
1007
+ _make_mcp_config(
1008
+ port,
1009
+ browser=browser,
1010
+ worker_profile_dir=worker_profile_dir,
1011
+ headless=headless,
1012
+ )
1013
+ ),
1014
+ encoding="utf-8",
1015
+ )
1016
+ return prompt_file
1017
+
1018
+
1019
+ def run_job(job: dict, port: int, worker_id: int = 0,
1020
+ model: str = "gpt-5.4-mini", backend: str = "codex",
1021
+ browser: str = "firefox", dry_run: bool = False,
1022
+ headless: bool = False) -> tuple[str, int]:
1023
+ """Run one auto-apply job through the selected backend."""
1024
+ resume_path = job.get("tailored_resume_path")
1025
+ txt_path = Path(resume_path).with_suffix(".txt") if resume_path else None
1026
+ resume_text = ""
1027
+ if txt_path and txt_path.exists():
1028
+ resume_text = txt_path.read_text(encoding="utf-8")
1029
+
1030
+ agent_prompt = prompt_mod.build_prompt(
1031
+ job=job,
1032
+ tailored_resume=resume_text,
1033
+ dry_run=dry_run,
1034
+ )
1035
+
1036
+ worker_profile_dir = setup_worker_profile(worker_id, browser)
1037
+ mcp_config_path = config.APP_DIR / f".mcp-apply-{worker_id}.json"
1038
+ mcp_config_path.write_text(
1039
+ json.dumps(
1040
+ _make_mcp_config(
1041
+ port,
1042
+ browser=browser,
1043
+ worker_profile_dir=worker_profile_dir,
1044
+ headless=headless,
1045
+ )
1046
+ ),
1047
+ encoding="utf-8",
1048
+ )
1049
+
1050
+ worker_dir = reset_worker_dir(worker_id)
1051
+ prompt_file = worker_dir / "apply_prompt.txt"
1052
+ prompt_file.write_text(agent_prompt, encoding="utf-8")
1053
+ cmd = _build_agent_command(backend, model, mcp_config_path, prompt_file)
1054
+
1055
+ env = os.environ.copy()
1056
+ env.pop("CLAUDECODE", None)
1057
+ env.pop("CLAUDE_CODE_ENTRYPOINT", None)
1058
+
1059
+ update_state(worker_id, status="applying", job_title=job["title"],
1060
+ company=job.get("site", ""), score=job.get("fit_score", 0),
1061
+ start_time=time.time(), actions=0, last_action=f"{backend} starting")
1062
+ add_event(
1063
+ f"[W{worker_id}] Starting via {backend}/{browser}: {job['title'][:40]} @ {job.get('site', '')}"
1064
+ )
1065
+
1066
+ worker_log = config.LOG_DIR / f"worker-{worker_id}.log"
1067
+ ts_header = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
1068
+ log_header = (
1069
+ f"\n{'=' * 60}\n"
1070
+ f"[{ts_header}] {job['title']} @ {job.get('site', '')}\n"
1071
+ f"Backend: {backend}\n"
1072
+ f"Browser: {browser}\n"
1073
+ f"URL: {job.get('application_url') or job['url']}\n"
1074
+ f"Score: {job.get('fit_score', 'N/A')}/10\n"
1075
+ f"{'=' * 60}\n"
1076
+ )
1077
+
1078
+ start = time.time()
1079
+ proc = None
1080
+ text_parts: list[str] = []
1081
+ stats: dict = {}
1082
+
1083
+ try:
1084
+ proc = subprocess.Popen(
1085
+ cmd,
1086
+ stdin=subprocess.PIPE,
1087
+ stdout=subprocess.PIPE,
1088
+ stderr=subprocess.STDOUT,
1089
+ text=True,
1090
+ encoding="utf-8",
1091
+ errors="replace",
1092
+ env=env,
1093
+ cwd=str(worker_dir),
1094
+ )
1095
+ with _claude_lock:
1096
+ _claude_procs[worker_id] = proc
1097
+
1098
+ if proc.stdin:
1099
+ proc.stdin.write(agent_prompt)
1100
+ proc.stdin.close()
1101
+
1102
+ with open(worker_log, "a", encoding="utf-8") as lf:
1103
+ lf.write(log_header)
1104
+ for raw_line in proc.stdout or []:
1105
+ line = raw_line.strip()
1106
+ if not line:
1107
+ continue
1108
+ if backend == "claude":
1109
+ try:
1110
+ msg = json.loads(line)
1111
+ except json.JSONDecodeError:
1112
+ msg = None
1113
+ if msg:
1114
+ msg_type = msg.get("type")
1115
+ if msg_type == "assistant":
1116
+ for block in msg.get("message", {}).get("content", []):
1117
+ if block.get("type") == "text":
1118
+ text = block["text"]
1119
+ text_parts.append(text)
1120
+ lf.write(text + "\n")
1121
+ elif block.get("type") == "tool_use":
1122
+ name = (
1123
+ block.get("name", "")
1124
+ .replace("mcp__playwright__", "")
1125
+ .replace("mcp__gmail__", "gmail:")
1126
+ )
1127
+ lf.write(f" >> {name}\n")
1128
+ ws = get_state(worker_id)
1129
+ cur_actions = ws.actions if ws else 0
1130
+ update_state(worker_id, actions=cur_actions + 1, last_action=name[:35])
1131
+ continue
1132
+ if msg_type == "result":
1133
+ stats = {
1134
+ "cost_usd": msg.get("total_cost_usd", 0),
1135
+ "turns": msg.get("num_turns", 0),
1136
+ }
1137
+ result_text = msg.get("result", "")
1138
+ text_parts.append(result_text)
1139
+ lf.write(result_text + "\n")
1140
+ continue
1141
+ text_parts.append(line)
1142
+ lf.write(line + "\n")
1143
+ update_state(worker_id, last_action=line[:35])
1144
+
1145
+ proc.wait(timeout=config.DEFAULTS["apply_timeout"])
1146
+ returncode = proc.returncode
1147
+ proc = None
1148
+
1149
+ duration_ms = int((time.time() - start) * 1000)
1150
+ if returncode and returncode < 0:
1151
+ return "skipped", duration_ms
1152
+
1153
+ output = "\n".join(text_parts)
1154
+ ts = datetime.now().strftime("%Y%m%d_%H%M%S")
1155
+ job_log = config.LOG_DIR / f"apply_agent_{backend}_{ts}_w{worker_id}_{job.get('site', 'unknown')[:20]}.txt"
1156
+ job_log.write_text(output, encoding="utf-8")
1157
+
1158
+ if stats:
1159
+ cost = stats.get("cost_usd", 0)
1160
+ ws = get_state(worker_id)
1161
+ prev_cost = ws.total_cost if ws else 0.0
1162
+ update_state(worker_id, total_cost=prev_cost + cost)
1163
+
1164
+ return _extract_result(output, worker_id, job, duration_ms)
1165
+
1166
+ except subprocess.TimeoutExpired:
1167
+ duration_ms = int((time.time() - start) * 1000)
1168
+ elapsed = max(1, duration_ms // 1000)
1169
+ add_event(f"[W{worker_id}] TIMEOUT ({elapsed}s)")
1170
+ update_state(worker_id, status="failed", last_action=f"TIMEOUT ({elapsed}s)")
1171
+ return "failed:timeout", duration_ms
1172
+ except Exception as e:
1173
+ duration_ms = int((time.time() - start) * 1000)
1174
+ add_event(f"[W{worker_id}] ERROR: {str(e)[:40]}")
1175
+ update_state(worker_id, status="failed", last_action=f"ERROR: {str(e)[:25]}")
1176
+ return f"failed:{str(e)[:100]}", duration_ms
1177
+ finally:
1178
+ with _claude_lock:
1179
+ _claude_procs.pop(worker_id, None)
1180
+ if proc is not None and proc.poll() is None:
1181
+ _kill_process_tree(proc.pid)
1182
+
1183
+
1184
+ def worker_loop(worker_id: int = 0, limit: int = 1,
1185
+ target_url: str | None = None,
1186
+ min_score: int = 7, headless: bool = False,
1187
+ model: str = "gpt-5.4-mini", backend: str = "codex",
1188
+ browser: str = "firefox",
1189
+ dry_run: bool = False) -> tuple[int, int]:
1190
+ """Run jobs sequentially until limit is reached or queue is empty."""
1191
+ applied = 0
1192
+ failed = 0
1193
+ continuous = limit == 0
1194
+ jobs_done = 0
1195
+ empty_polls = 0
1196
+ port = BASE_CDP_PORT + worker_id
1197
+
1198
+ while not _stop_event.is_set():
1199
+ if not continuous and jobs_done >= limit:
1200
+ break
1201
+
1202
+ update_state(worker_id, status="idle", job_title="", company="",
1203
+ last_action="waiting for job", actions=0)
1204
+
1205
+ job = acquire_job(target_url=target_url, min_score=min_score, worker_id=worker_id)
1206
+ if not job:
1207
+ if not continuous:
1208
+ add_event(f"[W{worker_id}] Queue empty")
1209
+ update_state(worker_id, status="done", last_action="queue empty")
1210
+ break
1211
+ empty_polls += 1
1212
+ update_state(worker_id, status="idle", last_action=f"polling ({empty_polls})")
1213
+ if empty_polls == 1:
1214
+ add_event(f"[W{worker_id}] Queue empty, polling every {POLL_INTERVAL}s...")
1215
+ if _stop_event.wait(timeout=POLL_INTERVAL):
1216
+ break
1217
+ continue
1218
+
1219
+ empty_polls = 0
1220
+ chrome_proc = None
1221
+ try:
1222
+ if browser == "chrome":
1223
+ add_event(f"[W{worker_id}] Launching Chrome...")
1224
+ chrome_proc = launch_chrome(worker_id, port=port, headless=headless)
1225
+ else:
1226
+ add_event(f"[W{worker_id}] Preparing {browser} profile...")
1227
+ result, duration_ms = run_job(
1228
+ job,
1229
+ port=port,
1230
+ worker_id=worker_id,
1231
+ model=model,
1232
+ backend=backend,
1233
+ browser=browser,
1234
+ dry_run=dry_run,
1235
+ headless=headless,
1236
+ )
1237
+
1238
+ if result == "skipped":
1239
+ release_lock(job["url"])
1240
+ add_event(f"[W{worker_id}] Skipped: {job['title'][:30]}")
1241
+ continue
1242
+ if result == "applied":
1243
+ mark_result(job["url"], "applied", duration_ms=duration_ms)
1244
+ applied += 1
1245
+ update_state(worker_id, jobs_applied=applied, jobs_done=applied + failed)
1246
+ else:
1247
+ reason = result.split(":", 1)[-1] if ":" in result else result
1248
+ mark_result(job["url"], "failed", reason,
1249
+ permanent=_is_permanent_failure(result),
1250
+ duration_ms=duration_ms)
1251
+ failed += 1
1252
+ update_state(worker_id, jobs_failed=failed, jobs_done=applied + failed)
1253
+ except KeyboardInterrupt:
1254
+ release_lock(job["url"])
1255
+ if _stop_event.is_set():
1256
+ break
1257
+ add_event(f"[W{worker_id}] Job skipped (Ctrl+C)")
1258
+ continue
1259
+ except Exception as e:
1260
+ logger.exception("Worker %d launcher error", worker_id)
1261
+ add_event(f"[W{worker_id}] Launcher error: {str(e)[:40]}")
1262
+ release_lock(job["url"])
1263
+ failed += 1
1264
+ update_state(worker_id, jobs_failed=failed)
1265
+ finally:
1266
+ if chrome_proc:
1267
+ cleanup_worker(worker_id, chrome_proc)
1268
+
1269
+ jobs_done += 1
1270
+ if target_url:
1271
+ break
1272
+
1273
+ update_state(worker_id, status="done", last_action="finished")
1274
+ return applied, failed
1275
+
1276
+
1277
+ def main(limit: int = 1, target_url: str | None = None,
1278
+ min_score: int = 7, headless: bool = False,
1279
+ model: str = "gpt-5.4-mini", backend: str = "codex",
1280
+ browser: str = "firefox",
1281
+ dry_run: bool = False, continuous: bool = False,
1282
+ poll_interval: int = 60, workers: int = 1) -> None:
1283
+ """Launch the apply pipeline."""
1284
+ global POLL_INTERVAL
1285
+ POLL_INTERVAL = poll_interval
1286
+ _stop_event.clear()
1287
+
1288
+ config.ensure_dirs()
1289
+ console = Console()
1290
+
1291
+ effective_limit = 0 if continuous else limit
1292
+ mode_label = "continuous" if continuous else f"{limit} jobs"
1293
+
1294
+ for i in range(workers):
1295
+ init_worker(i)
1296
+
1297
+ worker_label = f"{workers} worker{'s' if workers > 1 else ''}"
1298
+ console.print(
1299
+ f"Launching apply pipeline ({mode_label}, {worker_label}, "
1300
+ f"backend={backend}, browser={browser}, poll every {POLL_INTERVAL}s)..."
1301
+ )
1302
+ console.print("[dim]Ctrl+C = skip current job(s) | Ctrl+C x2 = stop[/dim]")
1303
+
1304
+ ctrl_c_count = 0
1305
+
1306
+ def _sigint_handler(sig, frame):
1307
+ nonlocal ctrl_c_count
1308
+ ctrl_c_count += 1
1309
+ if ctrl_c_count == 1:
1310
+ console.print("\n[yellow]Skipping current job(s)... (Ctrl+C again to STOP)[/yellow]")
1311
+ with _claude_lock:
1312
+ for cproc in list(_claude_procs.values()):
1313
+ if cproc.poll() is None:
1314
+ _kill_process_tree(cproc.pid)
1315
+ else:
1316
+ console.print("\n[red bold]STOPPING[/red bold]")
1317
+ _stop_event.set()
1318
+ with _claude_lock:
1319
+ for cproc in list(_claude_procs.values()):
1320
+ if cproc.poll() is None:
1321
+ _kill_process_tree(cproc.pid)
1322
+ kill_all_chrome()
1323
+ raise KeyboardInterrupt
1324
+
1325
+ signal.signal(signal.SIGINT, _sigint_handler)
1326
+
1327
+ try:
1328
+ with Live(render_full(), console=console, refresh_per_second=2) as live:
1329
+ dashboard_stop = threading.Event()
1330
+
1331
+ def _refresh():
1332
+ while not dashboard_stop.is_set():
1333
+ live.update(render_full())
1334
+ time.sleep(0.5)
1335
+
1336
+ refresh_thread = threading.Thread(target=_refresh, daemon=True)
1337
+ refresh_thread.start()
1338
+
1339
+ if workers == 1:
1340
+ total_applied, total_failed = worker_loop(
1341
+ worker_id=0,
1342
+ limit=effective_limit,
1343
+ target_url=target_url,
1344
+ min_score=min_score,
1345
+ headless=headless,
1346
+ model=model,
1347
+ backend=backend,
1348
+ browser=browser,
1349
+ dry_run=dry_run,
1350
+ )
1351
+ else:
1352
+ if effective_limit:
1353
+ base = effective_limit // workers
1354
+ extra = effective_limit % workers
1355
+ limits = [base + (1 if i < extra else 0) for i in range(workers)]
1356
+ else:
1357
+ limits = [0] * workers
1358
+
1359
+ with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="apply-worker") as executor:
1360
+ futures = {
1361
+ executor.submit(
1362
+ worker_loop,
1363
+ worker_id=i,
1364
+ limit=limits[i],
1365
+ target_url=target_url,
1366
+ min_score=min_score,
1367
+ headless=headless,
1368
+ model=model,
1369
+ backend=backend,
1370
+ browser=browser,
1371
+ dry_run=dry_run,
1372
+ ): i
1373
+ for i in range(workers)
1374
+ }
1375
+
1376
+ results: list[tuple[int, int]] = []
1377
+ for future in as_completed(futures):
1378
+ wid = futures[future]
1379
+ try:
1380
+ results.append(future.result())
1381
+ except Exception:
1382
+ logger.exception("Worker %d crashed", wid)
1383
+ results.append((0, 0))
1384
+
1385
+ total_applied = sum(r[0] for r in results)
1386
+ total_failed = sum(r[1] for r in results)
1387
+
1388
+ dashboard_stop.set()
1389
+ refresh_thread.join(timeout=2)
1390
+ live.update(render_full())
1391
+
1392
+ totals = get_totals()
1393
+ console.print(
1394
+ f"\n[bold]Done: {total_applied} applied, {total_failed} failed "
1395
+ f"(${totals['cost']:.3f})[/bold]"
1396
+ )
1397
+ console.print(f"Logs: {config.LOG_DIR}")
1398
+
1399
+ except KeyboardInterrupt:
1400
+ pass
1401
+ finally:
1402
+ _stop_event.set()
1403
+ kill_all_chrome()
1404
+