jobhunter-cli 1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of jobhunter-cli might be problematic. Click here for more details.

jobhunter/models.py ADDED
@@ -0,0 +1,81 @@
1
+ from pydantic import BaseModel, Field, computed_field
2
+
3
+ from jobhunter.core.settings import settings
4
+
5
+
6
+ class Job(BaseModel):
7
+ source: str
8
+ board: str
9
+ company: str
10
+ job_id: str
11
+ title: str
12
+ url: str
13
+ location: str
14
+ remote: bool | None = None
15
+ posted: str = ""
16
+ description: str
17
+
18
+ @computed_field
19
+ @property
20
+ def uid(self) -> str:
21
+ """Dynamically generates a unique identifier for the job."""
22
+ return f"{self.source}:{self.board}:{self.job_id or self.url}"
23
+
24
+
25
+ class Candidate(BaseModel):
26
+ uid: str = Field(..., description="Unique compound identifier: source:board:job_id_or_url")
27
+ priority: bool = False
28
+ stack_hits: int = 0
29
+ status: str = Field(default="pending", description="Pipeline status: pending, saved, applied, rejected")
30
+ source: str = ""
31
+ board: str = ""
32
+ company: str
33
+ job_id: str = ""
34
+ title: str
35
+ url: str
36
+ location: str
37
+ remote: bool | None = None
38
+ posted: str = ""
39
+ description: str = ""
40
+
41
+ @classmethod
42
+ def from_job(cls, job: Job, priority: bool, stack_hits: int) -> Candidate:
43
+ """Factory method to convert a raw scraped Job into a stored Candidate."""
44
+ max_chars = settings.limits.description_max_chars
45
+ truncated_description = (job.description or "No description provided.")[:max_chars]
46
+
47
+ return cls(
48
+ uid=job.uid,
49
+ priority=priority,
50
+ stack_hits=stack_hits,
51
+ status="pending",
52
+ source=job.source,
53
+ board=job.board,
54
+ company=job.company,
55
+ job_id=job.job_id,
56
+ title=job.title,
57
+ url=job.url,
58
+ location=job.location,
59
+ remote=job.remote,
60
+ posted=job.posted,
61
+ description=truncated_description,
62
+ )
63
+
64
+
65
+ class EvaluationResult(BaseModel):
66
+ """The exact JSON schema the LLM must return."""
67
+
68
+ reasoning: str = Field(description="A brief 2-3 sentence thought process analyzing the match before scoring.")
69
+ hard_fail: bool = Field(description="True if the job hits any hard gate or dealbreaker.")
70
+ hard_fail_reason: str = Field(default="", description="If hard_fail is true, a max 12-word explanation. Otherwise empty.")
71
+ strengths: list[str] = Field(description="2-3 short items showing requirements explicitly met by the CV.")
72
+ gaps: list[str] = Field(default_factory=list, description="Missing requirements or weak points compared to the CV.")
73
+ score: int = Field(description="Final match score (0-100) based on the rubric.")
74
+ score_explanation: str = Field(description="A 1-2 sentence explanation of exactly why the job passed or failed the score threshold.")
75
+ pitch: str = Field(default="", description="A short, one-sentence first-person pitch. Empty if score is below threshold.")
76
+
77
+
78
+ class EvaluatedCandidate(Candidate):
79
+ """A candidate that has been evaluated by the AI."""
80
+
81
+ evaluation: EvaluationResult
File without changes
@@ -0,0 +1,284 @@
1
+ import concurrent.futures
2
+ import logging
3
+ import time
4
+
5
+ import openai
6
+ from openai import OpenAI
7
+ from pydantic import ValidationError
8
+ from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
9
+ from rich.table import Column
10
+
11
+ from jobhunter.core.logger import console
12
+ from jobhunter.core.paths import DATA_PATH, read, write
13
+ from jobhunter.core.settings import settings
14
+ from jobhunter.models import Candidate, EvaluatedCandidate, EvaluationResult
15
+ from jobhunter.tools.prompt import build_system_prompt, build_user_prompt
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+
20
+ def setup_llm_client() -> OpenAI:
21
+ """Initializes the OpenAI client (supports local Ollama/vLLM via base_url)."""
22
+ logger.info("Initializing LLM client...")
23
+
24
+ if not settings.llm_api_key:
25
+ logger.error("LLM API key is missing. Set LLM_API_KEY in your .env file.")
26
+ console.print("[bold red]Error: LLM API key is missing. Set LLM_API_KEY in your .env file.[/bold red]")
27
+ raise ValueError("LLM_API_KEY is empty. Check your .env file.")
28
+
29
+ return OpenAI(
30
+ api_key=settings.llm_api_key,
31
+ base_url=settings.llm.base_url,
32
+ )
33
+
34
+
35
+ def build_evaluation_tasks(job_uid: str | None = None, limit: int | None = None) -> list[Candidate]:
36
+ """Loads candidates from disk and filters for those pending evaluation."""
37
+ logger.info("Building evaluation tasks...")
38
+
39
+ raw_candidates = read(DATA_PATH / "candidates.json", list)
40
+ raw_evals = read(DATA_PATH / "evaluated.json", list)
41
+
42
+ evaluated_uids = {item.get("uid") for item in raw_evals if "uid" in item}
43
+
44
+ tasks = []
45
+ for item in raw_candidates:
46
+ try:
47
+ c = Candidate(**item)
48
+
49
+ if job_uid:
50
+ if c.uid == job_uid:
51
+ tasks.append(c)
52
+ break
53
+ continue
54
+
55
+ if c.uid not in evaluated_uids:
56
+ tasks.append(c)
57
+ except Exception as e:
58
+ logger.error(f"Failed to parse candidate {item.get('uid', 'unknown')}: {e}")
59
+ console.print(f" [red]✖ Failed to parse candidate {item.get('uid', 'unknown')}[/red]")
60
+
61
+ final_limit = limit or settings.limits.max_candidates_per_run
62
+ if final_limit and final_limit > 0:
63
+ tasks = tasks[:final_limit]
64
+
65
+ logger.info(f"Generated {len(tasks)} evaluation tasks from {len(raw_candidates)} total candidates.")
66
+
67
+ return tasks
68
+
69
+
70
+ def evaluate_single_job(client: OpenAI, system_prompt: str, candidate: Candidate, retries: int = 5, backoff: float = 1.5) -> tuple[Candidate, EvaluationResult | None]:
71
+ """Calls the LLM to evaluate a single job and parses the result with automatic retry and JSON cleanup fallbacks."""
72
+ user_prompt = build_user_prompt(candidate)
73
+
74
+ for attempt in range(retries):
75
+ raw_json = ""
76
+
77
+ try:
78
+ response = client.chat.completions.create(
79
+ model=settings.llm.model or "gpt-4o-mini",
80
+ messages=[{"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}],
81
+ response_format={"type": "json_object"},
82
+ temperature=settings.llm.temperature,
83
+ timeout=settings.llm.timeout,
84
+ )
85
+
86
+ raw_json = response.choices[0].message.content or ""
87
+ if not raw_json:
88
+ raise ValueError("LLM returned empty content instead of JSON.")
89
+
90
+ try:
91
+ eval_result = EvaluationResult.model_validate_json(raw_json)
92
+ return candidate, eval_result
93
+
94
+ except ValidationError:
95
+ cleaned_json = raw_json.strip()
96
+ if cleaned_json.startswith("```json"):
97
+ cleaned_json = cleaned_json[7:]
98
+ if cleaned_json.endswith("```"):
99
+ cleaned_json = cleaned_json[:-3]
100
+ cleaned_json = cleaned_json.strip()
101
+
102
+ eval_result = EvaluationResult.model_validate_json(cleaned_json)
103
+ return candidate, eval_result
104
+
105
+ except openai.RateLimitError as rle:
106
+ if attempt < retries - 1:
107
+ sleep_time = backoff * (2**attempt)
108
+ logger.warning(f"Rate limit hit (429) for {candidate.company}. Retrying in {sleep_time}s (Attempt {attempt + 1}/{retries})...")
109
+ time.sleep(sleep_time)
110
+ else:
111
+ logger.error(f"Rate limit exceeded max retries for {candidate.company}: {rle}")
112
+ console.print(f" [red]✖ Rate limit exceeded for {candidate.company} after {retries} retries.[/red]")
113
+ return candidate, None
114
+
115
+ except openai.APIStatusError as ase:
116
+ if ase.status_code >= 500 and attempt < retries - 1:
117
+ sleep_time = backoff * (2**attempt)
118
+ time.sleep(sleep_time)
119
+ else:
120
+ console.print("\n───────── API ERROR DETAILS ─────────")
121
+ console.print(ase.response.json() if hasattr(ase, "response") else str(ase))
122
+ console.print("───────────────────────────────────────\n")
123
+ return candidate, None
124
+
125
+ except ValidationError as ve:
126
+ logger.error(f"AI returned invalid schema for {candidate.company}: {ve}")
127
+ console.print(f" [red]✖ AI returned invalid schema for {candidate.company}[/red]")
128
+ logger.debug(f"Offending raw JSON for {candidate.company}:\n{raw_json}")
129
+ return candidate, None
130
+
131
+ except Exception as e:
132
+ logger.error(f"LLM API error evaluating {candidate.company}: {e}")
133
+ console.print(f" [red]✖ LLM API error evaluating {candidate.company}: {e}[/red]")
134
+ return candidate, None
135
+
136
+ return candidate, None
137
+
138
+
139
+ def execute_evaluation_tasks(client: OpenAI, tasks: list[Candidate]) -> list[tuple[Candidate, EvaluationResult | None]]:
140
+ """Executes the AI evaluation tasks concurrently with a Rich progress bar."""
141
+ fetched_evals: list[tuple[Candidate, EvaluationResult | None]] = []
142
+ system_prompt = build_system_prompt()
143
+ target_total = len(tasks)
144
+
145
+ workers = max(1, min(settings.workers.evaluation_workers, 20))
146
+
147
+ logger.info(f"Executing evaluation tasks across {target_total} targets with {workers} workers.")
148
+ console.print(f"[bold cyan]Evaluating {target_total} jobs with AI...[/bold cyan]")
149
+
150
+ with Progress(
151
+ SpinnerColumn("dots", style="cyan"),
152
+ TextColumn("{task.description}", table_column=Column(ratio=1)),
153
+ TextColumn("[dim]{task.completed}/{task.total}[/dim]"),
154
+ BarColumn(bar_width=24, style="dim cyan", complete_style="green", finished_style="bold green"),
155
+ TaskProgressColumn(),
156
+ expand=True,
157
+ console=console,
158
+ ) as progress:
159
+ eval_task = progress.add_task("[cyan]Starting evaluation...", total=target_total)
160
+
161
+ with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
162
+ future_to_candidate = {executor.submit(evaluate_single_job, client, system_prompt, candidate): candidate for candidate in tasks}
163
+
164
+ for future in concurrent.futures.as_completed(future_to_candidate):
165
+ candidate = future_to_candidate[future]
166
+ name = candidate.company or "Unknown"
167
+
168
+ try:
169
+ result = future.result(timeout=settings.llm.timeout + 10)
170
+ fetched_evals.append(result)
171
+
172
+ if result[1] is not None:
173
+ score = result[1].score
174
+ color = "green" if score >= settings.evaluating.score_threshold else "yellow"
175
+ progress.update(eval_task, advance=1, description=f"[cyan]Evaluating... [dim](Finished {name} - [{color}]{score}[/{color}][/dim])")
176
+ else:
177
+ progress.update(eval_task, advance=1)
178
+ progress.console.print(f" [dim red]⚠ API/Validation Error for {name}. Skipping...[/dim red]")
179
+
180
+ except concurrent.futures.TimeoutError:
181
+ logger.error(f"Task for {name} timed out.")
182
+ console.print(f" [red]✖ Evaluation for {name} timed out.[/red]")
183
+ progress.update(eval_task, advance=1)
184
+
185
+ except Exception as e:
186
+ logger.exception(f"Error executing evaluation task for {name}: {e}")
187
+ console.print(f" [red]✖ Error evaluating {name}: {e}[/red]")
188
+ progress.update(eval_task, advance=1)
189
+
190
+ if settings.llm.request_delay > 0:
191
+ time.sleep(settings.llm.request_delay)
192
+
193
+ progress.update(eval_task, description="[bold green]Evaluation complete![/bold green]")
194
+
195
+ success_count = sum(1 for _, eval_result in fetched_evals if eval_result is not None)
196
+
197
+ logger.info(f"Successfully evaluated {success_count} jobs.")
198
+
199
+ return fetched_evals
200
+
201
+
202
+ def format_evaluation_results(fetched_evals: list[tuple[Candidate, EvaluationResult | None]]) -> tuple[list[EvaluatedCandidate], int, int, int, int]:
203
+ """Formats the AI results into EvaluatedCandidate models and calculates metrics."""
204
+ logger.info("Formatting LLM results and calculating statistics...")
205
+ console.print("[dim cyan]Formatting evaluations...[/dim cyan]")
206
+
207
+ passed = hard_failed = low_score = errors = 0
208
+ final_evaluations: list[EvaluatedCandidate] = []
209
+
210
+ for candidate, evaluation in fetched_evals:
211
+ if not evaluation:
212
+ errors += 1
213
+ continue
214
+
215
+ candidate.status = "evaluated"
216
+
217
+ evaluated_candidate = EvaluatedCandidate(**candidate.model_dump(), evaluation=evaluation)
218
+
219
+ if evaluation.hard_fail:
220
+ hard_failed += 1
221
+ logger.debug(f"Candidate {candidate.company} hard failed gates.")
222
+ elif evaluation.score < settings.evaluating.score_threshold:
223
+ low_score += 1
224
+ logger.debug(f"Candidate {candidate.company} failed score threshold ({evaluation.score}).")
225
+ else:
226
+ passed += 1
227
+
228
+ final_evaluations.append(evaluated_candidate)
229
+
230
+ logger.info(f"Formatting complete. {passed} passed, {hard_failed} hard failed, {low_score} low score, {errors} errors.")
231
+
232
+ return final_evaluations, passed, hard_failed, low_score, errors
233
+
234
+
235
+ def save_evaluation_results(final_evaluations: list[EvaluatedCandidate]):
236
+ """Saves evaluated candidates to disk, preventing duplicate UIDs, and sorts by score."""
237
+ logger.info("Saving evaluations...")
238
+ console.print("[dim cyan]Saving evaluations...[/dim cyan]")
239
+
240
+ existing_evals = read(DATA_PATH / "evaluated.json", list)
241
+
242
+ new_uids = {c.uid for c in final_evaluations}
243
+ filtered_evals = [e for e in existing_evals if e.get("uid") not in new_uids]
244
+
245
+ new_eval_dicts = [c.model_dump(mode="json") for c in final_evaluations]
246
+
247
+ filtered_evals.extend(new_eval_dicts)
248
+ filtered_evals.sort(key=lambda x: x.get("evaluation", {}).get("score", -1), reverse=True)
249
+
250
+ write(DATA_PATH / "evaluated.json", filtered_evals)
251
+
252
+ logger.info(f"Successfully saved {len(final_evaluations)} evaluations to disk.")
253
+
254
+
255
+ def main(job_uid: str | None = None, limit: int | None = None):
256
+ """Orchestrates the AI evaluation pipeline."""
257
+ logger.info("Starting AI Evaluation Pipeline run.")
258
+
259
+ client = setup_llm_client()
260
+
261
+ tasks = build_evaluation_tasks(job_uid, limit)
262
+ if not tasks:
263
+ logger.warning("No active candidates found to evaluate.")
264
+ console.print("[yellow]No active candidates found to evaluate.[/yellow]")
265
+ return
266
+
267
+ fetched_evals = execute_evaluation_tasks(client, tasks)
268
+ final_evaluations, passed, hard_failed, low_score, errors = format_evaluation_results(fetched_evals)
269
+
270
+ if final_evaluations:
271
+ save_evaluation_results(final_evaluations)
272
+
273
+ total_successful = passed + hard_failed + low_score
274
+ logger.info(f"Pipeline finished. Evaluated: {total_successful} | Passed: {passed} | Hard Failed: {hard_failed} | Low Score: {low_score} | Errors: {errors}")
275
+
276
+ console.print("\n[bold]AI Evaluation Summary[/bold]")
277
+ console.print("───────────────────────────────────────")
278
+ console.print(f"Total evaluated : [cyan]{total_successful}[/cyan]")
279
+ console.print(f"Passed : [bold green]{passed}[/bold green]")
280
+ console.print(f"Failed (Gates) : [red]{hard_failed}[/red]")
281
+ console.print(f"Failed (Score) : [yellow]{low_score}[/yellow]")
282
+ if errors:
283
+ console.print(f"API Errors : [bold red]{errors}[/bold red]")
284
+ console.print("───────────────────────────────────────\n")
@@ -0,0 +1,233 @@
1
+ import concurrent.futures
2
+ import logging
3
+
4
+ from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
5
+ from rich.table import Column
6
+
7
+ from jobhunter.core.filters import count_stack_hits, passes_all_filters
8
+ from jobhunter.core.logger import console
9
+ from jobhunter.core.paths import DATA_PATH, read, write
10
+ from jobhunter.core.settings import settings
11
+ from jobhunter.models import Candidate, Job
12
+ from jobhunter.sources import SOURCES
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+
17
+ def build_fetch_tasks(source_name: str | None, no_aggregators: bool) -> list[tuple]:
18
+ """Generates a list of (source, slug) tuples to be scraped."""
19
+ logger.info("Building fetch tasks...")
20
+
21
+ tasks = []
22
+ for name, source in SOURCES.items():
23
+ if source_name and name != source_name:
24
+ continue
25
+
26
+ is_aggregator = getattr(source, "is_aggregator", False)
27
+
28
+ if is_aggregator and not no_aggregators:
29
+ tasks.append((source, ""))
30
+ else:
31
+ slugs = settings.companies.get(name, [])
32
+ for slug in slugs:
33
+ tasks.append((source, slug))
34
+
35
+ logger.info(f"Generated {len(tasks)} fetch tasks.")
36
+
37
+ return tasks
38
+
39
+
40
+ def execute_fetch_tasks(tasks: list[tuple]) -> list[Job]:
41
+ """Executes the scraping tasks concurrently with a Rich progress bar."""
42
+ fetched_jobs: list[Job] = []
43
+ target_total = len(tasks)
44
+
45
+ workers = max(1, min(settings.workers.fetch_workers, 20))
46
+
47
+ logger.info(f"Executing fetch tasks across {target_total} targets with {workers} workers.")
48
+ console.print("[bold cyan]Fetching jobs from configured sources...[/bold cyan]")
49
+
50
+ with Progress(
51
+ SpinnerColumn("dots", style="cyan"),
52
+ TextColumn("{task.description}", table_column=Column(ratio=1)),
53
+ TextColumn("[dim]{task.completed}/{task.total}[/dim]"),
54
+ BarColumn(bar_width=24, style="dim cyan", complete_style="green", finished_style="bold green"),
55
+ TaskProgressColumn(),
56
+ expand=True,
57
+ console=console,
58
+ ) as progress:
59
+ fetch_task = progress.add_task("[cyan]Starting fetch...", total=target_total)
60
+
61
+ with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
62
+ future_to_name = {}
63
+ for source, slug in tasks:
64
+ name = f"{source.name}:{slug}" if slug else source.name
65
+ future = executor.submit(source.execute_scraper, slug)
66
+ future_to_name[future] = name
67
+
68
+ for future in concurrent.futures.as_completed(future_to_name):
69
+ name = future_to_name[future]
70
+
71
+ try:
72
+ result = future.result(timeout=60)
73
+ if result:
74
+ fetched_jobs.extend(result)
75
+
76
+ progress.update(fetch_task, advance=1, description=f"[cyan]Fetching... [dim](Finished {name})[/dim]")
77
+
78
+ except concurrent.futures.TimeoutError:
79
+ logger.error(f"Task for {name} timed out.")
80
+ console.print(f" [red]✖ Task for {name} timed out.[/red]")
81
+ progress.update(fetch_task, advance=1)
82
+
83
+ except Exception as e:
84
+ logger.exception(f"Error executing fetch task for {name}: {e}")
85
+ console.print(f" [red]✖ Error fetching {name}: {e}[/red]")
86
+ progress.update(fetch_task, advance=1)
87
+
88
+ progress.update(fetch_task, description="[bold green]Fetch complete![/bold green]")
89
+
90
+ logger.info(f"Successfully fetched {len(fetched_jobs)} raw jobs.")
91
+
92
+ return fetched_jobs
93
+
94
+
95
+ def filter_fetch_results(jobs: list[Job], ignore_seen: bool) -> tuple[list[Job], int, int, int, int]:
96
+ """Removes previously seen jobs, cross-source duplicates, and jobs failing filters."""
97
+ logger.info("Filtering and deduplicating jobs...")
98
+ console.print("[dim cyan]Filtering and deduplicating jobs...[/dim cyan]")
99
+
100
+ raw_candidates = read(DATA_PATH / "candidates.json", list)
101
+ seen_data = read(DATA_PATH / "seen.json", dict)
102
+
103
+ fetched_uids = {item.get("uid") for item in raw_candidates if "uid" in item}
104
+
105
+ skipped_pend = skipped_seen = skipped_dupes = skipped_filters = 0
106
+ run_duped_roles = set()
107
+ passing_jobs = []
108
+
109
+ for job in jobs:
110
+ if job.uid in fetched_uids:
111
+ skipped_pend += 1
112
+ continue
113
+
114
+ if job.uid in seen_data and not ignore_seen:
115
+ skipped_seen += 1
116
+ continue
117
+
118
+ company = (job.company or "Unknown").strip().lower()
119
+ title = (job.title or "Unknown").strip().lower()
120
+ location = (job.location or "Unknown").strip().lower()
121
+ role_key = f"{company}:{title}:{location}"
122
+
123
+ if role_key in run_duped_roles:
124
+ skipped_dupes += 1
125
+ continue
126
+
127
+ run_duped_roles.add(role_key)
128
+
129
+ if passes_all_filters(job):
130
+ passing_jobs.append(job)
131
+ else:
132
+ skipped_filters += 1
133
+
134
+ logger.info(f"Filtering complete: {len(passing_jobs)} passed, {skipped_pend} skipped (pend), {skipped_seen} skipped (seen), {skipped_dupes} dupes, {skipped_filters} failed filters.")
135
+
136
+ return passing_jobs, skipped_pend, skipped_seen, skipped_dupes, skipped_filters
137
+
138
+
139
+ def score_fetch_results(jobs: list[Job]) -> list[Candidate]:
140
+ """Calculates scores, sorts candidates, and converts Jobs into Candidates."""
141
+ logger.info("Scoring and sorting passing jobs...")
142
+ console.print("[dim cyan]Scoring and sorting candidates...[/dim cyan]")
143
+
144
+ priority_companies = {c.lower() for c in settings.filters.company_priority}
145
+
146
+ scored_candidates: list[Candidate] = []
147
+ for job in jobs:
148
+ is_priority = (job.company or "").lower() in priority_companies
149
+ hits = count_stack_hits(job)
150
+ candidate = Candidate.from_job(job, priority=is_priority, stack_hits=hits)
151
+ scored_candidates.append(candidate)
152
+
153
+ scored_candidates.sort(key=lambda c: (c.priority, c.stack_hits), reverse=True)
154
+
155
+ return scored_candidates
156
+
157
+
158
+ def limit_fetch_results(scored_candidates: list[Candidate], limit: int | None = None) -> tuple[list[Candidate], int]:
159
+ """Enforces the max candidate limit."""
160
+ logger.info("Enforcing limits...")
161
+ console.print("[dim cyan]Enforcing candidate limits...[/dim cyan]")
162
+
163
+ final_limit = limit or settings.limits.max_candidates_per_run
164
+
165
+ final_candidates = []
166
+ non_priority_count = 0
167
+
168
+ for candidate in scored_candidates:
169
+ if not candidate.priority:
170
+ if non_priority_count >= final_limit:
171
+ continue
172
+ non_priority_count += 1
173
+
174
+ final_candidates.append(candidate)
175
+
176
+ cut_over_cap = len(scored_candidates) - len(final_candidates)
177
+
178
+ logger.info(f"Generated {len(final_candidates)} candidates. Cut {cut_over_cap} jobs over the cap.")
179
+
180
+ return final_candidates, cut_over_cap
181
+
182
+
183
+ def save_fetch_results(final_candidates: list[Candidate]):
184
+ """Saves finalized candidates into disk, preventing duplicate UIDs."""
185
+ logger.info("Saving candidates...")
186
+ console.print("[dim cyan]Saving candidates...[/dim cyan]")
187
+
188
+ existing_candidates = read(DATA_PATH / "candidates.json", list)
189
+
190
+ new_uids = {c.uid for c in final_candidates}
191
+ filtered_existing = [c for c in existing_candidates if c.get("uid") not in new_uids]
192
+
193
+ new_candidate_dicts = [c.model_dump(mode="json") for c in final_candidates]
194
+
195
+ filtered_existing.extend(new_candidate_dicts)
196
+ filtered_existing.sort(key=lambda x: (x.get("priority", False), x.get("stack_hits", 0)), reverse=True)
197
+
198
+ write(DATA_PATH / "candidates.json", filtered_existing)
199
+
200
+ logger.info(f"Successfully saved {len(final_candidates)} candidates to disk.")
201
+
202
+
203
+ def main(source_name: str | None = None, no_aggregators: bool = False, ignore_seen: bool = False, limit: int | None = None):
204
+ """Orchestrates the job fetching pipeline."""
205
+ logger.info("Starting job fetch pipeline run.")
206
+
207
+ tasks = build_fetch_tasks(source_name, no_aggregators)
208
+ if not tasks:
209
+ logger.warning("No active sources/companies found to scrape.")
210
+ console.print("[yellow]No active sources/companies found to scrape.[/yellow]")
211
+ return
212
+
213
+ fetched_jobs = execute_fetch_tasks(tasks)
214
+
215
+ passing_jobs, skipped_pend, skipped_seen, skipped_dupes, skipped_filters = filter_fetch_results(fetched_jobs, ignore_seen)
216
+ scored_candidates = score_fetch_results(passing_jobs)
217
+ final_candidates, cut_over_cap = limit_fetch_results(scored_candidates, limit)
218
+
219
+ if final_candidates:
220
+ save_fetch_results(final_candidates)
221
+
222
+ logger.info(f"Pipeline finished. Raw: {len(fetched_jobs)} | Pend: {skipped_pend} | Seen: {skipped_seen} | Dupe: {skipped_dupes} | Failed Filters: {skipped_filters} | Cut: {cut_over_cap} | Candidates: {len(final_candidates)}")
223
+
224
+ console.print("\n[bold]Fetch Summary[/bold]")
225
+ console.print("───────────────────────────────────────")
226
+ console.print(f"Total raw jobs : [cyan]{len(fetched_jobs)}[/cyan]")
227
+ console.print(f"Skipped (pend) : [yellow]{skipped_pend}[/yellow]")
228
+ console.print(f"Skipped (seen) : [yellow]{skipped_seen}[/yellow]")
229
+ console.print(f"Skipped (dupe) : [yellow]{skipped_dupes}[/yellow]")
230
+ console.print(f"Failed filters : [magenta]{skipped_filters}[/magenta]")
231
+ console.print(f"Cut over cap : [yellow]{cut_over_cap}[/yellow]")
232
+ console.print(f"New candidates : [bold green]{len(final_candidates)}[/bold green]")
233
+ console.print("───────────────────────────────────────\n")