jobhunter-cli 1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of jobhunter-cli might be problematic. Click here for more details.
- jobhunter/__init__.py +0 -0
- jobhunter/__main__.py +4 -0
- jobhunter/core/__init__.py +0 -0
- jobhunter/core/filters.py +110 -0
- jobhunter/core/http.py +57 -0
- jobhunter/core/logger.py +54 -0
- jobhunter/core/paths.py +124 -0
- jobhunter/core/settings.py +123 -0
- jobhunter/core/textutil.py +33 -0
- jobhunter/main.py +101 -0
- jobhunter/models.py +81 -0
- jobhunter/pipeline/__init__.py +0 -0
- jobhunter/pipeline/evaluate.py +284 -0
- jobhunter/pipeline/fetch.py +233 -0
- jobhunter/pipeline/send.py +191 -0
- jobhunter/sources/__init__.py +21 -0
- jobhunter/sources/arbeitnow.py +77 -0
- jobhunter/sources/ashby.py +75 -0
- jobhunter/sources/base.py +108 -0
- jobhunter/sources/greenhouse.py +70 -0
- jobhunter/sources/lever.py +85 -0
- jobhunter/sources/personio.py +124 -0
- jobhunter/sources/recruitee.py +80 -0
- jobhunter/sources/remoteok.py +69 -0
- jobhunter/sources/remotive.py +61 -0
- jobhunter/sources/workable.py +94 -0
- jobhunter/tools/__init__.py +0 -0
- jobhunter/tools/discover.py +275 -0
- jobhunter/tools/init.py +74 -0
- jobhunter/tools/prompt.py +106 -0
- jobhunter/tools/prune.py +124 -0
- jobhunter_cli-1.0.dist-info/METADATA +136 -0
- jobhunter_cli-1.0.dist-info/RECORD +36 -0
- jobhunter_cli-1.0.dist-info/WHEEL +4 -0
- jobhunter_cli-1.0.dist-info/entry_points.txt +2 -0
- jobhunter_cli-1.0.dist-info/licenses/LICENSE +21 -0
jobhunter/models.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
from pydantic import BaseModel, Field, computed_field
|
|
2
|
+
|
|
3
|
+
from jobhunter.core.settings import settings
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Job(BaseModel):
|
|
7
|
+
source: str
|
|
8
|
+
board: str
|
|
9
|
+
company: str
|
|
10
|
+
job_id: str
|
|
11
|
+
title: str
|
|
12
|
+
url: str
|
|
13
|
+
location: str
|
|
14
|
+
remote: bool | None = None
|
|
15
|
+
posted: str = ""
|
|
16
|
+
description: str
|
|
17
|
+
|
|
18
|
+
@computed_field
|
|
19
|
+
@property
|
|
20
|
+
def uid(self) -> str:
|
|
21
|
+
"""Dynamically generates a unique identifier for the job."""
|
|
22
|
+
return f"{self.source}:{self.board}:{self.job_id or self.url}"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Candidate(BaseModel):
|
|
26
|
+
uid: str = Field(..., description="Unique compound identifier: source:board:job_id_or_url")
|
|
27
|
+
priority: bool = False
|
|
28
|
+
stack_hits: int = 0
|
|
29
|
+
status: str = Field(default="pending", description="Pipeline status: pending, saved, applied, rejected")
|
|
30
|
+
source: str = ""
|
|
31
|
+
board: str = ""
|
|
32
|
+
company: str
|
|
33
|
+
job_id: str = ""
|
|
34
|
+
title: str
|
|
35
|
+
url: str
|
|
36
|
+
location: str
|
|
37
|
+
remote: bool | None = None
|
|
38
|
+
posted: str = ""
|
|
39
|
+
description: str = ""
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def from_job(cls, job: Job, priority: bool, stack_hits: int) -> Candidate:
|
|
43
|
+
"""Factory method to convert a raw scraped Job into a stored Candidate."""
|
|
44
|
+
max_chars = settings.limits.description_max_chars
|
|
45
|
+
truncated_description = (job.description or "No description provided.")[:max_chars]
|
|
46
|
+
|
|
47
|
+
return cls(
|
|
48
|
+
uid=job.uid,
|
|
49
|
+
priority=priority,
|
|
50
|
+
stack_hits=stack_hits,
|
|
51
|
+
status="pending",
|
|
52
|
+
source=job.source,
|
|
53
|
+
board=job.board,
|
|
54
|
+
company=job.company,
|
|
55
|
+
job_id=job.job_id,
|
|
56
|
+
title=job.title,
|
|
57
|
+
url=job.url,
|
|
58
|
+
location=job.location,
|
|
59
|
+
remote=job.remote,
|
|
60
|
+
posted=job.posted,
|
|
61
|
+
description=truncated_description,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class EvaluationResult(BaseModel):
|
|
66
|
+
"""The exact JSON schema the LLM must return."""
|
|
67
|
+
|
|
68
|
+
reasoning: str = Field(description="A brief 2-3 sentence thought process analyzing the match before scoring.")
|
|
69
|
+
hard_fail: bool = Field(description="True if the job hits any hard gate or dealbreaker.")
|
|
70
|
+
hard_fail_reason: str = Field(default="", description="If hard_fail is true, a max 12-word explanation. Otherwise empty.")
|
|
71
|
+
strengths: list[str] = Field(description="2-3 short items showing requirements explicitly met by the CV.")
|
|
72
|
+
gaps: list[str] = Field(default_factory=list, description="Missing requirements or weak points compared to the CV.")
|
|
73
|
+
score: int = Field(description="Final match score (0-100) based on the rubric.")
|
|
74
|
+
score_explanation: str = Field(description="A 1-2 sentence explanation of exactly why the job passed or failed the score threshold.")
|
|
75
|
+
pitch: str = Field(default="", description="A short, one-sentence first-person pitch. Empty if score is below threshold.")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class EvaluatedCandidate(Candidate):
|
|
79
|
+
"""A candidate that has been evaluated by the AI."""
|
|
80
|
+
|
|
81
|
+
evaluation: EvaluationResult
|
|
File without changes
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
import concurrent.futures
|
|
2
|
+
import logging
|
|
3
|
+
import time
|
|
4
|
+
|
|
5
|
+
import openai
|
|
6
|
+
from openai import OpenAI
|
|
7
|
+
from pydantic import ValidationError
|
|
8
|
+
from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
|
|
9
|
+
from rich.table import Column
|
|
10
|
+
|
|
11
|
+
from jobhunter.core.logger import console
|
|
12
|
+
from jobhunter.core.paths import DATA_PATH, read, write
|
|
13
|
+
from jobhunter.core.settings import settings
|
|
14
|
+
from jobhunter.models import Candidate, EvaluatedCandidate, EvaluationResult
|
|
15
|
+
from jobhunter.tools.prompt import build_system_prompt, build_user_prompt
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def setup_llm_client() -> OpenAI:
|
|
21
|
+
"""Initializes the OpenAI client (supports local Ollama/vLLM via base_url)."""
|
|
22
|
+
logger.info("Initializing LLM client...")
|
|
23
|
+
|
|
24
|
+
if not settings.llm_api_key:
|
|
25
|
+
logger.error("LLM API key is missing. Set LLM_API_KEY in your .env file.")
|
|
26
|
+
console.print("[bold red]Error: LLM API key is missing. Set LLM_API_KEY in your .env file.[/bold red]")
|
|
27
|
+
raise ValueError("LLM_API_KEY is empty. Check your .env file.")
|
|
28
|
+
|
|
29
|
+
return OpenAI(
|
|
30
|
+
api_key=settings.llm_api_key,
|
|
31
|
+
base_url=settings.llm.base_url,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def build_evaluation_tasks(job_uid: str | None = None, limit: int | None = None) -> list[Candidate]:
|
|
36
|
+
"""Loads candidates from disk and filters for those pending evaluation."""
|
|
37
|
+
logger.info("Building evaluation tasks...")
|
|
38
|
+
|
|
39
|
+
raw_candidates = read(DATA_PATH / "candidates.json", list)
|
|
40
|
+
raw_evals = read(DATA_PATH / "evaluated.json", list)
|
|
41
|
+
|
|
42
|
+
evaluated_uids = {item.get("uid") for item in raw_evals if "uid" in item}
|
|
43
|
+
|
|
44
|
+
tasks = []
|
|
45
|
+
for item in raw_candidates:
|
|
46
|
+
try:
|
|
47
|
+
c = Candidate(**item)
|
|
48
|
+
|
|
49
|
+
if job_uid:
|
|
50
|
+
if c.uid == job_uid:
|
|
51
|
+
tasks.append(c)
|
|
52
|
+
break
|
|
53
|
+
continue
|
|
54
|
+
|
|
55
|
+
if c.uid not in evaluated_uids:
|
|
56
|
+
tasks.append(c)
|
|
57
|
+
except Exception as e:
|
|
58
|
+
logger.error(f"Failed to parse candidate {item.get('uid', 'unknown')}: {e}")
|
|
59
|
+
console.print(f" [red]✖ Failed to parse candidate {item.get('uid', 'unknown')}[/red]")
|
|
60
|
+
|
|
61
|
+
final_limit = limit or settings.limits.max_candidates_per_run
|
|
62
|
+
if final_limit and final_limit > 0:
|
|
63
|
+
tasks = tasks[:final_limit]
|
|
64
|
+
|
|
65
|
+
logger.info(f"Generated {len(tasks)} evaluation tasks from {len(raw_candidates)} total candidates.")
|
|
66
|
+
|
|
67
|
+
return tasks
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def evaluate_single_job(client: OpenAI, system_prompt: str, candidate: Candidate, retries: int = 5, backoff: float = 1.5) -> tuple[Candidate, EvaluationResult | None]:
|
|
71
|
+
"""Calls the LLM to evaluate a single job and parses the result with automatic retry and JSON cleanup fallbacks."""
|
|
72
|
+
user_prompt = build_user_prompt(candidate)
|
|
73
|
+
|
|
74
|
+
for attempt in range(retries):
|
|
75
|
+
raw_json = ""
|
|
76
|
+
|
|
77
|
+
try:
|
|
78
|
+
response = client.chat.completions.create(
|
|
79
|
+
model=settings.llm.model or "gpt-4o-mini",
|
|
80
|
+
messages=[{"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}],
|
|
81
|
+
response_format={"type": "json_object"},
|
|
82
|
+
temperature=settings.llm.temperature,
|
|
83
|
+
timeout=settings.llm.timeout,
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
raw_json = response.choices[0].message.content or ""
|
|
87
|
+
if not raw_json:
|
|
88
|
+
raise ValueError("LLM returned empty content instead of JSON.")
|
|
89
|
+
|
|
90
|
+
try:
|
|
91
|
+
eval_result = EvaluationResult.model_validate_json(raw_json)
|
|
92
|
+
return candidate, eval_result
|
|
93
|
+
|
|
94
|
+
except ValidationError:
|
|
95
|
+
cleaned_json = raw_json.strip()
|
|
96
|
+
if cleaned_json.startswith("```json"):
|
|
97
|
+
cleaned_json = cleaned_json[7:]
|
|
98
|
+
if cleaned_json.endswith("```"):
|
|
99
|
+
cleaned_json = cleaned_json[:-3]
|
|
100
|
+
cleaned_json = cleaned_json.strip()
|
|
101
|
+
|
|
102
|
+
eval_result = EvaluationResult.model_validate_json(cleaned_json)
|
|
103
|
+
return candidate, eval_result
|
|
104
|
+
|
|
105
|
+
except openai.RateLimitError as rle:
|
|
106
|
+
if attempt < retries - 1:
|
|
107
|
+
sleep_time = backoff * (2**attempt)
|
|
108
|
+
logger.warning(f"Rate limit hit (429) for {candidate.company}. Retrying in {sleep_time}s (Attempt {attempt + 1}/{retries})...")
|
|
109
|
+
time.sleep(sleep_time)
|
|
110
|
+
else:
|
|
111
|
+
logger.error(f"Rate limit exceeded max retries for {candidate.company}: {rle}")
|
|
112
|
+
console.print(f" [red]✖ Rate limit exceeded for {candidate.company} after {retries} retries.[/red]")
|
|
113
|
+
return candidate, None
|
|
114
|
+
|
|
115
|
+
except openai.APIStatusError as ase:
|
|
116
|
+
if ase.status_code >= 500 and attempt < retries - 1:
|
|
117
|
+
sleep_time = backoff * (2**attempt)
|
|
118
|
+
time.sleep(sleep_time)
|
|
119
|
+
else:
|
|
120
|
+
console.print("\n───────── API ERROR DETAILS ─────────")
|
|
121
|
+
console.print(ase.response.json() if hasattr(ase, "response") else str(ase))
|
|
122
|
+
console.print("───────────────────────────────────────\n")
|
|
123
|
+
return candidate, None
|
|
124
|
+
|
|
125
|
+
except ValidationError as ve:
|
|
126
|
+
logger.error(f"AI returned invalid schema for {candidate.company}: {ve}")
|
|
127
|
+
console.print(f" [red]✖ AI returned invalid schema for {candidate.company}[/red]")
|
|
128
|
+
logger.debug(f"Offending raw JSON for {candidate.company}:\n{raw_json}")
|
|
129
|
+
return candidate, None
|
|
130
|
+
|
|
131
|
+
except Exception as e:
|
|
132
|
+
logger.error(f"LLM API error evaluating {candidate.company}: {e}")
|
|
133
|
+
console.print(f" [red]✖ LLM API error evaluating {candidate.company}: {e}[/red]")
|
|
134
|
+
return candidate, None
|
|
135
|
+
|
|
136
|
+
return candidate, None
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def execute_evaluation_tasks(client: OpenAI, tasks: list[Candidate]) -> list[tuple[Candidate, EvaluationResult | None]]:
|
|
140
|
+
"""Executes the AI evaluation tasks concurrently with a Rich progress bar."""
|
|
141
|
+
fetched_evals: list[tuple[Candidate, EvaluationResult | None]] = []
|
|
142
|
+
system_prompt = build_system_prompt()
|
|
143
|
+
target_total = len(tasks)
|
|
144
|
+
|
|
145
|
+
workers = max(1, min(settings.workers.evaluation_workers, 20))
|
|
146
|
+
|
|
147
|
+
logger.info(f"Executing evaluation tasks across {target_total} targets with {workers} workers.")
|
|
148
|
+
console.print(f"[bold cyan]Evaluating {target_total} jobs with AI...[/bold cyan]")
|
|
149
|
+
|
|
150
|
+
with Progress(
|
|
151
|
+
SpinnerColumn("dots", style="cyan"),
|
|
152
|
+
TextColumn("{task.description}", table_column=Column(ratio=1)),
|
|
153
|
+
TextColumn("[dim]{task.completed}/{task.total}[/dim]"),
|
|
154
|
+
BarColumn(bar_width=24, style="dim cyan", complete_style="green", finished_style="bold green"),
|
|
155
|
+
TaskProgressColumn(),
|
|
156
|
+
expand=True,
|
|
157
|
+
console=console,
|
|
158
|
+
) as progress:
|
|
159
|
+
eval_task = progress.add_task("[cyan]Starting evaluation...", total=target_total)
|
|
160
|
+
|
|
161
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
|
|
162
|
+
future_to_candidate = {executor.submit(evaluate_single_job, client, system_prompt, candidate): candidate for candidate in tasks}
|
|
163
|
+
|
|
164
|
+
for future in concurrent.futures.as_completed(future_to_candidate):
|
|
165
|
+
candidate = future_to_candidate[future]
|
|
166
|
+
name = candidate.company or "Unknown"
|
|
167
|
+
|
|
168
|
+
try:
|
|
169
|
+
result = future.result(timeout=settings.llm.timeout + 10)
|
|
170
|
+
fetched_evals.append(result)
|
|
171
|
+
|
|
172
|
+
if result[1] is not None:
|
|
173
|
+
score = result[1].score
|
|
174
|
+
color = "green" if score >= settings.evaluating.score_threshold else "yellow"
|
|
175
|
+
progress.update(eval_task, advance=1, description=f"[cyan]Evaluating... [dim](Finished {name} - [{color}]{score}[/{color}][/dim])")
|
|
176
|
+
else:
|
|
177
|
+
progress.update(eval_task, advance=1)
|
|
178
|
+
progress.console.print(f" [dim red]⚠ API/Validation Error for {name}. Skipping...[/dim red]")
|
|
179
|
+
|
|
180
|
+
except concurrent.futures.TimeoutError:
|
|
181
|
+
logger.error(f"Task for {name} timed out.")
|
|
182
|
+
console.print(f" [red]✖ Evaluation for {name} timed out.[/red]")
|
|
183
|
+
progress.update(eval_task, advance=1)
|
|
184
|
+
|
|
185
|
+
except Exception as e:
|
|
186
|
+
logger.exception(f"Error executing evaluation task for {name}: {e}")
|
|
187
|
+
console.print(f" [red]✖ Error evaluating {name}: {e}[/red]")
|
|
188
|
+
progress.update(eval_task, advance=1)
|
|
189
|
+
|
|
190
|
+
if settings.llm.request_delay > 0:
|
|
191
|
+
time.sleep(settings.llm.request_delay)
|
|
192
|
+
|
|
193
|
+
progress.update(eval_task, description="[bold green]Evaluation complete![/bold green]")
|
|
194
|
+
|
|
195
|
+
success_count = sum(1 for _, eval_result in fetched_evals if eval_result is not None)
|
|
196
|
+
|
|
197
|
+
logger.info(f"Successfully evaluated {success_count} jobs.")
|
|
198
|
+
|
|
199
|
+
return fetched_evals
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def format_evaluation_results(fetched_evals: list[tuple[Candidate, EvaluationResult | None]]) -> tuple[list[EvaluatedCandidate], int, int, int, int]:
|
|
203
|
+
"""Formats the AI results into EvaluatedCandidate models and calculates metrics."""
|
|
204
|
+
logger.info("Formatting LLM results and calculating statistics...")
|
|
205
|
+
console.print("[dim cyan]Formatting evaluations...[/dim cyan]")
|
|
206
|
+
|
|
207
|
+
passed = hard_failed = low_score = errors = 0
|
|
208
|
+
final_evaluations: list[EvaluatedCandidate] = []
|
|
209
|
+
|
|
210
|
+
for candidate, evaluation in fetched_evals:
|
|
211
|
+
if not evaluation:
|
|
212
|
+
errors += 1
|
|
213
|
+
continue
|
|
214
|
+
|
|
215
|
+
candidate.status = "evaluated"
|
|
216
|
+
|
|
217
|
+
evaluated_candidate = EvaluatedCandidate(**candidate.model_dump(), evaluation=evaluation)
|
|
218
|
+
|
|
219
|
+
if evaluation.hard_fail:
|
|
220
|
+
hard_failed += 1
|
|
221
|
+
logger.debug(f"Candidate {candidate.company} hard failed gates.")
|
|
222
|
+
elif evaluation.score < settings.evaluating.score_threshold:
|
|
223
|
+
low_score += 1
|
|
224
|
+
logger.debug(f"Candidate {candidate.company} failed score threshold ({evaluation.score}).")
|
|
225
|
+
else:
|
|
226
|
+
passed += 1
|
|
227
|
+
|
|
228
|
+
final_evaluations.append(evaluated_candidate)
|
|
229
|
+
|
|
230
|
+
logger.info(f"Formatting complete. {passed} passed, {hard_failed} hard failed, {low_score} low score, {errors} errors.")
|
|
231
|
+
|
|
232
|
+
return final_evaluations, passed, hard_failed, low_score, errors
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def save_evaluation_results(final_evaluations: list[EvaluatedCandidate]):
|
|
236
|
+
"""Saves evaluated candidates to disk, preventing duplicate UIDs, and sorts by score."""
|
|
237
|
+
logger.info("Saving evaluations...")
|
|
238
|
+
console.print("[dim cyan]Saving evaluations...[/dim cyan]")
|
|
239
|
+
|
|
240
|
+
existing_evals = read(DATA_PATH / "evaluated.json", list)
|
|
241
|
+
|
|
242
|
+
new_uids = {c.uid for c in final_evaluations}
|
|
243
|
+
filtered_evals = [e for e in existing_evals if e.get("uid") not in new_uids]
|
|
244
|
+
|
|
245
|
+
new_eval_dicts = [c.model_dump(mode="json") for c in final_evaluations]
|
|
246
|
+
|
|
247
|
+
filtered_evals.extend(new_eval_dicts)
|
|
248
|
+
filtered_evals.sort(key=lambda x: x.get("evaluation", {}).get("score", -1), reverse=True)
|
|
249
|
+
|
|
250
|
+
write(DATA_PATH / "evaluated.json", filtered_evals)
|
|
251
|
+
|
|
252
|
+
logger.info(f"Successfully saved {len(final_evaluations)} evaluations to disk.")
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def main(job_uid: str | None = None, limit: int | None = None):
|
|
256
|
+
"""Orchestrates the AI evaluation pipeline."""
|
|
257
|
+
logger.info("Starting AI Evaluation Pipeline run.")
|
|
258
|
+
|
|
259
|
+
client = setup_llm_client()
|
|
260
|
+
|
|
261
|
+
tasks = build_evaluation_tasks(job_uid, limit)
|
|
262
|
+
if not tasks:
|
|
263
|
+
logger.warning("No active candidates found to evaluate.")
|
|
264
|
+
console.print("[yellow]No active candidates found to evaluate.[/yellow]")
|
|
265
|
+
return
|
|
266
|
+
|
|
267
|
+
fetched_evals = execute_evaluation_tasks(client, tasks)
|
|
268
|
+
final_evaluations, passed, hard_failed, low_score, errors = format_evaluation_results(fetched_evals)
|
|
269
|
+
|
|
270
|
+
if final_evaluations:
|
|
271
|
+
save_evaluation_results(final_evaluations)
|
|
272
|
+
|
|
273
|
+
total_successful = passed + hard_failed + low_score
|
|
274
|
+
logger.info(f"Pipeline finished. Evaluated: {total_successful} | Passed: {passed} | Hard Failed: {hard_failed} | Low Score: {low_score} | Errors: {errors}")
|
|
275
|
+
|
|
276
|
+
console.print("\n[bold]AI Evaluation Summary[/bold]")
|
|
277
|
+
console.print("───────────────────────────────────────")
|
|
278
|
+
console.print(f"Total evaluated : [cyan]{total_successful}[/cyan]")
|
|
279
|
+
console.print(f"Passed : [bold green]{passed}[/bold green]")
|
|
280
|
+
console.print(f"Failed (Gates) : [red]{hard_failed}[/red]")
|
|
281
|
+
console.print(f"Failed (Score) : [yellow]{low_score}[/yellow]")
|
|
282
|
+
if errors:
|
|
283
|
+
console.print(f"API Errors : [bold red]{errors}[/bold red]")
|
|
284
|
+
console.print("───────────────────────────────────────\n")
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import concurrent.futures
|
|
2
|
+
import logging
|
|
3
|
+
|
|
4
|
+
from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
|
|
5
|
+
from rich.table import Column
|
|
6
|
+
|
|
7
|
+
from jobhunter.core.filters import count_stack_hits, passes_all_filters
|
|
8
|
+
from jobhunter.core.logger import console
|
|
9
|
+
from jobhunter.core.paths import DATA_PATH, read, write
|
|
10
|
+
from jobhunter.core.settings import settings
|
|
11
|
+
from jobhunter.models import Candidate, Job
|
|
12
|
+
from jobhunter.sources import SOURCES
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def build_fetch_tasks(source_name: str | None, no_aggregators: bool) -> list[tuple]:
|
|
18
|
+
"""Generates a list of (source, slug) tuples to be scraped."""
|
|
19
|
+
logger.info("Building fetch tasks...")
|
|
20
|
+
|
|
21
|
+
tasks = []
|
|
22
|
+
for name, source in SOURCES.items():
|
|
23
|
+
if source_name and name != source_name:
|
|
24
|
+
continue
|
|
25
|
+
|
|
26
|
+
is_aggregator = getattr(source, "is_aggregator", False)
|
|
27
|
+
|
|
28
|
+
if is_aggregator and not no_aggregators:
|
|
29
|
+
tasks.append((source, ""))
|
|
30
|
+
else:
|
|
31
|
+
slugs = settings.companies.get(name, [])
|
|
32
|
+
for slug in slugs:
|
|
33
|
+
tasks.append((source, slug))
|
|
34
|
+
|
|
35
|
+
logger.info(f"Generated {len(tasks)} fetch tasks.")
|
|
36
|
+
|
|
37
|
+
return tasks
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def execute_fetch_tasks(tasks: list[tuple]) -> list[Job]:
|
|
41
|
+
"""Executes the scraping tasks concurrently with a Rich progress bar."""
|
|
42
|
+
fetched_jobs: list[Job] = []
|
|
43
|
+
target_total = len(tasks)
|
|
44
|
+
|
|
45
|
+
workers = max(1, min(settings.workers.fetch_workers, 20))
|
|
46
|
+
|
|
47
|
+
logger.info(f"Executing fetch tasks across {target_total} targets with {workers} workers.")
|
|
48
|
+
console.print("[bold cyan]Fetching jobs from configured sources...[/bold cyan]")
|
|
49
|
+
|
|
50
|
+
with Progress(
|
|
51
|
+
SpinnerColumn("dots", style="cyan"),
|
|
52
|
+
TextColumn("{task.description}", table_column=Column(ratio=1)),
|
|
53
|
+
TextColumn("[dim]{task.completed}/{task.total}[/dim]"),
|
|
54
|
+
BarColumn(bar_width=24, style="dim cyan", complete_style="green", finished_style="bold green"),
|
|
55
|
+
TaskProgressColumn(),
|
|
56
|
+
expand=True,
|
|
57
|
+
console=console,
|
|
58
|
+
) as progress:
|
|
59
|
+
fetch_task = progress.add_task("[cyan]Starting fetch...", total=target_total)
|
|
60
|
+
|
|
61
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
|
|
62
|
+
future_to_name = {}
|
|
63
|
+
for source, slug in tasks:
|
|
64
|
+
name = f"{source.name}:{slug}" if slug else source.name
|
|
65
|
+
future = executor.submit(source.execute_scraper, slug)
|
|
66
|
+
future_to_name[future] = name
|
|
67
|
+
|
|
68
|
+
for future in concurrent.futures.as_completed(future_to_name):
|
|
69
|
+
name = future_to_name[future]
|
|
70
|
+
|
|
71
|
+
try:
|
|
72
|
+
result = future.result(timeout=60)
|
|
73
|
+
if result:
|
|
74
|
+
fetched_jobs.extend(result)
|
|
75
|
+
|
|
76
|
+
progress.update(fetch_task, advance=1, description=f"[cyan]Fetching... [dim](Finished {name})[/dim]")
|
|
77
|
+
|
|
78
|
+
except concurrent.futures.TimeoutError:
|
|
79
|
+
logger.error(f"Task for {name} timed out.")
|
|
80
|
+
console.print(f" [red]✖ Task for {name} timed out.[/red]")
|
|
81
|
+
progress.update(fetch_task, advance=1)
|
|
82
|
+
|
|
83
|
+
except Exception as e:
|
|
84
|
+
logger.exception(f"Error executing fetch task for {name}: {e}")
|
|
85
|
+
console.print(f" [red]✖ Error fetching {name}: {e}[/red]")
|
|
86
|
+
progress.update(fetch_task, advance=1)
|
|
87
|
+
|
|
88
|
+
progress.update(fetch_task, description="[bold green]Fetch complete![/bold green]")
|
|
89
|
+
|
|
90
|
+
logger.info(f"Successfully fetched {len(fetched_jobs)} raw jobs.")
|
|
91
|
+
|
|
92
|
+
return fetched_jobs
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def filter_fetch_results(jobs: list[Job], ignore_seen: bool) -> tuple[list[Job], int, int, int, int]:
|
|
96
|
+
"""Removes previously seen jobs, cross-source duplicates, and jobs failing filters."""
|
|
97
|
+
logger.info("Filtering and deduplicating jobs...")
|
|
98
|
+
console.print("[dim cyan]Filtering and deduplicating jobs...[/dim cyan]")
|
|
99
|
+
|
|
100
|
+
raw_candidates = read(DATA_PATH / "candidates.json", list)
|
|
101
|
+
seen_data = read(DATA_PATH / "seen.json", dict)
|
|
102
|
+
|
|
103
|
+
fetched_uids = {item.get("uid") for item in raw_candidates if "uid" in item}
|
|
104
|
+
|
|
105
|
+
skipped_pend = skipped_seen = skipped_dupes = skipped_filters = 0
|
|
106
|
+
run_duped_roles = set()
|
|
107
|
+
passing_jobs = []
|
|
108
|
+
|
|
109
|
+
for job in jobs:
|
|
110
|
+
if job.uid in fetched_uids:
|
|
111
|
+
skipped_pend += 1
|
|
112
|
+
continue
|
|
113
|
+
|
|
114
|
+
if job.uid in seen_data and not ignore_seen:
|
|
115
|
+
skipped_seen += 1
|
|
116
|
+
continue
|
|
117
|
+
|
|
118
|
+
company = (job.company or "Unknown").strip().lower()
|
|
119
|
+
title = (job.title or "Unknown").strip().lower()
|
|
120
|
+
location = (job.location or "Unknown").strip().lower()
|
|
121
|
+
role_key = f"{company}:{title}:{location}"
|
|
122
|
+
|
|
123
|
+
if role_key in run_duped_roles:
|
|
124
|
+
skipped_dupes += 1
|
|
125
|
+
continue
|
|
126
|
+
|
|
127
|
+
run_duped_roles.add(role_key)
|
|
128
|
+
|
|
129
|
+
if passes_all_filters(job):
|
|
130
|
+
passing_jobs.append(job)
|
|
131
|
+
else:
|
|
132
|
+
skipped_filters += 1
|
|
133
|
+
|
|
134
|
+
logger.info(f"Filtering complete: {len(passing_jobs)} passed, {skipped_pend} skipped (pend), {skipped_seen} skipped (seen), {skipped_dupes} dupes, {skipped_filters} failed filters.")
|
|
135
|
+
|
|
136
|
+
return passing_jobs, skipped_pend, skipped_seen, skipped_dupes, skipped_filters
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def score_fetch_results(jobs: list[Job]) -> list[Candidate]:
|
|
140
|
+
"""Calculates scores, sorts candidates, and converts Jobs into Candidates."""
|
|
141
|
+
logger.info("Scoring and sorting passing jobs...")
|
|
142
|
+
console.print("[dim cyan]Scoring and sorting candidates...[/dim cyan]")
|
|
143
|
+
|
|
144
|
+
priority_companies = {c.lower() for c in settings.filters.company_priority}
|
|
145
|
+
|
|
146
|
+
scored_candidates: list[Candidate] = []
|
|
147
|
+
for job in jobs:
|
|
148
|
+
is_priority = (job.company or "").lower() in priority_companies
|
|
149
|
+
hits = count_stack_hits(job)
|
|
150
|
+
candidate = Candidate.from_job(job, priority=is_priority, stack_hits=hits)
|
|
151
|
+
scored_candidates.append(candidate)
|
|
152
|
+
|
|
153
|
+
scored_candidates.sort(key=lambda c: (c.priority, c.stack_hits), reverse=True)
|
|
154
|
+
|
|
155
|
+
return scored_candidates
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def limit_fetch_results(scored_candidates: list[Candidate], limit: int | None = None) -> tuple[list[Candidate], int]:
|
|
159
|
+
"""Enforces the max candidate limit."""
|
|
160
|
+
logger.info("Enforcing limits...")
|
|
161
|
+
console.print("[dim cyan]Enforcing candidate limits...[/dim cyan]")
|
|
162
|
+
|
|
163
|
+
final_limit = limit or settings.limits.max_candidates_per_run
|
|
164
|
+
|
|
165
|
+
final_candidates = []
|
|
166
|
+
non_priority_count = 0
|
|
167
|
+
|
|
168
|
+
for candidate in scored_candidates:
|
|
169
|
+
if not candidate.priority:
|
|
170
|
+
if non_priority_count >= final_limit:
|
|
171
|
+
continue
|
|
172
|
+
non_priority_count += 1
|
|
173
|
+
|
|
174
|
+
final_candidates.append(candidate)
|
|
175
|
+
|
|
176
|
+
cut_over_cap = len(scored_candidates) - len(final_candidates)
|
|
177
|
+
|
|
178
|
+
logger.info(f"Generated {len(final_candidates)} candidates. Cut {cut_over_cap} jobs over the cap.")
|
|
179
|
+
|
|
180
|
+
return final_candidates, cut_over_cap
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def save_fetch_results(final_candidates: list[Candidate]):
|
|
184
|
+
"""Saves finalized candidates into disk, preventing duplicate UIDs."""
|
|
185
|
+
logger.info("Saving candidates...")
|
|
186
|
+
console.print("[dim cyan]Saving candidates...[/dim cyan]")
|
|
187
|
+
|
|
188
|
+
existing_candidates = read(DATA_PATH / "candidates.json", list)
|
|
189
|
+
|
|
190
|
+
new_uids = {c.uid for c in final_candidates}
|
|
191
|
+
filtered_existing = [c for c in existing_candidates if c.get("uid") not in new_uids]
|
|
192
|
+
|
|
193
|
+
new_candidate_dicts = [c.model_dump(mode="json") for c in final_candidates]
|
|
194
|
+
|
|
195
|
+
filtered_existing.extend(new_candidate_dicts)
|
|
196
|
+
filtered_existing.sort(key=lambda x: (x.get("priority", False), x.get("stack_hits", 0)), reverse=True)
|
|
197
|
+
|
|
198
|
+
write(DATA_PATH / "candidates.json", filtered_existing)
|
|
199
|
+
|
|
200
|
+
logger.info(f"Successfully saved {len(final_candidates)} candidates to disk.")
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def main(source_name: str | None = None, no_aggregators: bool = False, ignore_seen: bool = False, limit: int | None = None):
|
|
204
|
+
"""Orchestrates the job fetching pipeline."""
|
|
205
|
+
logger.info("Starting job fetch pipeline run.")
|
|
206
|
+
|
|
207
|
+
tasks = build_fetch_tasks(source_name, no_aggregators)
|
|
208
|
+
if not tasks:
|
|
209
|
+
logger.warning("No active sources/companies found to scrape.")
|
|
210
|
+
console.print("[yellow]No active sources/companies found to scrape.[/yellow]")
|
|
211
|
+
return
|
|
212
|
+
|
|
213
|
+
fetched_jobs = execute_fetch_tasks(tasks)
|
|
214
|
+
|
|
215
|
+
passing_jobs, skipped_pend, skipped_seen, skipped_dupes, skipped_filters = filter_fetch_results(fetched_jobs, ignore_seen)
|
|
216
|
+
scored_candidates = score_fetch_results(passing_jobs)
|
|
217
|
+
final_candidates, cut_over_cap = limit_fetch_results(scored_candidates, limit)
|
|
218
|
+
|
|
219
|
+
if final_candidates:
|
|
220
|
+
save_fetch_results(final_candidates)
|
|
221
|
+
|
|
222
|
+
logger.info(f"Pipeline finished. Raw: {len(fetched_jobs)} | Pend: {skipped_pend} | Seen: {skipped_seen} | Dupe: {skipped_dupes} | Failed Filters: {skipped_filters} | Cut: {cut_over_cap} | Candidates: {len(final_candidates)}")
|
|
223
|
+
|
|
224
|
+
console.print("\n[bold]Fetch Summary[/bold]")
|
|
225
|
+
console.print("───────────────────────────────────────")
|
|
226
|
+
console.print(f"Total raw jobs : [cyan]{len(fetched_jobs)}[/cyan]")
|
|
227
|
+
console.print(f"Skipped (pend) : [yellow]{skipped_pend}[/yellow]")
|
|
228
|
+
console.print(f"Skipped (seen) : [yellow]{skipped_seen}[/yellow]")
|
|
229
|
+
console.print(f"Skipped (dupe) : [yellow]{skipped_dupes}[/yellow]")
|
|
230
|
+
console.print(f"Failed filters : [magenta]{skipped_filters}[/magenta]")
|
|
231
|
+
console.print(f"Cut over cap : [yellow]{cut_over_cap}[/yellow]")
|
|
232
|
+
console.print(f"New candidates : [bold green]{len(final_candidates)}[/bold green]")
|
|
233
|
+
console.print("───────────────────────────────────────\n")
|