stage-cli 1.0.0__py3-none-any.whl

stage/classify/role.py

@@ -0,0 +1,91 @@
+from dataclasses import dataclass
+from functools import lru_cache
+
+from stage.domain import RoleCategory
+from stage.lexicon import fold, role_lexicon, source_role_categories
+
+GENERAL_CS = RoleCategory.GENERAL_CS.value
+SWE = RoleCategory.SWE.value
+TECHNOLOGY_NOT_DISCIPLINE = frozenset({"gpu", "fpga", "soc", "asic", "rtos", "cuda"})
+OUTRANKED_BY_A_LEADING_QUALIFIER = (SWE,)
+
+
+@lru_cache(maxsize=1)
+def _by_first_token() -> dict[str, tuple[tuple[str, str], ...]]:
+    index: dict[str, list[tuple[str, str]]] = {}
+    for category, phrases in role_lexicon().items():
+        for phrase in phrases:
+            index.setdefault(phrase.split(" ", 1)[0], []).append((category, phrase))
+    return {token: tuple(entries) for token, entries in index.items()}
+
+
+@dataclass(frozen=True, slots=True)
+class RoleVerdict:
+    role: RoleCategory = RoleCategory.UNKNOWN
+    matched: tuple[str, ...] = ()
+    ambiguous: bool = False
+
+
+def _leading_specialist(folded: str, hits: dict[str, list[str]]) -> str | None:
+    outranked = [category for category in OUTRANKED_BY_A_LEADING_QUALIFIER if category in hits]
+    if not outranked or len(hits) < 2:
+        return None
+    if len(outranked) > 1:
+        return min(outranked, key=lambda c: min(folded.find(p) for p in hits[c]))
+    generic = min(folded.find(phrase) for category in outranked for phrase in hits[category])
+    ahead = {
+        category: min(
+            folded.find(phrase) for phrase in phrases if phrase not in TECHNOLOGY_NOT_DISCIPLINE
+        )
+        for category, phrases in hits.items()
+        if category not in outranked
+        and any(phrase not in TECHNOLOGY_NOT_DISCIPLINE for phrase in phrases)
+    }
+    ahead = {category: at for category, at in ahead.items() if 0 <= at < generic}
+    if len(ahead) != 1:
+        return None
+    return next(iter(ahead))
+
+
+def classify_role(title: str, description: str = "", source_category: str = "") -> RoleVerdict:
+    index = _by_first_token()
+    declared = source_role_categories().get(fold(source_category)) if source_category else None
+    for text in (title, description):
+        folded = fold(text)
+        if not folded:
+            continue
+        padded = f" {folded} "
+        hits: dict[str, list[str]] = {}
+        for token in set(folded.split()):
+            for category, phrase in index.get(token, ()):
+                if f" {phrase} " in padded:
+                    hits.setdefault(category, []).append(phrase)
+        if not hits:
+            continue
+        if len(hits) > 1:
+            hits.pop(GENERAL_CS, None)
+        hits = {
+            category: sorted(set(found), key=len, reverse=True) for category, found in hits.items()
+        }
+
+        leading = _leading_specialist(folded, hits) if text is title else None
+        if leading is not None:
+            matched = tuple(sorted({phrase for phrases in hits.values() for phrase in phrases}))
+            return RoleVerdict(role=RoleCategory(leading), matched=matched)
+
+        best = max(len(phrases[0]) for phrases in hits.values())
+        winners = [category for category, phrases in hits.items() if len(phrases[0]) == best]
+        matched = tuple(sorted({phrase for phrases in hits.values() for phrase in phrases}))
+        if len(winners) != 1:
+            winner = min(
+                winners,
+                key=lambda category: (
+                    min(folded.find(phrase) for phrase in hits[category] if len(phrase) == best),
+                    category,
+                ),
+            )
+            return RoleVerdict(role=RoleCategory(winner), matched=matched)
+        return RoleVerdict(role=RoleCategory(winners[0]), matched=matched)
+    if declared is not None:
+        return RoleVerdict(role=RoleCategory(declared), matched=(fold(source_category),))
+    return RoleVerdict()

stage/classify/scope.py

@@ -0,0 +1,47 @@
+from dataclasses import dataclass
+
+from stage.domain import Job, LocationBucket, QuarantinedJob, RejectionReason
+
+
+@dataclass(frozen=True, slots=True)
+class Rejection:
+    reason: RejectionReason
+    matched_phrase: str
+
+
+def screen_is_internship(job: Job) -> Rejection | None:
+    from stage.classify.internship import screen_internship
+
+    verdict = screen_internship(job.title_raw, job.signals.employment_type, job.signals.category)
+    if verdict.is_internship:
+        return None
+    return Rejection(
+        reason=RejectionReason.NOT_AN_INTERNSHIP,
+        matched_phrase=verdict.disqualified_by or "no internship marker in title",
+    )
+
+
+def screen_location(job: Job) -> Rejection | None:
+    if job.location is not LocationBucket.INTERNATIONAL:
+        return None
+    return Rejection(
+        reason=RejectionReason.OUT_OF_SCOPE_LOCATION,
+        matched_phrase=job.location_raw or "outside Canada and the United States",
+    )
+
+
+def to_quarantined(job: Job, rejection: Rejection) -> QuarantinedJob:
+    return QuarantinedJob(
+        id=job.id,
+        source=job.source,
+        company=job.company,
+        title_raw=job.title_raw,
+        reason=rejection.reason,
+        first_seen=job.first_seen,
+        last_seen=job.last_seen,
+        apply_url_raw=job.apply_url_raw,
+        location_raw=job.location_raw,
+        location=job.location,
+        remote_scope=job.remote_scope,
+        matched_phrase=rejection.matched_phrase,
+    )

stage/cli/app.py

@@ -0,0 +1,4 @@
+from stage.cli import commands
+from stage.cli.options import app, main, run_async, show_help
+
+__all__ = ["app", "commands", "main", "run_async", "show_help"]

stage/cli/commands/__init__.py

@@ -0,0 +1,8 @@
+import importlib
+
+_ORDER = ("postings", "pipeline", "schedule", "insight", "discovery")
+
+for _name in _ORDER:
+    importlib.import_module(f"{__name__}.{_name}")
+
+__all__ = list(_ORDER)

stage/cli/commands/discovery.py

@@ -0,0 +1,294 @@
+from collections.abc import AsyncIterator
+from pathlib import Path
+from typing import Annotated, Any
+
+import typer
+
+from stage.cli.options import (
+    WORD,
+    DatabaseOption,
+    InvalidOptionError,
+    RegistryOption,
+    _adopt_unregistered,
+    _count,
+    _lock_path,
+    _parse_enum,
+    _print_failure,
+    _require_enum,
+    app,
+    run_async,
+)
+
+
+@app.command(
+    help="Find the job-board platform a company uses", rich_help_panel="Registry and maintenance"
+)
+def discover(
+    companies: Annotated[
+        list[str] | None,
+        typer.Argument(metavar="NAME...", click_type=WORD, help="Company names to probe"),
+    ] = None,
+    url: Annotated[
+        str | None,
+        typer.Option(
+            "--url",
+            metavar="URL",
+            help="Extract the platform from a careers URL without fetching that URL",
+        ),
+    ] = None,
+    name: Annotated[
+        str | None,
+        typer.Option("--name", metavar="NAME", help="Company name to show with --url"),
+    ] = None,
+    platform: Annotated[
+        list[str] | None,
+        typer.Option(
+            "--platform", metavar="PLATFORM", help="Probe only these platform adapters; repeatable"
+        ),
+    ] = None,
+    exclude: Annotated[
+        list[str] | None,
+        typer.Option(
+            "--exclude",
+            metavar="NAME",
+            help="Skip these platform adapters while probing; repeatable",
+        ),
+    ] = None,
+    only: Annotated[
+        list[str] | None,
+        typer.Option(
+            "--company",
+            metavar="NAME",
+            help="With --verify, recheck only these registry companies; repeatable",
+        ),
+    ] = None,
+    expect_size: Annotated[
+        str | None,
+        typer.Option(
+            "--expect-size",
+            metavar="SIZE",
+            help="Optional employer-size hint: startup, mid, large",
+        ),
+    ] = None,
+    request_log: Annotated[
+        Path | None,
+        typer.Option(
+            "--request-log",
+            metavar="FILE",
+            help="Write outbound HTTP requests to this JSON Lines file",
+        ),
+    ] = None,
+    scheduled_progress: Annotated[
+        Path | None,
+        typer.Option("--scheduled-progress", hidden=True),
+    ] = None,
+    verify: Annotated[
+        bool,
+        typer.Option(
+            "--verify",
+            help="Recheck existing registry companies",
+        ),
+    ] = False,
+    unregistered: Annotated[
+        bool,
+        typer.Option(
+            "--unregistered",
+            help="Probe companies found in feeds but missing from the registry",
+        ),
+    ] = False,
+    direct_only: Annotated[
+        bool,
+        typer.Option(
+            "--direct-only",
+            help="With --unregistered, verify only ATS links already present in feed applications",
+        ),
+    ] = False,
+    adopt_unnamed: Annotated[
+        bool,
+        typer.Option(
+            "--adopt-unnamed",
+            help="With --direct-only, also adopt boards whose platform publishes no board name",
+        ),
+    ] = False,
+    limit: Annotated[
+        int,
+        typer.Option(
+            "--limit",
+            metavar="N",
+            click_type=_count(1, 2000),
+            help="Maximum unregistered company names to probe",
+        ),
+    ] = 40,
+    show_all: Annotated[
+        bool,
+        typer.Option("--all", help="List every result instead of the first few"),
+    ] = False,
+    db: DatabaseOption = None,
+    apply: Annotated[
+        bool,
+        typer.Option(
+            "--apply",
+            help="Write results to the registry; requires --verify or --unregistered",
+        ),
+    ] = False,
+    registry: RegistryOption = None,
+) -> None:
+    from contextlib import ExitStack
+
+    from stage.cli.logfile import open_request_log
+    from stage.cli.render import failure, plain, render_discovery, terminal
+    from stage.cli.runlock import AnotherRunInProgressError, single_run
+    from stage.cli.schedule_state import ScheduleStateWriter
+    from stage.companies import RegistryError
+    from stage.companies import load_companies as load_registry
+    from stage.domain import DiscoveryEvent, DiscoveryFinished, EmployerSize, Platform
+    from stage.services.discover import NoMatchingCompanyError
+
+    console = terminal()
+    scheduled_state = (
+        ScheduleStateWriter.open(scheduled_progress, "discover")
+        if scheduled_progress is not None
+        else None
+    )
+    progress = scheduled_state.discovery_event if scheduled_state is not None else None
+
+    if url is not None and companies:
+        console.print("[red]Pass either company names or --url, not both.[/red]")
+        raise typer.Exit(code=2)
+    if verify and unregistered:
+        console.print("[red]Pass either --verify or --unregistered, not both.[/red]")
+        raise typer.Exit(code=2)
+    if direct_only and not unregistered:
+        console.print("[red]--direct-only requires --unregistered.[/red]")
+        raise typer.Exit(code=2)
+    if adopt_unnamed and not direct_only:
+        console.print(
+            "[red]--adopt-unnamed requires --direct-only[/red] — the token must come from an "
+            "apply URL."
+        )
+        raise typer.Exit(code=2)
+    if apply and not (verify or unregistered):
+        console.print("[red]--apply only means something with --verify or --unregistered.[/red]")
+        raise typer.Exit(code=2)
+    if platform and exclude:
+        console.print("[red]Pass either --platform or --exclude, not both.[/red]")
+        raise typer.Exit(code=2)
+    if not verify and not unregistered and url is None and not companies:
+        console.print(
+            "[red]Nothing to discover.[/red] Pass a company name, a careers page with "
+            "[bold]--url[/bold], or [bold]--verify[/bold] to re-probe the registry."
+        )
+        raise typer.Exit(code=2)
+
+    try:
+        size = _parse_enum(expect_size, EmployerSize, "--expect-size")
+        platforms = (
+            [_require_enum(value, Platform, "--platform") for value in platform]
+            if platform
+            else None
+        )
+        excluded = (
+            [_require_enum(value, Platform, "--exclude") for value in exclude] if exclude else None
+        )
+    except InvalidOptionError as exc:
+        _print_failure(exc)
+        raise typer.Exit(code=2) from exc
+
+    async def run() -> object:
+        from datetime import UTC, datetime
+
+        from stage.services.discover import probe_companies, resolve_careers_url
+
+        today = datetime.now(UTC).date()
+        with ExitStack() as stack:
+            stream = (
+                stack.enter_context(open_request_log(request_log))
+                if request_log is not None
+                else None
+            )
+            if verify:
+                from stage.companies import load_companies, update_registry
+                from stage.services.discover import apply_verification, verify_registry
+
+                rows = checked if checked is not None else load_companies(registry)
+                outcome = await render_discovery(
+                    console,
+                    verify_registry(rows, platforms=platforms, excluded=excluded, only=only),
+                    verified_on=today,
+                    request_log=stream,
+                    collect=True,
+                    progress=progress,
+                )
+                if apply and isinstance(outcome, DiscoveryFinished):
+
+                    def update(
+                        existing: tuple[Any, ...],
+                    ) -> tuple[tuple[Any, ...], tuple[int, int]]:
+                        updated, ok, off = apply_verification(existing, outcome, today)
+                        return updated, (ok, off)
+
+                    target, (ok, off) = update_registry(update, registry)
+                    console.print(
+                        plain(
+                            f"\napplied — {ok} row(s) verified, {off} disabled, written to {target}"
+                        )
+                    )
+                return isinstance(outcome, DiscoveryFinished) and bool(outcome.matched)
+            if url is not None:
+
+                async def once() -> AsyncIterator[DiscoveryEvent]:
+                    yield resolve_careers_url(url)
+
+                return await render_discovery(console, once(), display_name=name, progress=progress)
+
+            if unregistered:
+                adopted = await _adopt_unregistered(
+                    console,
+                    registry=registry,
+                    db=db,
+                    platforms=platforms,
+                    excluded=excluded,
+                    size=size,
+                    limit=limit,
+                    direct_only=direct_only,
+                    adopt_unnamed=adopt_unnamed,
+                    apply_rows=apply,
+                    today=today,
+                    stream=stream,
+                    progress=progress,
+                    show_all=show_all,
+                )
+                if scheduled_state is not None:
+                    scheduled_state.heartbeat()
+                return adopted
+
+            events = probe_companies(
+                companies or [], platforms=platforms, excluded=excluded, size=size
+            )
+            return await render_discovery(
+                console, events, verified_on=today, request_log=stream, progress=progress
+            )
+
+    checked = None
+    if verify:
+        try:
+            checked = load_registry(registry)
+        except RegistryError as exc:
+            console.print(failure(exc))
+            raise typer.Exit(code=2) from exc
+
+    try:
+        if url is not None:
+            resolved = run_async(run())
+        else:
+            with single_run("discover", _lock_path(db)):
+                resolved = run_async(run())
+    except AnotherRunInProgressError as exc:
+        if scheduled_state is not None:
+            scheduled_state.blocked(str(exc))
+        console.print(failure(exc))
+        raise typer.Exit(code=2) from exc
+    except (RegistryError, NoMatchingCompanyError) as exc:
+        console.print(failure(exc))
+        raise typer.Exit(code=2) from exc
+    raise typer.Exit(code=0 if resolved or unregistered else 1)