stage-cli 1.0.0__py3-none-any.whl

stage/sources/__init__.py

@@ -0,0 +1,82 @@
+import importlib
+import pkgutil
+from types import MappingProxyType
+
+from stage.domain import Platform
+from stage.sources.base import (
+    Adapter,
+    AdapterError,
+    FetchResult,
+    PayloadValidationError,
+    capture_payload,
+    convert_rows,
+)
+from stage.sources.feed import (
+    FeedAdapter,
+    get_feeds,
+    register_feed,
+    upcoming_season_year,
+)
+
+_ADAPTERS: dict[str, Adapter] = {}
+_LOADED = False
+
+
+def register[A: Adapter](cls: type[A]) -> type[A]:
+    adapter = cls()
+    existing = _ADAPTERS.get(adapter.name)
+    if existing is not None and type(existing) is not cls:
+        raise AdapterError(f"two adapters claim the name {adapter.name!r}")
+    _ADAPTERS[adapter.name] = adapter
+    return cls
+
+
+def load_builtins() -> None:
+    global _LOADED
+    if _LOADED:
+        return
+    for module in pkgutil.iter_modules(__path__):
+        if module.name.startswith("_") or module.name in ("base", "feed", "platforms"):
+            continue
+        importlib.import_module(f"{__name__}.{module.name}")
+    _LOADED = True
+
+
+def get_adapters() -> MappingProxyType[str, Adapter]:
+    load_builtins()
+    return MappingProxyType(_ADAPTERS)
+
+
+def get_adapter(name: str) -> Adapter:
+    adapters = get_adapters()
+    try:
+        return adapters[name]
+    except KeyError as exc:
+        known = ", ".join(sorted(adapters)) or "none"
+        raise AdapterError(f"no adapter named {name!r} (known: {known})") from exc
+
+
+def adapter_for_platform(platform: Platform) -> Adapter | None:
+    for adapter in get_adapters().values():
+        if adapter.platform is platform:
+            return adapter
+    return None
+
+
+__all__ = [
+    "Adapter",
+    "AdapterError",
+    "FeedAdapter",
+    "FetchResult",
+    "PayloadValidationError",
+    "adapter_for_platform",
+    "capture_payload",
+    "convert_rows",
+    "get_feeds",
+    "get_adapter",
+    "get_adapters",
+    "load_builtins",
+    "register",
+    "register_feed",
+    "upcoming_season_year",
+]

stage/sources/_text.py

@@ -0,0 +1,79 @@
+import html
+import re
+
+LONGEST_TAG = 4096
+
+_RAW_OPEN = re.compile(rf"<(script|style)\b[^>]{{0,{LONGEST_TAG}}}>", re.IGNORECASE)
+_RAW_CLOSE = {
+    "script": re.compile(r"</script\s*>", re.IGNORECASE),
+    "style": re.compile(r"</style\s*>", re.IGNORECASE),
+}
+_BLOCK_BREAKS = re.compile(r"</(p|div|li|tr|h[1-6])>|<br\s*/?>", re.IGNORECASE)
+_CONTROL = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]")
+_PICTOGRAPH = "\U0001f000-\U0001faff\u2600-\u27bf\u2b00-\u2bff"
+_JOINER = "\ufe0f\u200d"
+_PICTOGRAPHS = re.compile(
+    f"[{_PICTOGRAPH}{_JOINER}]{{0,64}}[{_PICTOGRAPH}][{_PICTOGRAPH}{_JOINER}]{{0,64}}"
+)
+_BLANK_LINES = re.compile(r"\n{3,}")
+
+
+def _drop_raw_text_elements(text: str) -> str:
+    kept: list[str] = []
+    index = 0
+    unclosed: set[str] = set()
+    while (opened := _RAW_OPEN.search(text, index)) is not None:
+        name = opened.group(1).lower()
+        closed = None if name in unclosed else _RAW_CLOSE[name].search(text, opened.end())
+        if closed is None:
+            unclosed.add(name)
+            kept.append(text[index : opened.end()])
+            index = opened.end()
+            continue
+        kept.append(text[index : opened.start()])
+        kept.append(" ")
+        index = closed.end()
+    kept.append(text[index:])
+    return "".join(kept)
+
+
+def _drop_tags(text: str) -> str:
+    kept: list[str] = []
+    index = 0
+    closing = text.find(">")
+    while (opening := text.find("<", index)) != -1:
+        if closing < opening:
+            closing = text.find(">", opening + 1)
+        if closing == -1:
+            break
+        span = closing - opening - 1
+        if 1 <= span <= LONGEST_TAG:
+            kept.append(text[index:opening])
+            index = closing + 1
+            closing = text.find(">", index)
+        else:
+            kept.append(text[index : opening + 1])
+            index = opening + 1
+    kept.append(text[index:])
+    return "".join(kept)
+
+
+def _strip_line_ends(text: str) -> str:
+    return "\n".join(line.rstrip(" \t") for line in text.split("\n"))
+
+
+def strip_html(raw: str) -> str:
+    text = html.unescape(raw)
+    text = _drop_raw_text_elements(text)
+    text = _BLOCK_BREAKS.sub("\n", text)
+    text = _drop_tags(text)
+    text = html.unescape(text)
+    text = _CONTROL.sub("", text)
+    text = _strip_line_ends(text)
+    return _BLANK_LINES.sub("\n\n", text).strip()
+
+
+def collapse_whitespace(raw: str) -> str:
+    stripped = _CONTROL.sub("", raw)
+    cleaned = " ".join(_PICTOGRAPHS.sub(" ", stripped).split())
+    return cleaned or " ".join(stripped.split())

stage/sources/ashby.py

@@ -0,0 +1,93 @@
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from stage.domain import Company, Job, Platform, SourceSignals, job_id
+from stage.sources import register
+from stage.sources._text import collapse_whitespace, strip_html
+from stage.sources.base import BoardAdapter, NullableBool, NullableStr
+from stage.sources.platforms import safe_path_slug
+
+BASE_URL = "https://api.ashbyhq.com/posting-api/job-board/{slug}"
+HOST = "api.ashbyhq.com"
+
+
+class AshbyLocation(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    locationName: NullableStr = ""
+
+
+class AshbyPosting(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: str
+    title: str
+    location: NullableStr = ""
+    secondaryLocations: list[AshbyLocation] = Field(default_factory=list)
+    department: NullableStr = ""
+    team: NullableStr = ""
+    employmentType: NullableStr = ""
+    isListed: NullableBool = True
+    isRemote: NullableBool = False
+    publishedAt: datetime | None = None
+    jobUrl: NullableStr = ""
+    applyUrl: NullableStr = ""
+    descriptionPlain: NullableStr = ""
+    descriptionHtml: NullableStr = ""
+
+    def where(self) -> str:
+        names = [self.location] if self.location else []
+        names.extend(entry.locationName for entry in self.secondaryLocations)
+        seen = [name for name in dict.fromkeys(names) if name]
+        if not seen and self.isRemote:
+            return "Remote"
+        return " / ".join(seen)
+
+    def body(self) -> str:
+        if self.descriptionPlain:
+            return collapse_whitespace(self.descriptionPlain)
+        return collapse_whitespace(strip_html(self.descriptionHtml))
+
+
+class AshbyBoard(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    jobs: list[Any]
+
+
+@register
+class AshbyAdapter(BoardAdapter):
+    name: ClassVar[str] = "ashby"
+    platform: ClassVar[Platform] = Platform.ASHBY
+    rate_profile: ClassVar[str] = "standard"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    detail_budget: ClassVar[int] = 0
+    max_requests_per_company: ClassVar[int] = 1
+
+    base_url: ClassVar[str] = BASE_URL
+    slug_validator = safe_path_slug
+    root_model: ClassVar[type[BaseModel] | None] = AshbyBoard
+    rows_field: ClassVar[str] = "jobs"
+    row_model: ClassVar[type[BaseModel]] = AshbyPosting
+
+    def keep(self, row: Any) -> bool:
+        return bool(row.isListed)
+
+    def to_job(self, company: Company, row: Any, now: datetime) -> Job:
+        title = collapse_whitespace(row.title)
+        return Job(
+            id=job_id(self.name, company.slug, row.id),
+            source=self.name,
+            company=company.name,
+            title_raw=title,
+            title_normalized=title.lower(),
+            apply_url_raw=row.jobUrl or row.applyUrl,
+            description=row.body(),
+            location_raw=collapse_whitespace(row.where()),
+            first_seen=now,
+            last_seen=now,
+            source_posted_at=row.publishedAt,
+            signals=SourceSignals(employment_type=row.employmentType),
+        )

stage/sources/bamboohr.py

@@ -0,0 +1,80 @@
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict
+
+from stage.domain import Company, Job, Platform, job_id
+from stage.sources import register
+from stage.sources._text import collapse_whitespace
+from stage.sources.base import BoardAdapter, NullableBool, NullableStr
+
+HOST_TEMPLATE = "{slug}.bamboohr.com"
+PATH = "/careers/list"
+
+
+class BambooLocation(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    city: NullableStr = ""
+    state: NullableStr = ""
+    country: NullableStr = ""
+
+    def label(self) -> str:
+        return ", ".join(part for part in (self.city, self.state, self.country) if part)
+
+
+class BambooPosting(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: int
+    jobOpeningName: str
+    departmentLabel: NullableStr = ""
+    employmentStatusLabel: NullableStr = ""
+    location: BambooLocation | None = None
+    atsLocation: BambooLocation | None = None
+    isRemote: NullableBool = False
+
+    def where(self) -> str:
+        for candidate in (self.location, self.atsLocation):
+            if candidate is not None:
+                label = candidate.label()
+                if label:
+                    return label
+        return "Remote" if self.isRemote else ""
+
+
+class BambooBoard(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    result: list[Any]
+
+
+@register
+class BambooHrAdapter(BoardAdapter):
+    name: ClassVar[str] = "bamboohr"
+    platform: ClassVar[Platform] = Platform.BAMBOOHR
+    rate_profile: ClassVar[str] = "moderate"
+    bucket_key: ClassVar[str] = "bamboohr"
+    detail_budget: ClassVar[int] = 0
+    max_requests_per_company: ClassVar[int] = 1
+
+    host_template: ClassVar[str] = HOST_TEMPLATE
+    path: ClassVar[str] = PATH
+    root_model: ClassVar[type[BaseModel] | None] = BambooBoard
+    rows_field: ClassVar[str] = "result"
+    row_model: ClassVar[type[BaseModel]] = BambooPosting
+
+    def to_job(self, company: Company, row: Any, now: datetime) -> Job:
+        title = collapse_whitespace(row.jobOpeningName)
+        return Job(
+            id=job_id(self.name, company.slug, str(row.id)),
+            source=self.name,
+            company=company.name,
+            title_raw=title,
+            title_normalized=title.lower(),
+            apply_url_raw=f"https://{self.host_for(company)}/careers/{row.id}",
+            description="",
+            location_raw=collapse_whitespace(row.where()),
+            first_seen=now,
+            last_seen=now,
+        )

stage/sources/base.py

@@ -0,0 +1,225 @@
+import json
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Annotated, Any, ClassVar, Protocol, runtime_checkable
+
+from pydantic import BaseModel, BeforeValidator, StringConstraints, ValidationError
+
+from stage.domain import Company, Job, Platform, WorkdayCrawlStep, board_key
+from stage.http import HttpClient
+from stage.paths import capture_dir
+from stage.sources.platforms import SlugRejectedError, safe_slug
+
+
+class AdapterError(Exception):
+    pass
+
+
+class PayloadValidationError(AdapterError):
+    pass
+
+
+NullableStr = Annotated[str, BeforeValidator(lambda value: "" if value is None else value)]
+NonEmptyStr = Annotated[str, StringConstraints(min_length=1, strip_whitespace=True)]
+NullableBool = Annotated[bool, BeforeValidator(lambda value: False if value is None else value)]
+
+
+def validate_rows[ModelT: BaseModel](
+    model: type[ModelT], rows: Sequence[Any], *, source: str, slug: str
+) -> tuple[list[ModelT], int]:
+    kept: list[ModelT] = []
+    dropped = 0
+    for row in rows:
+        try:
+            kept.append(model.model_validate(row))
+        except ValidationError:
+            dropped += 1
+            capture_payload(f"{source}-posting", slug, row)
+    return kept, dropped
+
+
+def convert_rows[ModelT: BaseModel](
+    build: Callable[[ModelT], Job], rows: Sequence[ModelT], *, source: str, slug: str
+) -> tuple[list[Job], int]:
+    kept: list[Job] = []
+    dropped = 0
+    for row in rows:
+        try:
+            kept.append(build(row))
+        except (ValueError, OverflowError, OSError):
+            dropped += 1
+            capture_payload(f"{source}-posting", slug, row.model_dump(mode="json"))
+    return kept, dropped
+
+
+def malformed_note(dropped: int) -> str:
+    if not dropped:
+        return ""
+    return (
+        f"{dropped} posting(s) failed validation and were dropped, raw rows "
+        "captured; the listing is incomplete so it closes nothing"
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class FetchResult:
+    jobs: tuple[Job, ...] = field(default_factory=tuple)
+    not_modified: bool = False
+    authoritative: bool = True
+    degraded: str = ""
+    stale_urls: tuple[str, ...] = field(default_factory=tuple)
+    detail_fetches: tuple[object, ...] = field(default_factory=tuple)
+    facets: tuple[object, ...] = field(default_factory=tuple)
+    forgotten_facets: tuple[object, ...] = field(default_factory=tuple)
+    workday_crawl: WorkdayCrawlStep | None = None
+
+
+@runtime_checkable
+class Adapter(Protocol):
+    name: ClassVar[str]
+    platform: ClassVar[Platform]
+    rate_profile: ClassVar[str]
+    hosts: ClassVar[frozenset[str]]
+
+    bucket_key: ClassVar[str]
+
+    detail_budget: ClassVar[int]
+
+    rotation_slice: ClassVar[int]
+
+    max_requests_per_company: ClassVar[int]
+
+    def board_key(self, company: Company) -> str:
+        pass
+
+    def hosts_for(self, companies: Sequence[Company]) -> frozenset[str]:
+        pass
+
+    def plan(self, company: Company) -> tuple[str, ...]:
+        pass
+
+    async def fetch(
+        self,
+        company: Company,
+        client: HttpClient,
+        now: datetime,
+        facets: Any = None,
+        details: Sequence[str] = (),
+    ) -> FetchResult:
+        pass
+
+
+class BoardAdapter:
+    name: ClassVar[str]
+    platform: ClassVar[Platform]
+    rate_profile: ClassVar[str]
+    hosts: ClassVar[frozenset[str]] = frozenset()
+    bucket_key: ClassVar[str] = ""
+    detail_budget: ClassVar[int]
+    rotation_slice: ClassVar[int] = 0
+    max_requests_per_company: ClassVar[int]
+
+    row_model: ClassVar[type[BaseModel]]
+    root_model: ClassVar[type[BaseModel] | None] = None
+    rows_field: ClassVar[str] = ""
+    base_url: ClassVar[str] = ""
+    host_template: ClassVar[str] = ""
+    path: ClassVar[str] = ""
+    query: ClassVar[tuple[tuple[str, str], ...]] = ()
+    slug_validator: ClassVar[Callable[[str], str]] = safe_slug
+
+    def host_for(self, company: Company) -> str:
+        return self.host_template.format(slug=type(self).slug_validator(company.slug))
+
+    def url_for(self, company: Company) -> str:
+        if self.host_template:
+            return f"https://{self.host_for(company)}{self.path}"
+        return self.base_url.format(slug=type(self).slug_validator(company.slug))
+
+    def hosts_for(self, companies: Sequence[Company]) -> frozenset[str]:
+        if not self.host_template:
+            return self.hosts
+        allowed: set[str] = set()
+        for company in companies:
+            try:
+                allowed.add(self.host_for(company))
+            except SlugRejectedError:
+                continue
+        return frozenset(allowed)
+
+    def board_key(self, company: Company) -> str:
+        return board_key(self.name, company.slug)
+
+    def plan(self, company: Company) -> tuple[str, ...]:
+        url = self.url_for(company)
+        if self.query:
+            url = f"{url}?" + "&".join(f"{key}={value}" for key, value in self.query)
+        return (url,)
+
+    def params(self) -> dict[str, str] | None:
+        return dict(self.query) or None
+
+    async def fetch(
+        self,
+        company: Company,
+        client: HttpClient,
+        now: datetime,
+        facets: object = None,
+        details: Sequence[str] = (),
+    ) -> FetchResult:
+        response = await client.get_json(self.url_for(company), params=self.params())
+        if response.not_modified:
+            return FetchResult(not_modified=True)
+        return self.result(company, response.payload, now)
+
+    def result(self, company: Company, payload: Any, now: datetime) -> FetchResult:
+        rows, dropped = self.validate(company, payload)
+        kept = [row for row in rows if self.keep(row)]
+        return FetchResult(
+            jobs=tuple(self.to_job(company, row, now) for row in kept),
+            degraded=malformed_note(dropped),
+            authoritative=not dropped,
+        )
+
+    def keep(self, row: Any) -> bool:
+        return True
+
+    def validate(self, company: Company, payload: Any) -> tuple[list[Any], int]:
+        return validate_rows(
+            self.row_model, self.rows(company, payload), source=self.name, slug=company.slug
+        )
+
+    def rows(self, company: Company, payload: Any) -> Sequence[Any]:
+        if self.root_model is None:
+            if not isinstance(payload, list):
+                captured = capture_payload(self.name, company.slug, payload)
+                raise PayloadValidationError(
+                    f"{self.name}/{company.slug}: field '<root>' failed validation (expected "
+                    f"a JSON list of postings); raw payload captured at {captured}"
+                )
+            return payload
+        try:
+            root = self.root_model.model_validate(payload)
+        except ValidationError as exc:
+            captured = capture_payload(self.name, company.slug, payload)
+            first = exc.errors()[0]
+            field_name = ".".join(str(part) for part in first["loc"]) or "<root>"
+            raise PayloadValidationError(
+                f"{self.name}/{company.slug}: field {field_name!r} failed validation "
+                f"({first['msg']}); raw payload captured at {captured}"
+            ) from exc
+        listed = getattr(root, self.rows_field)
+        return listed if isinstance(listed, list) else []
+
+    def to_job(self, company: Company, row: Any, now: datetime) -> Job:
+        raise NotImplementedError
+
+
+def capture_payload(source: str, slug: str, payload: Any) -> str:
+    stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S%f")
+    target = capture_dir() / f"{source}-{slug}-{stamp}.json"
+    target.write_text(
+        json.dumps(payload, indent=2, ensure_ascii=False, default=str), encoding="utf-8"
+    )
+    return str(target)

stage/sources/breezy.py

@@ -0,0 +1,90 @@
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict
+
+from stage.domain import Company, Job, Platform, job_id
+from stage.sources import register
+from stage.sources._text import collapse_whitespace, strip_html
+from stage.sources.base import BoardAdapter, NullableBool, NullableStr
+
+HOST_TEMPLATE = "{slug}.breezy.hr"
+PATH = "/json"
+
+
+class BreezyCountry(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    name: NullableStr = ""
+
+
+class BreezyLocation(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    name: NullableStr = ""
+    city: NullableStr = ""
+    country: BreezyCountry | None = None
+    is_remote: NullableBool = False
+
+    def label(self) -> str:
+        if self.name:
+            return self.name
+        parts = [self.city, self.country.name if self.country else ""]
+        seen = [part for part in parts if part]
+        if not seen and self.is_remote:
+            return "Remote"
+        return ", ".join(seen)
+
+
+class BreezyType(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    name: str = ""
+
+
+class BreezyPosting(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: str
+    name: str
+    friendly_id: NullableStr = ""
+    type: BreezyType | None = None
+    education: NullableStr = ""
+    department: NullableStr = ""
+    description: NullableStr = ""
+    location: BreezyLocation | None = None
+    url: NullableStr = ""
+    published_date: datetime | None = None
+
+    def where(self) -> str:
+        return self.location.label() if self.location else ""
+
+
+@register
+class BreezyAdapter(BoardAdapter):
+    name: ClassVar[str] = "breezy"
+    platform: ClassVar[Platform] = Platform.BREEZY
+    rate_profile: ClassVar[str] = "moderate"
+    bucket_key: ClassVar[str] = "breezy"
+    detail_budget: ClassVar[int] = 0
+    max_requests_per_company: ClassVar[int] = 1
+
+    host_template: ClassVar[str] = HOST_TEMPLATE
+    path: ClassVar[str] = PATH
+    row_model: ClassVar[type[BaseModel]] = BreezyPosting
+
+    def to_job(self, company: Company, row: Any, now: datetime) -> Job:
+        title = collapse_whitespace(row.name)
+        return Job(
+            id=job_id(self.name, company.slug, row.id),
+            source=self.name,
+            company=company.name,
+            title_raw=title,
+            title_normalized=title.lower(),
+            apply_url_raw=row.url,
+            description=collapse_whitespace(strip_html(row.description)),
+            location_raw=collapse_whitespace(row.where()),
+            first_seen=now,
+            last_seen=now,
+            source_posted_at=row.published_date,
+        )