stage-cli 1.0.0__py3-none-any.whl

stage/sources/espresso.py

@@ -0,0 +1,154 @@
+from datetime import datetime
+from typing import ClassVar
+
+from bs4 import BeautifulSoup
+from bs4.element import Tag
+
+from stage.domain import Job, SourceSignals, job_id
+from stage.http import HttpClient, HttpStatusError
+from stage.sources import register_feed
+from stage.sources._text import collapse_whitespace
+from stage.sources.base import FetchResult, PayloadValidationError, capture_payload, malformed_note
+
+HOST = "www.espresso-jobs.com"
+SEARCH = f"https://{HOST}/emploi"
+POSTING = f"https://{HOST}/emploi/{{id}}/{{slug}}"
+TERMS = ("stage", "stagiaire", "intern", "internship")
+PAGE_CAP = 3
+PAGE_SIZE = 21
+MAX_ROWS = 1000
+INTERNSHIP_BADGE = "stage"
+ROW_SELECTOR = "div.job_index-content_list_item"
+
+
+def badge(row: Tag) -> str:
+    found = row.select_one("p.job_index-content_list_item_infos-type")
+    if found is None:
+        return ""
+    return collapse_whitespace(found.get_text(" ", strip=True))
+
+
+def field(row: Tag, selector: str) -> str:
+    found = row.select_one(selector)
+    if found is None:
+        return ""
+    return collapse_whitespace(found.get_text(" ", strip=True))
+
+
+def where(row: Tag) -> str:
+    found = row.select_one("div.job-location-info")
+    if found is None:
+        return "Québec, Canada"
+    city = collapse_whitespace(str(found.get("data-city") or ""))
+    province = collapse_whitespace(str(found.get("data-province") or ""))
+    parts = [part for part in (city, province, "Canada") if part]
+    return ", ".join(parts)
+
+
+@register_feed
+class EspressoJobsFeed:
+    name: ClassVar[str] = "espresso-jobs"
+    rate_profile: ClassVar[str] = "feeds"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    bucket_key: ClassVar[str] = "espresso-jobs"
+
+    def season_year(self, now: datetime) -> int:
+        return now.year
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        return tuple(f"{SEARCH}?keyword={term}&distance=all&page_no=1" for term in TERMS)
+
+    async def fetch(self, client: HttpClient, now: datetime) -> FetchResult:
+        seen: dict[str, Job] = {}
+        malformed = 0
+        truncated = False
+        searched = False
+        empty: list[str] = []
+        last_empty = ""
+
+        exhausted: list[str] = []
+
+        for term in TERMS:
+            for page in range(1, PAGE_CAP + 1):
+                url = f"{SEARCH}?keyword={term}&distance=all&page_no={page}"
+                try:
+                    text = await client.get_text(url, revalidate=page > 1)
+                except HttpStatusError as exc:
+                    if page > 1 and exc.status == 404:
+                        exhausted.append(term)
+                        break
+                    raise
+                if text.not_modified:
+                    break
+                rows = self._rows(text.text)
+                if not rows:
+                    if page == 1:
+                        empty.append(term)
+                        last_empty = text.text
+                    break
+                searched = True
+                for row in rows:
+                    job, dropped = self._to_job(row, now)
+                    malformed += dropped
+                    if job is not None:
+                        seen.setdefault(job.id, job)
+                if len(seen) >= MAX_ROWS:
+                    truncated = True
+                    break
+                if len(rows) < PAGE_SIZE:
+                    break
+            if truncated:
+                break
+
+        if not searched:
+            captured = capture_payload(self.name, "search", {"head": last_empty[:4000]})
+            raise PayloadValidationError(
+                f"{self.name}: no query returned a listing row, so the results page changed "
+                f"shape rather than matching nothing; captured at {captured}"
+            )
+
+        notes = []
+        if empty:
+            notes.append(f"{len(empty)} of {len(TERMS)} searches matched nothing")
+        if exhausted:
+            notes.append(f"{len(exhausted)} search(es) ran past their last page, which answers 404")
+        if truncated:
+            notes.append(f"stopped at the {MAX_ROWS}-posting cap for one run")
+        if malformed:
+            notes.append(malformed_note(malformed))
+        return FetchResult(
+            jobs=tuple(seen.values()),
+            authoritative=False,
+            degraded="; ".join(notes),
+        )
+
+    @staticmethod
+    def _rows(text: str) -> list[Tag]:
+        return BeautifulSoup(text, "html.parser").select(ROW_SELECTOR)
+
+    def _to_job(self, row: Tag, now: datetime) -> tuple[Job | None, int]:
+        identifier = collapse_whitespace(str(row.get("id") or ""))
+        slug = collapse_whitespace(str(row.get("data-slug") or ""))
+        title = field(row, "h2.job_index-content_list_item-title")
+        if not identifier or not slug or not title:
+            return None, 1
+        declared = badge(row)
+        if declared.lower() != INTERNSHIP_BADGE:
+            return None, 0
+        company = field(row, "p.job_index-content_list_item-company") or "Espresso-Jobs"
+        return (
+            Job(
+                id=job_id(self.name, "search", identifier),
+                source=self.name,
+                company=company,
+                title_raw=title,
+                title_normalized=title.lower(),
+                apply_url_raw=POSTING.format(id=identifier, slug=slug),
+                description="",
+                location_raw=where(row),
+                first_seen=now,
+                last_seen=now,
+                signals=SourceSignals(employment_type=declared.lower()),
+            ),
+            0,
+        )

stage/sources/feed.py

@@ -0,0 +1,44 @@
+from datetime import datetime
+from typing import ClassVar, Protocol, runtime_checkable
+
+from stage.http import HttpClient
+from stage.sources.base import FetchResult
+
+_FEEDS: dict[str, "FeedAdapter"] = {}
+
+
+@runtime_checkable
+class FeedAdapter(Protocol):
+    name: ClassVar[str]
+    rate_profile: ClassVar[str]
+    hosts: ClassVar[frozenset[str]]
+    bucket_key: ClassVar[str]
+
+    def season_year(self, now: datetime) -> int:
+        pass
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        pass
+
+    async def fetch(self, client: HttpClient, now: datetime) -> FetchResult:
+        pass
+
+
+def register_feed[F: FeedAdapter](cls: type[F]) -> type[F]:
+    adapter = cls()
+    existing = _FEEDS.get(adapter.name)
+    if existing is not None and type(existing) is not cls:
+        raise ValueError(f"two feeds claim the name {adapter.name!r}")
+    _FEEDS[adapter.name] = adapter
+    return cls
+
+
+def get_feeds() -> dict[str, FeedAdapter]:
+    from stage.sources import load_builtins
+
+    load_builtins()
+    return dict(_FEEDS)
+
+
+def upcoming_season_year(now: datetime, rolls_in_month: int = 8) -> int:
+    return now.year + 1 if now.month >= rolls_in_month else now.year

stage/sources/greenhouse.py

@@ -0,0 +1,104 @@
+from collections.abc import Sequence
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict
+
+from stage.domain import Company, Job, Platform, job_id
+from stage.http import HttpClient, ResponseTooLargeError
+from stage.sources import register
+from stage.sources._text import collapse_whitespace, strip_html
+from stage.sources.base import BoardAdapter, FetchResult, malformed_note
+
+BASE_URL = "https://boards-api.greenhouse.io/v1/boards/{slug}/jobs"
+HOST = "boards-api.greenhouse.io"
+
+
+class GreenhouseLocation(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    name: str | None = ""
+
+
+class GreenhouseJob(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: int
+    title: str
+    absolute_url: str
+    updated_at: datetime | None = None
+    location: GreenhouseLocation | None = None
+    content: str = ""
+
+
+class GreenhouseBoard(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    jobs: list[Any]
+
+
+@register
+class GreenhouseAdapter(BoardAdapter):
+    name: ClassVar[str] = "greenhouse"
+    platform: ClassVar[Platform] = Platform.GREENHOUSE
+    rate_profile: ClassVar[str] = "broad"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    detail_budget: ClassVar[int] = 0
+    max_requests_per_company: ClassVar[int] = 2
+
+    base_url: ClassVar[str] = BASE_URL
+    query: ClassVar[tuple[tuple[str, str], ...]] = (("content", "true"),)
+    root_model: ClassVar[type[BaseModel] | None] = GreenhouseBoard
+    rows_field: ClassVar[str] = "jobs"
+    row_model: ClassVar[type[BaseModel]] = GreenhouseJob
+
+    async def fetch(
+        self,
+        company: Company,
+        client: HttpClient,
+        now: datetime,
+        facets: object = None,
+        details: Sequence[str] = (),
+    ) -> FetchResult:
+        url = self.url_for(company)
+        try:
+            response = await client.get_json(url, params={"content": "true"})
+        except ResponseTooLargeError:
+            return await self._without_descriptions(company, client, now, url)
+        if response.not_modified:
+            return FetchResult(not_modified=True)
+        return self.result(company, response.payload, now)
+
+    async def _without_descriptions(
+        self, company: Company, client: HttpClient, now: datetime, url: str
+    ) -> FetchResult:
+        response = await client.get_json(url, params={"content": "false"})
+        if response.not_modified:
+            return FetchResult(not_modified=True)
+        postings, dropped = self.validate(company, response.payload)
+        notes = ["board exceeds the response cap with content=true; fetched without descriptions"]
+        if dropped:
+            notes.append(malformed_note(dropped))
+        return FetchResult(
+            jobs=tuple(self.to_job(company, posting, now) for posting in postings),
+            degraded="; ".join(notes),
+            authoritative=not dropped,
+            stale_urls=(f"{url}?content=false",),
+        )
+
+    def to_job(self, company: Company, row: Any, now: datetime) -> Job:
+        return Job(
+            id=job_id(self.name, company.slug, str(row.id)),
+            source=self.name,
+            company=company.name,
+            title_raw=row.title,
+            title_normalized=collapse_whitespace(row.title),
+            apply_url_raw=row.absolute_url,
+            description=strip_html(row.content),
+            location_raw=collapse_whitespace(
+                row.location.name if row.location and row.location.name else ""
+            ),
+            first_seen=now,
+            last_seen=now,
+            source_posted_at=row.updated_at,
+        )

stage/sources/jobbank.py

@@ -0,0 +1,147 @@
+import re
+from datetime import datetime
+from typing import ClassVar
+
+from bs4 import BeautifulSoup
+from bs4.element import Tag
+
+from stage.domain import Job, SourceSignals, job_id
+from stage.http import HttpClient
+from stage.sources import register_feed
+from stage.sources._text import collapse_whitespace
+from stage.sources.base import FetchResult, PayloadValidationError, capture_payload, malformed_note
+
+HOST = "www.jobbank.gc.ca"
+SEARCH = f"https://{HOST}/jobsearch/jobsearch"
+POSTING = f"https://{HOST}/jobsearch/jobposting/{{id}}"
+TERMS = ("programmer", "developer", "software", "informatique", "programmeur", "développeur")
+PROVINCES = ("QC", "ON", "BC", "AB")
+PAGE_CAP = 2
+MAX_ROWS = 2000
+_ARTICLE_ID = re.compile(r"^article-(\d+)$")
+KEPT_FLAGS = ("jobinternshipflag", "jobstudentflag")
+
+
+def posting_id(article: Tag) -> str:
+    found = _ARTICLE_ID.match(str(article.get("id") or ""))
+    return found.group(1) if found else ""
+
+
+def declared_term(article: Tag) -> str:
+    for flag in KEPT_FLAGS:
+        found = article.select_one(f".{flag}")
+        if found is not None:
+            return collapse_whitespace(found.get_text(" ", strip=True))
+    return ""
+
+
+def field(article: Tag, selector: str) -> str:
+    found = article.select_one(selector)
+    if found is None:
+        return ""
+    for hidden in found.select(".wb-inv"):
+        hidden.decompose()
+    return collapse_whitespace(found.get_text(" ", strip=True))
+
+
+@register_feed
+class JobBankFeed:
+    name: ClassVar[str] = "jobbank"
+    rate_profile: ClassVar[str] = "jobbank"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    bucket_key: ClassVar[str] = "jobbank"
+
+    def season_year(self, now: datetime) -> int:
+        return now.year
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        return tuple(
+            f"{SEARCH}?searchstring={term}&fprov={province}"
+            for province in PROVINCES
+            for term in TERMS
+        )
+
+    async def fetch(self, client: HttpClient, now: datetime) -> FetchResult:
+        seen: dict[str, Job] = {}
+        malformed = 0
+        truncated = False
+        searched = False
+        empty: list[str] = []
+        last_empty = ""
+        for province in PROVINCES:
+            for term in TERMS:
+                for page in range(1, PAGE_CAP + 1):
+                    url = f"{SEARCH}?searchstring={term}&fprov={province}&page={page}"
+                    text = await client.get_text(url, revalidate=page > 1)
+                    if text.not_modified:
+                        break
+                    rows = self._articles(text.text)
+                    if not rows:
+                        if page == 1:
+                            empty.append(f"{province}/{term}")
+                            last_empty = text.text
+                        break
+                    searched = True
+                    for article in rows:
+                        job, dropped = self._to_job(article, now)
+                        malformed += dropped
+                        if job is not None:
+                            seen.setdefault(job.id, job)
+                    if len(seen) >= MAX_ROWS:
+                        truncated = True
+                        break
+                if truncated:
+                    break
+            if truncated:
+                break
+
+        if not searched:
+            captured = capture_payload(self.name, "search", {"head": last_empty[:4000]})
+            raise PayloadValidationError(
+                f"{self.name}: no query returned an <article> row, so the results page changed "
+                f"shape rather than matching nothing; captured at {captured}"
+            )
+
+        notes = []
+        if empty:
+            notes.append(f"{len(empty)} of {len(PROVINCES) * len(TERMS)} searches matched nothing")
+        if truncated:
+            notes.append(f"stopped at the {MAX_ROWS}-posting cap for one run")
+        if malformed:
+            notes.append(malformed_note(malformed))
+        return FetchResult(
+            jobs=tuple(seen.values()),
+            authoritative=False,
+            degraded="; ".join(notes),
+        )
+
+    @staticmethod
+    def _articles(text: str) -> list[Tag]:
+        return BeautifulSoup(text, "html.parser").select("article")
+
+    def _to_job(self, article: Tag, now: datetime) -> tuple[Job | None, int]:
+        identifier = posting_id(article)
+        title = field(article, ".noctitle")
+        term = declared_term(article)
+        if not identifier or not title:
+            return None, 1
+        if not term:
+            return None, 0
+        company = field(article, ".business") or "Job Bank"
+        city = field(article, ".location")
+        return (
+            Job(
+                id=job_id(self.name, "search", identifier),
+                source=self.name,
+                company=company,
+                title_raw=title,
+                title_normalized=title.lower(),
+                apply_url_raw=POSTING.format(id=identifier),
+                description="",
+                location_raw=city or "Canada",
+                first_seen=now,
+                last_seen=now,
+                signals=SourceSignals(employment_type=term.lower()),
+            ),
+            0,
+        )

stage/sources/jobvite.py

@@ -0,0 +1,133 @@
+import re
+from collections.abc import Sequence
+from datetime import datetime
+from typing import Any, ClassVar
+from urllib.parse import urljoin
+
+from bs4 import Tag
+from pydantic import BaseModel, ConfigDict
+
+from stage.domain import Company, Job, Platform, job_id
+from stage.http import HttpClient
+from stage.sources import register
+from stage.sources._text import collapse_whitespace
+from stage.sources.base import (
+    BoardAdapter,
+    FetchResult,
+    NonEmptyStr,
+    PayloadValidationError,
+    capture_payload,
+    malformed_note,
+    validate_rows,
+)
+from stage.sources.custom_json import html_rows
+
+HOST = "jobs.jobvite.com"
+BASE_URL = "https://jobs.jobvite.com/{slug}/jobs"
+LISTING = "table.jv-job-list"
+ROW = "table.jv-job-list tbody tr"
+NAME_CELL = "td.jv-job-list-name"
+LOCATION_CELL = "td.jv-job-list-location"
+ONCLICK = re.compile(r"""location\.href\s*=\s*['"](?P<href>[^'"]{1,300})['"]""")
+
+
+class JobviteRow(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    title: NonEmptyStr
+    url: NonEmptyStr
+    location: str = ""
+
+    def posting_id(self) -> str:
+        return self.url.split("?", 1)[0].rstrip("/").rsplit("/", 1)[-1]
+
+
+def _text(block: Tag, selector: str) -> str:
+    found = block.select_one(selector)
+    return collapse_whitespace(found.get_text(" ", strip=True)) if found is not None else ""
+
+
+def _href(block: Tag) -> str:
+    link = block.select_one(f"{NAME_CELL} a")
+    if link is not None:
+        return str(link.get("href", ""))
+    found = ONCLICK.search(str(block.get("onclick", "")))
+    return found.group("href") if found is not None else ""
+
+
+def _row(block: Tag) -> dict[str, str]:
+    href = _href(block)
+    return {
+        "title": _text(block, NAME_CELL),
+        "url": urljoin(f"https://{HOST}/", href) if href else "",
+        "location": _text(block, LOCATION_CELL),
+    }
+
+
+@register
+class JobviteAdapter(BoardAdapter):
+    name: ClassVar[str] = "jobvite"
+    platform: ClassVar[Platform] = Platform.JOBVITE
+    rate_profile: ClassVar[str] = "moderate"
+    bucket_key: ClassVar[str] = "jobvite"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    detail_budget: ClassVar[int] = 0
+    max_requests_per_company: ClassVar[int] = 1
+
+    base_url: ClassVar[str] = BASE_URL
+    row_model: ClassVar[type[BaseModel]] = JobviteRow
+
+    async def fetch(
+        self,
+        company: Company,
+        client: HttpClient,
+        now: datetime,
+        facets: object = None,
+        details: Sequence[str] = (),
+    ) -> FetchResult:
+        response = await client.get_text(self.url_for(company))
+        if response.not_modified:
+            return FetchResult(not_modified=True)
+        return self.result(company, response.text, now)
+
+    def result(self, company: Company, payload: Any, now: datetime) -> FetchResult:
+        blocks = self._blocks(company, str(payload))
+        listed = [block for block in blocks if block.select_one(NAME_CELL) is not None]
+        rows, dropped = validate_rows(
+            self.row_model, [_row(block) for block in listed], source=self.name, slug=company.slug
+        )
+        truncated = len(listed) != len(blocks)
+        notes = [malformed_note(dropped)] if dropped else []
+        if truncated:
+            notes.append(
+                "the board hides the rest of some categories behind a Show More search page, "
+                "so this listing closes nothing"
+            )
+        return FetchResult(
+            jobs=tuple(self.to_job(company, row, now) for row in rows),
+            degraded="; ".join(note for note in notes if note),
+            authoritative=not dropped and not truncated,
+        )
+
+    def _blocks(self, company: Company, text: str) -> list[Tag]:
+        if not html_rows(text, LISTING):
+            captured = capture_payload(self.name, company.slug, {"text": text})
+            raise PayloadValidationError(
+                f"{self.name}/{company.slug}: the board page carries no {LISTING!r} listing, "
+                f"so this is drift or a retired tenant; raw page captured at {captured}"
+            )
+        return html_rows(text, ROW)
+
+    def to_job(self, company: Company, row: Any, now: datetime) -> Job:
+        return Job(
+            id=job_id(self.name, company.slug, row.posting_id()),
+            source=self.name,
+            company=company.name,
+            title_raw=row.title,
+            title_normalized=row.title.lower(),
+            apply_url_raw=row.url,
+            description="",
+            location_raw=row.location,
+            first_seen=now,
+            last_seen=now,
+        )

stage/sources/lever.py

@@ -0,0 +1,76 @@
+from datetime import UTC, datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from stage.domain import Company, Job, Platform, SourceSignals, job_id
+from stage.sources import register
+from stage.sources._text import collapse_whitespace
+from stage.sources.base import BoardAdapter
+from stage.sources.platforms import safe_cased_slug
+
+BASE_URL = "https://api.lever.co/v0/postings/{slug}"
+HOST = "api.lever.co"
+
+
+class LeverCategories(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    location: str = ""
+    allLocations: list[str] = Field(default_factory=list)
+    commitment: str = ""
+    team: str = ""
+    department: str = ""
+
+    def label(self) -> str:
+        if self.allLocations:
+            return " / ".join(dict.fromkeys(self.allLocations))
+        return self.location
+
+
+class LeverPosting(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: str
+    text: str
+    hostedUrl: str = ""
+    applyUrl: str = ""
+    createdAt: int | None = None
+    categories: LeverCategories | None = None
+    descriptionPlain: str = ""
+    additionalPlain: str = ""
+
+
+@register
+class LeverAdapter(BoardAdapter):
+    name: ClassVar[str] = "lever"
+    platform: ClassVar[Platform] = Platform.LEVER
+    rate_profile: ClassVar[str] = "standard"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    detail_budget: ClassVar[int] = 0
+    max_requests_per_company: ClassVar[int] = 1
+
+    base_url: ClassVar[str] = BASE_URL
+    query: ClassVar[tuple[tuple[str, str], ...]] = (("mode", "json"),)
+    row_model: ClassVar[type[BaseModel]] = LeverPosting
+    slug_validator = safe_cased_slug
+
+    def to_job(self, company: Company, row: Any, now: datetime) -> Job:
+        posted = datetime.fromtimestamp(row.createdAt / 1000, tz=UTC) if row.createdAt else None
+        parts = (row.descriptionPlain, row.additionalPlain)
+        return Job(
+            id=job_id(self.name, company.slug, row.id),
+            source=self.name,
+            company=company.name,
+            title_raw=row.text,
+            title_normalized=collapse_whitespace(row.text),
+            apply_url_raw=row.hostedUrl or row.applyUrl,
+            description="\n\n".join(part for part in parts if part),
+            location_raw=collapse_whitespace(row.categories.label() if row.categories else ""),
+            first_seen=now,
+            last_seen=now,
+            source_posted_at=posted,
+            signals=SourceSignals(
+                employment_type=row.categories.commitment if row.categories else ""
+            ),
+        )