stage-cli 1.0.0__py3-none-any.whl

stage/sources/workday.py

@@ -0,0 +1,524 @@
+from collections.abc import Mapping, Sequence
+from dataclasses import replace
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from stage.domain import (
+    Company,
+    DetailFetch,
+    Job,
+    Platform,
+    SourceSignals,
+    WorkdayCrawl,
+    WorkdayCrawlStep,
+    WorkdayFacet,
+    board_key,
+    job_id,
+)
+from stage.http import HttpClient, HttpError
+from stage.lexicon import fold, internship_lexicon, workday_facet_lexicon
+from stage.sources import register
+from stage.sources._text import collapse_whitespace, strip_html
+from stage.sources.base import (
+    FetchResult,
+    PayloadValidationError,
+    capture_payload,
+    malformed_note,
+    validate_rows,
+)
+from stage.sources.platforms import SlugRejectedError, workday_target
+
+PAGE_SIZE = 20
+MAX_PAGES = 25
+MAX_CRAWL_PAGES = 100
+RESULT_CAP = 10_000
+CRAWL_PAGE_CAP = 6
+RETRY_RESERVE = 40
+
+
+class WorkdayPosting(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    title: str
+    externalPath: str = ""
+    locationsText: str = ""
+    postedOn: str = ""
+    bulletFields: list[str] = Field(default_factory=list)
+
+    def requisition(self) -> str:
+        for field in self.bulletFields:
+            cleaned = field.strip()
+            if cleaned:
+                return cleaned
+        tail = self.externalPath.rstrip("/").rsplit("_", 1)
+        return tail[-1] if len(tail) == 2 and tail[-1] else self.externalPath.rstrip("/")
+
+
+class WorkdayFacetValue(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: str = ""
+    descriptor: str = ""
+    count: int = 0
+
+
+class WorkdayFacetGroup(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    facetParameter: str = ""
+    descriptor: str = ""
+    values: list[WorkdayFacetValue] = Field(default_factory=list)
+
+
+class WorkdayPage(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    total: int = 0
+    jobPostings: list[WorkdayPosting]
+    facets: list[WorkdayFacetGroup] | None = None
+
+
+class WorkdayRawPage(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    total: int = 0
+    jobPostings: list[dict[str, Any]]
+    facets: list[WorkdayFacetGroup] | None = None
+
+
+def _matches(folded: str, descriptors: frozenset[str]) -> bool:
+    padded = f" {folded} "
+    if any(f" {phrase} " in padded for phrase in internship_lexicon().blocked_bigrams):
+        return False
+    return any(f" {phrase} " in padded for phrase in descriptors)
+
+
+def resolve_facet(page: WorkdayPage, tenant: str, site: str, now: datetime) -> WorkdayFacet | None:
+    parameters, descriptors = workday_facet_lexicon()
+    groups = {group.facetParameter: group for group in page.facets or ()}
+    for parameter in parameters:
+        group = groups.get(parameter)
+        if group is None:
+            continue
+        matched = [
+            value
+            for value in group.values
+            if value.id and _matches(fold(value.descriptor), descriptors)
+        ]
+        if matched:
+            return WorkdayFacet(
+                tenant=tenant,
+                site=site,
+                parameter=parameter,
+                facet_ids=tuple(value.id for value in matched),
+                descriptor=", ".join(value.descriptor for value in matched),
+                resolved_at=now,
+            )
+    return None
+
+
+def facet_still_offered(page: WorkdayPage, facet: WorkdayFacet) -> bool:
+    if page.facets is None:
+        return True
+    for group in page.facets:
+        if group.facetParameter != facet.parameter:
+            continue
+        offered = {value.id for value in group.values}
+        if all(facet_id in offered for facet_id in facet.facet_ids):
+            return True
+    return False
+
+
+@register
+class WorkdayAdapter:
+    name: ClassVar[str] = "workday"
+    platform: ClassVar[Platform] = Platform.WORKDAY
+    rate_profile: ClassVar[str] = "workday"
+    hosts: ClassVar[frozenset[str]] = frozenset()
+    bucket_key: ClassVar[str] = "workday"
+
+    detail_budget: ClassVar[int] = 60
+
+    rotation_slice: ClassVar[int] = 300
+
+    max_requests_per_company: ClassVar[int] = MAX_PAGES
+
+    crawl_page_cap: ClassVar[int] = CRAWL_PAGE_CAP
+    retry_reserve: ClassVar[int] = RETRY_RESERVE
+
+    @classmethod
+    def crawl_budgets(
+        cls,
+        companies: Sequence[Company],
+        crawls: Mapping[str, WorkdayCrawl],
+        facets: Mapping[tuple[str, str], WorkdayFacet],
+        ceiling: int,
+    ) -> tuple[dict[str, int], int]:
+        if not companies:
+            raise ValueError("a Workday crawl needs at least one board")
+        available = max(1, ceiling - cls.retry_reserve)
+        budgets = {company.registry_key: 1 for company in companies}
+        remaining = max(0, available - len(budgets))
+        demands: list[tuple[int, str]] = []
+        for company in companies:
+            crawl = crawls.get(board_key(cls.name, _board(company)))
+            facet = _pinned_facet(company) or facets.get(
+                (company.workday_tenant or "", company.workday_site or "")
+            )
+            if crawl is None or crawl.total is None:
+                continue
+            if crawl.facet_parameter != (facet.parameter if facet is not None else ""):
+                continue
+            if crawl.facet_ids != (facet.facet_ids if facet is not None else ()):
+                continue
+            start = max(0, crawl.next_offset - PAGE_SIZE)
+            pages = max(1, (max(0, crawl.total - start) + PAGE_SIZE - 1) // PAGE_SIZE)
+            demands.append((min(MAX_CRAWL_PAGES, pages), company.registry_key))
+        for pages, key in sorted(demands, key=lambda item: (-item[0], item[1])):
+            extra = max(0, min(remaining, pages - budgets[key]))
+            budgets[key] += extra
+            remaining -= extra
+        for company in companies:
+            key = company.registry_key
+            extra = max(0, min(remaining, cls.crawl_page_cap - budgets[key]))
+            budgets[key] += extra
+            remaining -= extra
+        details = min(cls.detail_budget, max(0, remaining))
+        return budgets, details
+
+    def hosts_for(self, companies: Sequence[Company]) -> frozenset[str]:
+        allowed: set[str] = set()
+        for company in companies:
+            try:
+                host, _ = workday_target(
+                    company.workday_tenant or "",
+                    company.workday_site or "",
+                    company.workday_dc or "",
+                )
+            except SlugRejectedError:
+                continue
+            allowed.add(host)
+        return frozenset(allowed)
+
+    def board_key(self, company: Company) -> str:
+        return board_key(self.name, _board(company))
+
+    def plan(self, company: Company) -> tuple[str, ...]:
+        host, path = workday_target(
+            company.workday_tenant or "",
+            company.workday_site or "",
+            company.workday_dc or "",
+        )
+        return (f"https://{host}{path}",)
+
+    async def fetch(
+        self,
+        company: Company,
+        client: HttpClient,
+        now: datetime,
+        facets: Mapping[tuple[str, str], WorkdayFacet] | None = None,
+        details: Sequence[str] = (),
+        crawl: WorkdayCrawl | None = None,
+        page_budget: int = MAX_PAGES,
+    ) -> FetchResult:
+        if page_budget < 1:
+            raise ValueError("Workday page budget must be positive")
+        if page_budget > MAX_CRAWL_PAGES:
+            raise ValueError("Workday page budget exceeds its crawl cap")
+        url = self.plan(company)[0]
+        tenant = company.workday_tenant or ""
+        site = company.workday_site or ""
+        facet = _pinned_facet(company) or (facets or {}).get((tenant, site))
+        applied = {facet.parameter: list(facet.facet_ids)} if facet is not None else {}
+        crawl_reset = crawl is not None and (
+            crawl.facet_parameter != (facet.parameter if facet is not None else "")
+            or crawl.facet_ids != (facet.facet_ids if facet is not None else ())
+        )
+        persisted = None if crawl_reset else crawl
+        degraded = ""
+
+        discovered: WorkdayFacet | None = None
+        forgotten: WorkdayFacet | None = None
+        restarted = False
+        malformed = 0
+        fell_back = False
+        stale_facet = False
+        drifted = False
+        postings: list[WorkdayPosting] = []
+        reached_end = False
+        offset = 0
+        if persisted is not None:
+            offset = persisted.next_offset
+            if page_budget > 1:
+                offset = max(0, offset - PAGE_SIZE)
+        pages = 0
+        total = persisted.total if persisted is not None else None
+        total_changed = False
+
+        while pages < page_budget:
+            body: dict[str, Any] = {
+                "appliedFacets": applied,
+                "limit": PAGE_SIZE,
+                "offset": offset,
+                "searchText": "",
+            }
+            response = await client.post_json(url, body=body)
+            page, dropped = _validate(response.payload, company)
+            malformed += dropped
+            pages += 1
+
+            if facet is not None and page.facets is None:
+                if not drifted:
+                    drifted = True
+                    captured = capture_payload("workday-nofacets", company.slug, response.payload)
+                    degraded = (
+                        f"no facet list at all while facet {facet.facet_ids!r} applied, "
+                        f"so staleness is undecided; facet kept, nothing closed. "
+                        f"Payload captured at {captured}"
+                    )
+            elif facet is not None and not facet_still_offered(page, facet):
+                if facet.pinned:
+                    stale_facet = True
+                    degraded = (
+                        f"pinned facet {facet.facet_ids!r} is no longer offered under "
+                        f"{facet.parameter!r}; honoured anyway. Re-pin with "
+                        "`stage discover --url` or clear `workday_facet`"
+                    )
+                else:
+                    probe_body: dict[str, Any] = {
+                        "appliedFacets": {},
+                        "limit": PAGE_SIZE,
+                        "offset": 0,
+                        "searchText": "",
+                    }
+                    probe_response = await client.post_json(url, body=probe_body)
+                    probe, probe_dropped = _validate(probe_response.payload, company)
+                    malformed += probe_dropped
+                    pages += 1
+                    if not facet_still_offered(probe, facet):
+                        stale_facet = True
+                        degraded = (
+                            f"cached facet {facet.facet_ids!r} is no longer offered under "
+                            f"{facet.parameter!r}; re-resolving from this tenant's own facet list"
+                        )
+                        forgotten = facet
+                        facet = None
+                        applied = {}
+                        postings.clear()
+                        offset = 0
+                        total = None
+                        crawl_reset = True
+                        page = probe
+                        dropped = probe_dropped
+                        response = probe_response
+
+            if facet is None and not applied:
+                resolved = resolve_facet(page, tenant, site, now)
+                if resolved is not None:
+                    discovered = resolved
+                    forgotten = None
+                else:
+                    fell_back = True
+                    degraded = _fallback_reason(page, company, response.payload)
+
+            if discovered is not None and not applied and not restarted:
+                applied = {discovered.parameter: list(discovered.facet_ids)}
+                restarted = True
+                postings.clear()
+                offset = 0
+                total = None
+                crawl_reset = True
+                continue
+
+            postings.extend(page.jobPostings)
+            if page.total > 0:
+                if total is None:
+                    total = page.total
+                elif total != page.total:
+                    total_changed = True
+                    total = page.total
+
+            returned = len(page.jobPostings) + dropped
+            if returned < PAGE_SIZE:
+                reached_end = True
+                break
+            offset += PAGE_SIZE
+            if offset >= (RESULT_CAP if total is None else min(total, RESULT_CAP)):
+                reached_end = True
+                break
+
+        capped = not reached_end
+        if capped:
+            if page_budget != MAX_PAGES and page_budget < MAX_CRAWL_PAGES:
+                degraded = (
+                    f"resumable crawl paused after {pages} page(s); resumes from offset {offset} "
+                    "on the next sync and closes nothing yet"
+                )
+            else:
+                degraded = f"stopped at the {page_budget}-page cap; the board may be truncated"
+        if total_changed:
+            degraded = (
+                "reported total changed before the board ended; progress was retained and "
+                f"resumes from offset {offset} on the next sync"
+                if not reached_end
+                else "reported total changed during the completed crawl; jobs were refreshed, "
+                "but nothing closed and the next sync starts a fresh pass"
+            )
+        if malformed:
+            degraded = malformed_note(malformed) + (f" ({degraded})" if degraded else "")
+
+        faceted = "internship" if applied else ""
+        paired = [(posting, _to_job(company, posting, now, faceted)) for posting in postings]
+        wanted = set(details)
+        fetched: list[DetailFetch] = []
+        if wanted:
+            paired, fetched = await _attach_descriptions(company, client, paired, wanted)
+
+        crawl_step = None
+        crawling = page_budget < MAX_PAGES or crawl is not None
+        safe = not (malformed or fell_back or stale_facet or drifted or total_changed)
+        if crawling:
+            crawl_step = WorkdayCrawlStep(
+                board=self.board_key(company),
+                next_offset=0 if reached_end else offset,
+                total=total,
+                facet_parameter=next(iter(applied), ""),
+                facet_ids=tuple(next(iter(applied.values()), ())),
+                seen_ids=tuple(dict.fromkeys(job.id for _, job in paired)),
+                complete=reached_end and safe,
+                reset=crawl_reset,
+                discard=bool(malformed) or (reached_end and not safe),
+            )
+
+        return FetchResult(
+            jobs=tuple(job for _, job in paired),
+            degraded=degraded,
+            authoritative=reached_end and safe,
+            facets=(discovered,) if discovered is not None else (),
+            forgotten_facets=(forgotten,) if forgotten is not None else (),
+            detail_fetches=tuple(fetched),
+            workday_crawl=crawl_step,
+        )
+
+
+def _fallback_reason(page: WorkdayPage, company: Company, payload: Any) -> str:
+    if page.facets:
+        names = {group.facetParameter for group in page.facets if group.facetParameter}
+        advertised = ", ".join(sorted(names))
+        return (
+            f"no internship facet among the values this tenant advertises ({advertised}); "
+            "walking the whole board instead, which the bilingual classifier then filters"
+        )
+    if page.facets == []:
+        return (
+            "this tenant advertises an empty facet list, so there is no internship facet "
+            "to resolve; walking the whole board instead"
+        )
+    captured = capture_payload("workday-nofacets", company.slug, payload)
+    return f"no facet list at all, so resolution could not run; payload captured at {captured}"
+
+
+def _pinned_facet(company: Company) -> WorkdayFacet | None:
+    if not company.workday_facet:
+        return None
+    parameter, _, value = company.workday_facet.partition(":")
+    if not value:
+        return WorkdayFacet(
+            tenant=company.workday_tenant or "",
+            site=company.workday_site or "",
+            parameter=workday_facet_lexicon()[0][0],
+            facet_ids=(parameter,),
+            pinned=True,
+        )
+    return WorkdayFacet(
+        tenant=company.workday_tenant or "",
+        site=company.workday_site or "",
+        parameter=parameter,
+        facet_ids=tuple(value.split(",")),
+        pinned=True,
+    )
+
+
+def _validate(payload: Any, company: Company) -> tuple[WorkdayPage, int]:
+    try:
+        raw = WorkdayRawPage.model_validate(payload)
+    except ValidationError as exc:
+        captured = capture_payload("workday", company.slug, payload)
+        raise PayloadValidationError(
+            f"workday payload for {company.name} failed validation: {exc} (captured {captured})"
+        ) from exc
+
+    kept, dropped = validate_rows(
+        WorkdayPosting, raw.jobPostings, source="workday", slug=company.slug
+    )
+    return WorkdayPage(total=raw.total, jobPostings=kept, facets=raw.facets), dropped
+
+
+async def _attach_descriptions(
+    company: Company,
+    client: HttpClient,
+    paired: list[tuple[WorkdayPosting, Job]],
+    wanted: set[str],
+) -> tuple[list[tuple[WorkdayPosting, Job]], list[DetailFetch]]:
+    host, _ = workday_target(
+        company.workday_tenant or "", company.workday_site or "", company.workday_dc or ""
+    )
+    outcomes: list[DetailFetch] = []
+    merged: list[tuple[WorkdayPosting, Job]] = []
+    for posting, job in paired:
+        if job.id not in wanted or not posting.externalPath:
+            merged.append((posting, job))
+            continue
+        path = posting.externalPath
+        url = f"https://{host}/wday/cxs/{company.workday_tenant}/{company.workday_site}{path}"
+        try:
+            response = await client.get_json(url)
+        except HttpError:
+            outcomes.append(DetailFetch(id=job.id, resolved=False, failed=True))
+            merged.append((posting, job))
+            continue
+        body = _description_from(response.payload)
+        outcomes.append(DetailFetch(id=job.id, resolved=bool(body)))
+        merged.append((posting, replace(job, description=body) if body else job))
+    return merged, outcomes
+
+
+def _description_from(payload: Any) -> str:
+    if not isinstance(payload, dict):
+        return ""
+    info = payload.get("jobPostingInfo")
+    if not isinstance(info, dict):
+        return ""
+    body = info.get("jobDescription")
+    return collapse_whitespace(strip_html(body)) if isinstance(body, str) else ""
+
+
+def _board(company: Company) -> str:
+    return f"{company.workday_tenant or company.slug}-{company.workday_site or ''}"
+
+
+def _to_job(
+    company: Company, posting: WorkdayPosting, now: datetime, employment_type: str = ""
+) -> Job:
+    host, _ = workday_target(
+        company.workday_tenant or "", company.workday_site or "", company.workday_dc or ""
+    )
+    raw_path = posting.externalPath
+    path = raw_path if raw_path.startswith("/") else f"/{raw_path}"
+    apply_url = f"https://{host}/{company.workday_site}{path}" if raw_path else ""
+    title = collapse_whitespace(posting.title)
+    return Job(
+        id=job_id("workday", _board(company), posting.requisition()),
+        source="workday",
+        company=company.name,
+        title_raw=title,
+        title_normalized=title.lower(),
+        apply_url_raw=apply_url,
+        description="",
+        location_raw=collapse_whitespace(posting.locationsText),
+        first_seen=now,
+        last_seen=now,
+        signals=SourceSignals(employment_type=employment_type),
+    )

stage/sources/zshah.py

@@ -0,0 +1,99 @@
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from stage.domain import Job, SourceSignals, job_id
+from stage.http import HttpClient
+from stage.sources._text import collapse_whitespace
+from stage.sources.base import (
+    FetchResult,
+    NonEmptyStr,
+    PayloadValidationError,
+    capture_payload,
+    validate_rows,
+)
+from stage.sources.feed import register_feed, upcoming_season_year
+
+URL = "https://zshah101.github.io/Automated-List-Of-Summer-{year}-and-Fall-2026-Tech-Internships/api/jobs.json"
+HOST = "zshah101.github.io"
+
+
+class ZshahListing(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: NonEmptyStr
+    company: NonEmptyStr
+    title: NonEmptyStr
+    location: str = ""
+    url: NonEmptyStr
+    program: str = ""
+
+
+class ZshahEnvelope(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    jobs: list[Any] = Field(default_factory=list)
+
+
+@register_feed
+class ZshahFeed:
+    name: ClassVar[str] = "zshah101"
+    rate_profile: ClassVar[str] = "feeds"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    bucket_key: ClassVar[str] = ""
+
+    def season_year(self, now: datetime) -> int:
+        return upcoming_season_year(now)
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        return (URL.format(year=self.season_year(now)),)
+
+    async def fetch(self, client: HttpClient, now: datetime) -> FetchResult:
+        url = self.plan(now)[0]
+        response = await client.get_json(url)
+        if response.not_modified:
+            return FetchResult(not_modified=True)
+        try:
+            envelope = ZshahEnvelope.model_validate(response.payload)
+        except Exception as exc:
+            captured = capture_payload(self.name, str(self.season_year(now)), response.payload)
+            raise PayloadValidationError(
+                f"{self.name}/{url}: payload envelope failed validation (captured {captured})"
+            ) from exc
+        listings, malformed = validate_rows(
+            ZshahListing, envelope.jobs, source=self.name, slug=str(self.season_year(now))
+        )
+        internships = [
+            listing for listing in listings if listing.program.casefold() == "internship"
+        ]
+        if not internships:
+            captured = capture_payload(self.name, str(self.season_year(now)), response.payload)
+            raise PayloadValidationError(
+                f"{self.name}/{url}: no internship records were found (captured {captured})"
+            )
+        jobs = tuple(
+            Job(
+                id=job_id(self.name, listing.company, listing.id),
+                source=self.name,
+                company=collapse_whitespace(listing.company),
+                title_raw=collapse_whitespace(listing.title),
+                title_normalized=collapse_whitespace(listing.title),
+                apply_url_raw=listing.url,
+                description="",
+                location_raw=collapse_whitespace(listing.location),
+                first_seen=now,
+                last_seen=now,
+                signals=SourceSignals(employment_type="internship"),
+            )
+            for listing in internships
+        )
+        return FetchResult(
+            jobs=jobs,
+            degraded=(
+                f"{malformed} malformed posting(s) were dropped; the feed closes nothing"
+                if malformed
+                else ""
+            ),
+            authoritative=not malformed,
+        )

stage/storage/__init__.py

@@ -0,0 +1,29 @@
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from stage.storage.repository import Repository, SourceBatch, SourceBatchResult
+from stage.storage.sqlite_repo import SqliteRepository
+from stage.storage.writer import AsyncRepository, DatabaseWriter, WriterNotStartedError
+
+
+@asynccontextmanager
+async def open_repository(db_path: Path) -> AsyncIterator[AsyncRepository]:
+    writer = DatabaseWriter(db_path)
+    await writer.start()
+    try:
+        yield AsyncRepository(writer)
+    finally:
+        await writer.aclose()
+
+
+__all__ = [
+    "AsyncRepository",
+    "DatabaseWriter",
+    "Repository",
+    "SourceBatch",
+    "SourceBatchResult",
+    "SqliteRepository",
+    "WriterNotStartedError",
+    "open_repository",
+]