stage-cli 1.0.0__py3-none-any.whl

stage/sources/speedyapply.py

@@ -0,0 +1,200 @@
+import base64
+import binascii
+import re
+from datetime import datetime, timedelta
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict
+
+from stage.domain import Job, SourceSignals, job_id
+from stage.http import HttpClient
+from stage.sources._text import collapse_whitespace, strip_html
+from stage.sources.base import (
+    FetchResult,
+    NonEmptyStr,
+    PayloadValidationError,
+    capture_payload,
+    convert_rows,
+    malformed_note,
+    validate_rows,
+)
+from stage.sources.feed import register_feed, upcoming_season_year
+
+HOST = "api.github.com"
+CONTENT_URL = "https://api.github.com/repos/speedyapply/{repository}/contents/{path}?ref=main"
+HREF = re.compile(r'<a\s+href="(?P<url>[^"]+)"', re.IGNORECASE)
+STRONG = re.compile(r"<strong>(?P<text>.*?)</strong>", re.IGNORECASE)
+AGE = re.compile(r"(?P<days>\d+)d")
+
+
+class GitHubContent(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    encoding: NonEmptyStr
+    content: NonEmptyStr
+
+
+class SpeedyApplyListing(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    company: NonEmptyStr
+    title: NonEmptyStr
+    location: str = ""
+    url: NonEmptyStr
+    age: str = ""
+
+    @property
+    def identity(self) -> str:
+        return f"{self.company}:{self.title}:{self.location}:{self.url}"
+
+
+@register_feed
+class SpeedyApplyFeed:
+    name: ClassVar[str] = "speedyapply"
+    rate_profile: ClassVar[str] = "feeds"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    bucket_key: ClassVar[str] = ""
+
+    def season_year(self, now: datetime) -> int:
+        return upcoming_season_year(now)
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        year = self.season_year(now)
+        sources = (
+            (f"{year}-SWE-College-Jobs", "README.md"),
+            (f"{year}-SWE-College-Jobs", "INTERN_INTL.md"),
+            (f"{year}-AI-College-Jobs", "README.md"),
+            (f"{year}-AI-College-Jobs", "INTERN_INTL.md"),
+        )
+        return tuple(
+            CONTENT_URL.format(repository=repository, path=path) for repository, path in sources
+        )
+
+    async def fetch(self, client: HttpClient, now: datetime) -> FetchResult:
+        jobs: list[Job] = []
+        modified = False
+        failures: list[str] = []
+        stale: list[str] = []
+        malformed = 0
+        for url in self.plan(now):
+            try:
+                response = await client.get_json(url)
+            except Exception as exc:
+                failures.append(f"{url}: {type(exc).__name__}")
+                stale.append(url)
+                continue
+            if response.not_modified:
+                continue
+            modified = True
+            listings, dropped = self._validate(response.payload, url, now)
+            converted, unconvertible = convert_rows(
+                lambda listing: self._to_job(listing, now),
+                listings,
+                source=self.name,
+                slug=str(self.season_year(now)),
+            )
+            malformed += dropped + unconvertible
+            jobs.extend(converted)
+        if failures and not jobs:
+            raise PayloadValidationError(f"{self.name}: every file failed — {'; '.join(failures)}")
+        if not modified and not failures:
+            return FetchResult(not_modified=True)
+        unique = {job.id: job for job in jobs}
+        notes = []
+        if failures:
+            notes.append(f"{len(failures)} of {len(self.plan(now))} file(s) unavailable")
+        if malformed:
+            notes.append(malformed_note(malformed))
+        return FetchResult(
+            jobs=tuple(unique.values()),
+            degraded="; ".join(notes),
+            authoritative=not (failures or malformed),
+            stale_urls=tuple(stale),
+        )
+
+    def _validate(
+        self, payload: Any, url: str, now: datetime
+    ) -> tuple[list[SpeedyApplyListing], int]:
+        try:
+            content = GitHubContent.model_validate(payload)
+        except Exception as exc:
+            captured = capture_payload(self.name, str(self.season_year(now)), payload)
+            raise PayloadValidationError(
+                f"{self.name}/{url}: content envelope failed validation (captured {captured})"
+            ) from exc
+        if content.encoding != "base64":
+            captured = capture_payload(self.name, str(self.season_year(now)), payload)
+            raise PayloadValidationError(
+                f"{self.name}/{url}: unsupported content encoding {content.encoding!r} "
+                f"(captured {captured})"
+            )
+        try:
+            text = base64.b64decode("".join(content.content.split()), validate=True).decode("utf-8")
+        except (UnicodeDecodeError, binascii.Error, ValueError) as exc:
+            captured = capture_payload(self.name, str(self.season_year(now)), payload)
+            raise PayloadValidationError(
+                f"{self.name}/{url}: content is not valid base64 UTF-8 (captured {captured})"
+            ) from exc
+        rows, malformed = self._rows(text, str(self.season_year(now)))
+        if not rows:
+            captured = capture_payload(self.name, str(self.season_year(now)), payload)
+            raise PayloadValidationError(
+                f"{self.name}/{url}: no internship table rows were found (captured {captured})"
+            )
+        listings, dropped = validate_rows(
+            SpeedyApplyListing, rows, source=self.name, slug=str(self.season_year(now))
+        )
+        return listings, malformed + dropped
+
+    def _rows(self, text: str, slug: str) -> tuple[list[dict[str, str]], int]:
+        rows: list[dict[str, str]] = []
+        malformed = 0
+        for line in text.splitlines():
+            stripped = line.strip()
+            if not stripped.startswith("|"):
+                continue
+            cells = [cell.strip() for cell in stripped.strip("|").split("|")]
+            if not cells or not cells[0].lower().startswith("<a href="):
+                continue
+            if len(cells) == 6:
+                apply_cell, age_cell = cells[4], cells[5]
+            elif len(cells) == 5:
+                apply_cell, age_cell = cells[3], cells[4]
+            else:
+                malformed += 1
+                capture_payload(f"{self.name}-posting", slug, {"line": line})
+                continue
+            company = STRONG.search(cells[0])
+            apply = HREF.search(apply_cell)
+            if company is None or apply is None:
+                malformed += 1
+                capture_payload(f"{self.name}-posting", slug, {"line": line})
+                continue
+            rows.append(
+                {
+                    "company": collapse_whitespace(strip_html(company.group("text"))),
+                    "title": collapse_whitespace(strip_html(cells[1])),
+                    "location": collapse_whitespace(strip_html(cells[2])),
+                    "url": apply.group("url"),
+                    "age": collapse_whitespace(strip_html(age_cell)),
+                }
+            )
+        return rows, malformed
+
+    def _to_job(self, listing: SpeedyApplyListing, now: datetime) -> Job:
+        age = AGE.fullmatch(listing.age)
+        posted = now - timedelta(days=int(age.group("days"))) if age is not None else None
+        return Job(
+            id=job_id(self.name, listing.company, listing.identity),
+            source=self.name,
+            company=collapse_whitespace(listing.company),
+            title_raw=collapse_whitespace(listing.title),
+            title_normalized=collapse_whitespace(listing.title),
+            apply_url_raw=listing.url,
+            description="",
+            location_raw=collapse_whitespace(listing.location),
+            first_seen=now,
+            last_seen=now,
+            source_posted_at=posted,
+            signals=SourceSignals(employment_type="internship"),
+        )

stage/sources/themuse.py

@@ -0,0 +1,157 @@
+from datetime import datetime
+from typing import Any, ClassVar
+from urllib.parse import quote
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from stage.domain import Job, SourceSignals, job_id
+from stage.http import HttpClient
+from stage.sources import register_feed
+from stage.sources._text import collapse_whitespace
+from stage.sources.base import (
+    FetchResult,
+    NonEmptyStr,
+    PayloadValidationError,
+    capture_payload,
+    malformed_note,
+    validate_rows,
+)
+
+HOST = "www.themuse.com"
+SEARCH = f"https://{HOST}/api/public/jobs"
+LOCATIONS = (
+    "Toronto, Canada",
+    "Montreal, Canada",
+    "Vancouver, Canada",
+    "Ottawa, Canada",
+    "Calgary, Canada",
+    "Edmonton, Canada",
+    "Waterloo, Canada",
+    "Quebec City, Canada",
+    "Halifax, Canada",
+    "Winnipeg, Canada",
+    "New York City, NY",
+    "San Francisco, CA",
+)
+PAGE_CAP = 8
+
+
+class MuseCompany(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    name: str = ""
+
+
+class MuseLocation(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    name: str = ""
+
+
+class MuseRefs(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    landing_page: str = ""
+
+
+class MuseListing(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: int = Field(gt=0)
+    name: NonEmptyStr
+    company: MuseCompany | None = None
+    locations: list[MuseLocation] = Field(default_factory=list)
+    refs: MuseRefs | None = None
+    contents: str = ""
+    categories: list[Any] = Field(default_factory=list)
+
+
+class MusePage(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    results: list[Any]
+    page_count: int = Field(ge=0)
+
+
+@register_feed
+class TheMuseFeed:
+    name: ClassVar[str] = "themuse"
+    rate_profile: ClassVar[str] = "feeds"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    bucket_key: ClassVar[str] = "themuse"
+
+    def season_year(self, now: datetime) -> int:
+        return now.year
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        return (self._url(1),)
+
+    async def fetch(self, client: HttpClient, now: datetime) -> FetchResult:
+        seen: dict[str, Job] = {}
+        malformed = 0
+        truncated = False
+        pages = PAGE_CAP
+        for page in range(1, PAGE_CAP + 1):
+            if page > pages:
+                break
+            response = await client.get_json(self._url(page))
+            rows, dropped, pages = self._validate(response.payload, page)
+            malformed += dropped
+            for listing in rows:
+                job = self._to_job(listing, now)
+                seen.setdefault(job.id, job)
+        else:
+            truncated = pages > PAGE_CAP
+
+        notes = []
+        if truncated:
+            notes.append(f"stopped at the {PAGE_CAP}-page cap with {pages} pages reported")
+        if malformed:
+            notes.append(malformed_note(malformed))
+        return FetchResult(
+            jobs=tuple(seen.values()),
+            authoritative=False,
+            degraded="; ".join(notes),
+        )
+
+    def _url(self, page: int) -> str:
+        locations = "&".join(f"location={quote(name)}" for name in LOCATIONS)
+        return f"{SEARCH}?page={page}&level={quote('Internship')}&{locations}"
+
+    def _validate(self, payload: Any, page: int) -> tuple[list[MuseListing], int, int]:
+        if payload is None:
+            return [], 0, page - 1
+        try:
+            parsed = MusePage.model_validate(payload)
+        except Exception as exc:
+            captured = capture_payload(self.name, f"page-{page}", payload)
+            raise PayloadValidationError(
+                f"{self.name}: search response failed validation; raw payload captured "
+                f"at {captured}"
+            ) from exc
+        rows, dropped = validate_rows(
+            MuseListing, parsed.results, source=self.name, slug=f"page-{page}"
+        )
+        return rows, dropped, parsed.page_count
+
+    def _to_job(self, listing: MuseListing, now: datetime) -> Job:
+        title = collapse_whitespace(listing.name)
+        company = collapse_whitespace(listing.company.name if listing.company else "") or "The Muse"
+        places = [collapse_whitespace(place.name) for place in listing.locations if place.name]
+        landing = listing.refs.landing_page if listing.refs else ""
+        category = ""
+        if listing.categories and isinstance(listing.categories[0], dict):
+            category = collapse_whitespace(str(listing.categories[0].get("name") or ""))
+        return Job(
+            id=job_id(self.name, "internships", str(listing.id)),
+            source=self.name,
+            company=company,
+            title_raw=title,
+            title_normalized=title.lower(),
+            apply_url_raw=landing or f"https://{HOST}/jobs/{listing.id}",
+            description=listing.contents,
+            location_raw=" / ".join(places) or "Remote",
+            first_seen=now,
+            last_seen=now,
+            signals=SourceSignals(employment_type="internship", category=category),
+        )

stage/sources/workable.py

@@ -0,0 +1,83 @@
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict
+
+from stage.domain import Company, Job, Platform, SourceSignals, job_id
+from stage.sources import register
+from stage.sources._text import collapse_whitespace, strip_html
+from stage.sources.base import BoardAdapter, NullableBool, NullableStr
+
+BASE_URL = "https://apply.workable.com/api/v1/widget/accounts/{slug}"
+HOST = "apply.workable.com"
+
+
+class WorkablePosting(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    shortcode: str
+    title: str
+    code: NullableStr = ""
+    department: NullableStr = ""
+    url: NullableStr = ""
+    application_url: NullableStr = ""
+    shortlink: NullableStr = ""
+    country: NullableStr = ""
+    city: NullableStr = ""
+    state: NullableStr = ""
+    employment_type: NullableStr = ""
+    telecommuting: NullableBool = False
+    published_on: NullableStr = ""
+    description: NullableStr = ""
+    requirements: NullableStr = ""
+    benefits: NullableStr = ""
+
+    def where(self) -> str:
+        parts = [part for part in (self.city, self.state, self.country) if part]
+        if not parts and self.telecommuting:
+            return "Remote"
+        return ", ".join(parts)
+
+    def body(self) -> str:
+        sections = (self.description, self.requirements, self.benefits)
+        joined = "\n\n".join(section for section in sections if section)
+        return collapse_whitespace(strip_html(joined))
+
+
+class WorkableBoard(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    name: str = ""
+    jobs: list[Any]
+
+
+@register
+class WorkableAdapter(BoardAdapter):
+    name: ClassVar[str] = "workable"
+    platform: ClassVar[Platform] = Platform.WORKABLE
+    rate_profile: ClassVar[str] = "conservative"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    detail_budget: ClassVar[int] = 0
+    max_requests_per_company: ClassVar[int] = 1
+
+    base_url: ClassVar[str] = BASE_URL
+    query: ClassVar[tuple[tuple[str, str], ...]] = (("details", "true"),)
+    root_model: ClassVar[type[BaseModel] | None] = WorkableBoard
+    rows_field: ClassVar[str] = "jobs"
+    row_model: ClassVar[type[BaseModel]] = WorkablePosting
+
+    def to_job(self, company: Company, row: Any, now: datetime) -> Job:
+        title = collapse_whitespace(row.title)
+        return Job(
+            id=job_id(self.name, company.slug, row.shortcode),
+            source=self.name,
+            company=company.name,
+            title_raw=title,
+            title_normalized=title.lower(),
+            apply_url_raw=row.url or row.application_url or row.shortlink,
+            description=row.body(),
+            location_raw=collapse_whitespace(row.where()),
+            first_seen=now,
+            last_seen=now,
+            signals=SourceSignals(employment_type=row.employment_type),
+        )