stage-cli 1.0.0__py3-none-any.whl

stage/sources/collage.py

@@ -0,0 +1,60 @@
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from stage.domain import Company, Job, Platform, job_id
+from stage.sources import register
+from stage.sources._text import collapse_whitespace, strip_html
+from stage.sources.base import BoardAdapter, NullableStr
+
+HOST = "api.collage.co"
+
+
+class CollagePosition(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: int
+    title: str
+    location: NullableStr = ""
+    descriptionPlain: NullableStr = ""
+    applyUrl: NullableStr = ""
+    hostedUrl: NullableStr = ""
+
+
+class CollageBoard(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    positions: list[Any] = Field(default_factory=list)
+
+
+@register
+class CollageAdapter(BoardAdapter):
+    name: ClassVar[str] = "collage"
+    platform: ClassVar[Platform] = Platform.COLLAGE
+    rate_profile: ClassVar[str] = "moderate"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    bucket_key: ClassVar[str] = "collage"
+    detail_budget: ClassVar[int] = 0
+    max_requests_per_company: ClassVar[int] = 1
+
+    base_url: ClassVar[str] = "https://api.collage.co/v1/positions/{slug}"
+    root_model: ClassVar[type[BaseModel] | None] = CollageBoard
+    rows_field: ClassVar[str] = "positions"
+    row_model: ClassVar[type[BaseModel]] = CollagePosition
+
+    def to_job(self, company: Company, row: Any, now: datetime) -> Job:
+        apply_url = row.applyUrl or row.hostedUrl
+        title = collapse_whitespace(row.title)
+        return Job(
+            id=job_id(self.name, company.slug, str(row.id)),
+            source=self.name,
+            company=company.name,
+            title_raw=title,
+            title_normalized=title.lower(),
+            apply_url_raw=apply_url,
+            description=collapse_whitespace(strip_html(row.descriptionPlain)),
+            location_raw=collapse_whitespace(row.location),
+            first_seen=now,
+            last_seen=now,
+        )

stage/sources/community_feeds.py

@@ -0,0 +1,142 @@
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from stage.domain import Job, SourceSignals, job_id
+from stage.http import HttpClient
+from stage.sources._text import collapse_whitespace
+from stage.sources.base import (
+    FetchResult,
+    NonEmptyStr,
+    PayloadValidationError,
+    capture_payload,
+    convert_rows,
+    malformed_note,
+    validate_rows,
+)
+from stage.sources.feed import register_feed, upcoming_season_year
+
+HOST = "raw.githubusercontent.com"
+
+
+class CommunityListing(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: str = ""
+    company_name: NonEmptyStr
+    title: NonEmptyStr
+    url: str = ""
+    locations: list[str] = Field(default_factory=list)
+    active: bool = True
+    is_visible: bool = True
+    date_posted: int | None = None
+    season: str = ""
+    sponsorship: str = ""
+
+    @property
+    def identity(self) -> str:
+        return self.id or f"{self.company_name}:{self.title}:{self.url}"
+
+
+class _CommunityFeed:
+    name: ClassVar[str] = ""
+    rate_profile: ClassVar[str] = "feeds"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    bucket_key: ClassVar[str] = ""
+    templates: ClassVar[tuple[str, ...]] = ()
+
+    def season_year(self, now: datetime) -> int:
+        return upcoming_season_year(now)
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        year = self.season_year(now)
+        return tuple(template.format(year=year) for template in self.templates)
+
+    async def fetch(self, client: HttpClient, now: datetime) -> FetchResult:
+        jobs: list[Job] = []
+        modified = False
+        failures: list[str] = []
+        stale: list[str] = []
+        malformed = 0
+        for url in self.plan(now):
+            try:
+                response = await client.get_json(url)
+            except Exception as exc:
+                failures.append(f"{url}: {type(exc).__name__}")
+                stale.append(url)
+                continue
+            if response.not_modified:
+                continue
+            modified = True
+            listings, dropped = self._validate(response.payload, url, now)
+            converted, unconvertible = convert_rows(
+                lambda listing: self._to_job(listing, now),
+                [listing for listing in listings if listing.active and listing.is_visible],
+                source=self.name,
+                slug=str(self.season_year(now)),
+            )
+            malformed += dropped + unconvertible
+            jobs.extend(converted)
+        if failures and not jobs:
+            raise PayloadValidationError(f"{self.name}: every file failed — {'; '.join(failures)}")
+        if not modified and not failures:
+            return FetchResult(not_modified=True)
+        notes = []
+        if failures:
+            notes.append(f"{len(failures)} of {len(self.plan(now))} file(s) unavailable")
+        if malformed:
+            notes.append(malformed_note(malformed))
+        return FetchResult(
+            jobs=tuple(jobs),
+            degraded="; ".join(notes),
+            authoritative=not (failures or malformed),
+            stale_urls=tuple(stale),
+        )
+
+    def _validate(
+        self, payload: Any, url: str, now: datetime
+    ) -> tuple[list[CommunityListing], int]:
+        rows = payload.get("listings") if isinstance(payload, dict) else payload
+        if not isinstance(rows, list):
+            captured = capture_payload(self.name, str(self.season_year(now)), payload)
+            raise PayloadValidationError(
+                f"{self.name}/{url}: payload failed validation (expected a JSON list of "
+                f"listings); raw payload captured at {captured}"
+            )
+        return validate_rows(
+            CommunityListing, rows, source=self.name, slug=str(self.season_year(now))
+        )
+
+    def _to_job(self, listing: CommunityListing, now: datetime) -> Job:
+        posted = (
+            datetime.fromtimestamp(listing.date_posted, tz=now.tzinfo)
+            if listing.date_posted
+            else None
+        )
+        return Job(
+            id=job_id(self.name, listing.company_name, listing.identity),
+            source=self.name,
+            company=collapse_whitespace(listing.company_name),
+            title_raw=listing.title,
+            title_normalized=collapse_whitespace(listing.title),
+            apply_url_raw=listing.url,
+            description="",
+            location_raw=collapse_whitespace(" / ".join(listing.locations)),
+            first_seen=now,
+            last_seen=now,
+            source_posted_at=posted,
+            signals=SourceSignals(
+                season=listing.season,
+                sponsorship=listing.sponsorship,
+            ),
+        )
+
+
+@register_feed
+class VanshFeed(_CommunityFeed):
+    name: ClassVar[str] = "vanshb03"
+    templates: ClassVar[tuple[str, ...]] = (
+        "https://raw.githubusercontent.com/vanshb03/Summer{year}-Internships/dev/"
+        ".github/scripts/listings.json",
+    )

stage/sources/curated_markdown.py

@@ -0,0 +1,289 @@
+from __future__ import annotations
+
+import base64
+import binascii
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict
+
+from stage.domain import Job, SourceSignals, job_id
+from stage.http import HttpClient, HttpStatusError, JsonResponse
+from stage.sources._text import collapse_whitespace, strip_html
+from stage.sources.base import FetchResult, NonEmptyStr, PayloadValidationError, capture_payload
+from stage.sources.feed import register_feed, upcoming_season_year
+
+HOST = "api.github.com"
+CONTENT_URL = "https://api.github.com/repos/{repository}/contents/{path}?ref=main"
+LINK = re.compile(r"\[([^\]]*)\]\(\s*<?(?P<url>https?://[^)\s<>]+)>?\s*\)")
+URL = re.compile(r"\]\(\s*<?(?P<url>https?://[^)\s<>]+)>?\s*\)")
+COMMENT = re.compile(r"<!--.*?-->", re.DOTALL)
+INTERN = re.compile(r"\b(?:intern(?:ship)?s?|co-?op|stagiaire|stage|alternance|student)\b", re.I)
+DECLARED_EMPTY = re.compile(r"_\s*no\s+(?:roles|positions|openings|jobs)[^_\n]{0,40}_", re.I)
+
+
+class GitHubContent(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    encoding: NonEmptyStr
+    content: NonEmptyStr
+
+
+@dataclass(frozen=True, slots=True)
+class Listing:
+    company: str
+    title: str
+    location: str
+    url: str
+    description: str = ""
+
+
+def _plain(value: str) -> str:
+    text = COMMENT.sub("", value)
+    text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text)
+    text = text.replace("**", "").replace("`", "")
+    return collapse_whitespace(strip_html(text)).strip("| ")
+
+
+def _cells(line: str) -> list[str] | None:
+    stripped = line.strip()
+    if not stripped.startswith("|") or not stripped.endswith("|"):
+        return None
+    cells = [cell.strip() for cell in stripped[1:-1].split("|")]
+    if not cells or all(set(cell) <= {"-", ":", " "} for cell in cells):
+        return None
+    return cells
+
+
+def _links(value: str) -> tuple[tuple[str, str], ...]:
+    return tuple((match.group(1), match.group("url")) for match in LINK.finditer(value))
+
+
+def _urls(value: str) -> tuple[str, ...]:
+    return tuple(match.group("url") for match in URL.finditer(value))
+
+
+class _GitHubMarkdownFeed:
+    name: ClassVar[str] = ""
+    rate_profile: ClassVar[str] = "feeds"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    bucket_key: ClassVar[str] = ""
+
+    def season_year(self, now: datetime) -> int:
+        return upcoming_season_year(now)
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        raise NotImplementedError
+
+    async def _available(
+        self, client: HttpClient, urls: tuple[str, ...]
+    ) -> tuple[str, JsonResponse]:
+        missing: HttpStatusError | None = None
+        for url in urls:
+            try:
+                return url, await client.get_json(url)
+            except HttpStatusError as exc:
+                if exc.status != 404:
+                    raise
+                missing = exc
+        if missing is None:
+            raise PayloadValidationError(f"{self.name}: no feed url was planned")
+        raise missing
+
+    async def fetch(self, client: HttpClient, now: datetime) -> FetchResult:
+        url, response = await self._available(client, self.plan(now))
+        if response.not_modified:
+            return FetchResult(not_modified=True)
+        text = self._decode(response.payload, url, now)
+        listings, malformed = self.rows(text)
+        if not listings:
+            if not DECLARED_EMPTY.search(text):
+                captured = capture_payload(self.name, str(self.season_year(now)), {"text": text})
+                raise PayloadValidationError(
+                    f"{self.name}/{url}: no current internship rows were found "
+                    f"(captured {captured})"
+                )
+            return FetchResult(
+                degraded=f"{self.name} publishes no roles right now; the feed closes nothing",
+                authoritative=False,
+            )
+        jobs = tuple(self._to_job(listing, now) for listing in listings)
+        return FetchResult(
+            jobs=jobs,
+            degraded=(
+                f"{malformed} malformed Markdown row(s) were skipped; the feed closes nothing"
+                if malformed
+                else ""
+            ),
+            authoritative=not malformed,
+        )
+
+    def _decode(self, payload: Any, url: str, now: datetime) -> str:
+        try:
+            content = GitHubContent.model_validate(payload)
+        except Exception as exc:
+            captured = capture_payload(self.name, str(self.season_year(now)), payload)
+            raise PayloadValidationError(
+                f"{self.name}/{url}: GitHub content envelope failed validation "
+                f"(captured {captured})"
+            ) from exc
+        if content.encoding != "base64":
+            captured = capture_payload(self.name, str(self.season_year(now)), payload)
+            raise PayloadValidationError(
+                f"{self.name}/{url}: unsupported content encoding {content.encoding!r} "
+                f"(captured {captured})"
+            )
+        try:
+            return base64.b64decode("".join(content.content.split()), validate=True).decode("utf-8")
+        except (UnicodeDecodeError, binascii.Error, ValueError) as exc:
+            captured = capture_payload(self.name, str(self.season_year(now)), payload)
+            raise PayloadValidationError(
+                f"{self.name}/{url}: content is not valid base64 UTF-8 (captured {captured})"
+            ) from exc
+
+    def rows(self, text: str) -> tuple[list[Listing], int]:
+        raise NotImplementedError
+
+    def _to_job(self, listing: Listing, now: datetime) -> Job:
+        return Job(
+            id=job_id(
+                self.name, listing.company, f"{listing.title}:{listing.location}:{listing.url}"
+            ),
+            source=self.name,
+            company=listing.company,
+            title_raw=listing.title,
+            title_normalized=listing.title,
+            apply_url_raw=listing.url,
+            description=listing.description,
+            location_raw=listing.location,
+            first_seen=now,
+            last_seen=now,
+            signals=SourceSignals(employment_type="internship"),
+        )
+
+
+@register_feed
+class NegarFeed(_GitHubMarkdownFeed):
+    name: ClassVar[str] = "negar"
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        repository = "negarprh/Canadian-Tech-Internships-2026"
+        return (
+            CONTENT_URL.format(repository=repository, path=f"README-{self.season_year(now)}.md"),
+            CONTENT_URL.format(repository=repository, path="README.md"),
+        )
+
+    def rows(self, text: str) -> tuple[list[Listing], int]:
+        rows: list[Listing] = []
+        malformed = 0
+        company = ""
+        for line in text.splitlines():
+            cells = _cells(line)
+            if cells is None or len(cells) != 5 or _plain(cells[0]).casefold() == "company":
+                continue
+            candidate = _plain(cells[0])
+            if candidate and candidate != "↳":
+                company = candidate
+            urls = _urls(cells[3])
+            if not company or not urls:
+                if cells[3].strip() and "closed" not in cells[3].casefold():
+                    malformed += 1
+                continue
+            rows.append(
+                Listing(
+                    company=company,
+                    title=_plain(cells[1]),
+                    location=_plain(cells[2]),
+                    url=urls[-1],
+                )
+            )
+        return rows, malformed
+
+
+@register_feed
+class HanziliFeed(_GitHubMarkdownFeed):
+    name: ClassVar[str] = "hanzili"
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        return (
+            CONTENT_URL.format(repository="hanzili/canada_sde_intern_position", path="README.md"),
+        )
+
+    def rows(self, text: str) -> tuple[list[Listing], int]:
+        rows: list[Listing] = []
+        malformed = 0
+        for line in text.splitlines():
+            cells = _cells(line)
+            if cells is None or len(cells) != 7 or _plain(cells[0]).casefold() == "title":
+                continue
+            title, company, description, _, details, location, apply = cells
+            if not INTERN.search(f"{title} {details}"):
+                continue
+            urls = _urls(apply)
+            if not urls:
+                malformed += 1
+                continue
+            rows.append(
+                Listing(
+                    company=_plain(company),
+                    title=_plain(title),
+                    location=_plain(location),
+                    url=urls[-1],
+                    description=_plain(description),
+                )
+            )
+        return rows, malformed
+
+
+@register_feed
+class NorthwesternQuantFeed(_GitHubMarkdownFeed):
+    name: ClassVar[str] = "northwestern-quant"
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        return (
+            CONTENT_URL.format(
+                repository=f"northwesternfintech/{self.season_year(now)}QuantInternships",
+                path="README.md",
+            ),
+        )
+
+    def rows(self, text: str) -> tuple[list[Listing], int]:
+        rows: list[Listing] = []
+        malformed = 0
+        company = ""
+        location = ""
+        titles = {
+            "qd": "Quantitative Developer Intern",
+            "qr": "Quantitative Research Intern",
+            "qt": "Quantitative Trader Intern",
+            "swe": "Software Engineer Intern",
+            "ml": "Machine Learning Intern",
+            "hw": "Hardware Engineer Intern",
+            "fpga": "FPGA Engineer Intern",
+            "devops/sre": "DevOps/SRE Intern",
+        }
+        for line in text.splitlines():
+            if line.startswith("## "):
+                company = _plain(line[3:])
+                location = ""
+                continue
+            if line.startswith("**Locations**:"):
+                location = _plain(line.partition(":")[2])
+                continue
+            cells = _cells(line)
+            if cells is None or len(cells) != 2 or _plain(cells[0]).casefold() == "role":
+                continue
+            role = _plain(cells[0])
+            base = titles.get(role.casefold())
+            if not company or base is None or "fellowship" in role.casefold():
+                continue
+            links = _links(cells[1])
+            if not links and cells[1].strip():
+                malformed += 1
+            for label, url in links:
+                qualifier = _plain(label).replace("✅", "").strip()
+                title = f"{base} — {qualifier}" if qualifier else base
+                rows.append(Listing(company=company, title=title, location=location, url=url))
+        return rows, malformed