stage-cli 1.0.0__py3-none-any.whl

stage/sources/quebec_emploi.py

@@ -0,0 +1,146 @@
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from stage.domain import Job, SourceSignals, job_id
+from stage.http import HttpClient
+from stage.sources import register_feed
+from stage.sources._text import collapse_whitespace
+from stage.sources.base import (
+    FetchResult,
+    NonEmptyStr,
+    PayloadValidationError,
+    capture_payload,
+    malformed_note,
+    validate_rows,
+)
+
+HOST = "www.quebecemploi.gouv.qc.ca"
+URL = f"https://{HOST}/search/postingFilteredAI"
+PAGE_CAP = 4
+
+
+class QuebecEmploiListing(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    ide_affch: int = Field(gt=0)
+    titre: NonEmptyStr
+    employeur: str = ""
+    nom_ville: str = ""
+
+
+class QuebecEmploiMeta(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    total_hits: int = Field(ge=0)
+
+
+class QuebecEmploiPage(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    items: list[Any] = Field(default_factory=list)
+    meta: QuebecEmploiMeta
+
+
+@register_feed
+class QuebecEmploiFeed:
+    name: ClassVar[str] = "quebec-emploi"
+    rate_profile: ClassVar[str] = "paginated"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    bucket_key: ClassVar[str] = "quebec-emploi"
+
+    def season_year(self, now: datetime) -> int:
+        return now.year
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        return (URL,)
+
+    async def fetch(self, client: HttpClient, now: datetime) -> FetchResult:
+        listings: list[QuebecEmploiListing] = []
+        malformed = 0
+        truncated = False
+
+        for page in range(1, PAGE_CAP + 1):
+            response = await client.post_json(URL, body=self._request(page))
+            rows, dropped, total = self._validate(response.payload)
+            listings.extend(rows)
+            malformed += dropped
+            if len(listings) + malformed >= total:
+                break
+        else:
+            truncated = True
+
+        jobs = tuple(self._to_job(listing, now) for listing in listings)
+        notes = []
+        if truncated:
+            notes.append(f"stopped at Québec Emploi's {PAGE_CAP}-page public-search cap")
+        if malformed:
+            notes.append(malformed_note(malformed))
+        return FetchResult(
+            jobs=jobs,
+            authoritative=not (truncated or malformed),
+            degraded="; ".join(notes),
+        )
+
+    def _validate(self, payload: Any) -> tuple[list[QuebecEmploiListing], int, int]:
+        try:
+            page = QuebecEmploiPage.model_validate(payload)
+        except Exception as exc:
+            captured = capture_payload(self.name, "stages-students", payload)
+            raise PayloadValidationError(
+                f"{self.name}: public search response failed validation; raw payload captured at "
+                f"{captured}"
+            ) from exc
+        rows, dropped = validate_rows(
+            QuebecEmploiListing, page.items, source=self.name, slug="stages-students"
+        )
+        return rows, dropped, page.meta.total_hits
+
+    def _to_job(self, listing: QuebecEmploiListing, now: datetime) -> Job:
+        title = collapse_whitespace(listing.titre)
+        company = collapse_whitespace(listing.employeur) or "Québec Emploi"
+        city = collapse_whitespace(listing.nom_ville)
+        location = f"{city}, Québec, Canada" if city else "Québec, Canada"
+        return Job(
+            id=job_id(self.name, "stages-students", str(listing.ide_affch)),
+            source=self.name,
+            company=company,
+            title_raw=title,
+            title_normalized=title.lower(),
+            apply_url_raw=(f"https://{HOST}/plateforme-emploi/poste/{listing.ide_affch}"),
+            description="",
+            location_raw=location,
+            first_seen=now,
+            last_seen=now,
+            signals=SourceSignals(employment_type="student stage"),
+        )
+
+    @staticmethod
+    def _request(page: int) -> dict[str, object]:
+        return {
+            "sort": {"type": "AUTO"},
+            "langue": "fr",
+            "page": page,
+            "identAWS": "stage-public-feed",
+            "filter": {
+                "inputSearch": "",
+                "address": "",
+                "localisation": {"longitude": "", "latitude": "", "distance": 20},
+                "adminRegion": [],
+                "offerType": ["2", "3"],
+                "commitment": [],
+                "jobDuration": [],
+                "levelEducation": [],
+                "studyDiscipline": [],
+                "mrc": [],
+                "bsq": [],
+                "scian": [],
+                "postedSince": "",
+                "excludeAgencies": False,
+                "isUkrainian": False,
+                "isExperimente": False,
+                "isSubsidized": False,
+                "isTrainingProgram": False,
+            },
+        }

stage/sources/recruitee.py

@@ -0,0 +1,96 @@
+from datetime import UTC, datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict
+
+from stage.domain import Company, Job, Platform, SourceSignals, job_id
+from stage.sources import register
+from stage.sources._text import collapse_whitespace, strip_html
+from stage.sources.base import BoardAdapter, NullableBool, NullableStr
+
+HOST_TEMPLATE = "{slug}.recruitee.com"
+PATH = "/api/offers/"
+
+
+class RecruiteeOffer(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: int
+    title: str
+    slug: NullableStr = ""
+    description: NullableStr = ""
+    requirements: NullableStr = ""
+    location: NullableStr = ""
+    city: NullableStr = ""
+    country_code: NullableStr = ""
+    department: NullableStr = ""
+    employment_type_code: NullableStr = ""
+    careers_url: NullableStr = ""
+    careers_apply_url: NullableStr = ""
+    remote: NullableBool = False
+    published_at: NullableStr = ""
+
+    def where(self) -> str:
+        if self.location:
+            return self.location
+        parts = [part for part in (self.city, self.country_code) if part]
+        if not parts and self.remote:
+            return "Remote"
+        return ", ".join(parts)
+
+    def posted(self) -> datetime | None:
+        raw = self.published_at.strip()
+        if not raw:
+            return None
+        for pattern in ("%Y-%m-%d %H:%M:%S %Z", "%Y-%m-%d %H:%M:%S"):
+            try:
+                return datetime.strptime(raw, pattern).replace(tzinfo=UTC)
+            except ValueError:
+                continue
+        try:
+            return datetime.fromisoformat(raw)
+        except ValueError:
+            return None
+
+    def body(self) -> str:
+        joined = "\n\n".join(part for part in (self.description, self.requirements) if part)
+        return collapse_whitespace(strip_html(joined))
+
+
+class RecruiteeBoard(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    offers: list[Any]
+
+
+@register
+class RecruiteeAdapter(BoardAdapter):
+    name: ClassVar[str] = "recruitee"
+    platform: ClassVar[Platform] = Platform.RECRUITEE
+    rate_profile: ClassVar[str] = "moderate"
+    bucket_key: ClassVar[str] = "recruitee"
+    detail_budget: ClassVar[int] = 0
+    max_requests_per_company: ClassVar[int] = 1
+
+    host_template: ClassVar[str] = HOST_TEMPLATE
+    path: ClassVar[str] = PATH
+    root_model: ClassVar[type[BaseModel] | None] = RecruiteeBoard
+    rows_field: ClassVar[str] = "offers"
+    row_model: ClassVar[type[BaseModel]] = RecruiteeOffer
+
+    def to_job(self, company: Company, row: Any, now: datetime) -> Job:
+        title = collapse_whitespace(row.title)
+        return Job(
+            id=job_id(self.name, company.slug, str(row.id)),
+            source=self.name,
+            company=company.name,
+            title_raw=title,
+            title_normalized=title.lower(),
+            apply_url_raw=row.careers_url or row.careers_apply_url,
+            description=row.body(),
+            location_raw=collapse_whitespace(row.where()),
+            first_seen=now,
+            last_seen=now,
+            source_posted_at=row.posted(),
+            signals=SourceSignals(employment_type=row.employment_type_code),
+        )

stage/sources/simplify.py

@@ -0,0 +1,110 @@
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from stage.domain import Job, SourceSignals, job_id
+from stage.http import HttpClient
+from stage.sources._text import collapse_whitespace, strip_html
+from stage.sources.base import (
+    FetchResult,
+    NonEmptyStr,
+    PayloadValidationError,
+    capture_payload,
+    convert_rows,
+    malformed_note,
+    validate_rows,
+)
+from stage.sources.feed import register_feed, upcoming_season_year
+
+HOST = "raw.githubusercontent.com"
+LISTINGS_URL = (
+    "https://raw.githubusercontent.com/SimplifyJobs/Summer{year}-Internships/dev/.github/"
+    "scripts/listings.json"
+)
+
+
+class SimplifyListing(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: NonEmptyStr
+    company_name: NonEmptyStr
+    title: NonEmptyStr
+    url: str = ""
+    locations: list[str] = Field(default_factory=list)
+    active: bool = True
+    is_visible: bool = True
+    date_posted: int | None = None
+    terms: list[str] = Field(default_factory=list)
+    sponsorship: str = ""
+    degrees: list[str] = Field(default_factory=list)
+    category: str = ""
+
+
+@register_feed
+class SimplifyFeed:
+    name: ClassVar[str] = "simplify"
+    rate_profile: ClassVar[str] = "feeds"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    bucket_key: ClassVar[str] = ""
+
+    def season_year(self, now: datetime) -> int:
+        return upcoming_season_year(now)
+
+    def plan(self, now: datetime) -> tuple[str, ...]:
+        return (LISTINGS_URL.format(year=self.season_year(now)),)
+
+    async def fetch(self, client: HttpClient, now: datetime) -> FetchResult:
+        response = await client.get_json(self.plan(now)[0])
+        if response.not_modified:
+            return FetchResult(not_modified=True)
+        listings, dropped = self._validate(response.payload, now)
+        jobs, unconvertible = convert_rows(
+            lambda listing: self._to_job(listing, now),
+            [listing for listing in listings if listing.active and listing.is_visible],
+            source=self.name,
+            slug=str(self.season_year(now)),
+        )
+        dropped += unconvertible
+        return FetchResult(
+            jobs=tuple(jobs),
+            degraded=malformed_note(dropped),
+            authoritative=not dropped,
+        )
+
+    def _validate(self, payload: Any, now: datetime) -> tuple[list[SimplifyListing], int]:
+        if not isinstance(payload, list):
+            captured = capture_payload(self.name, str(self.season_year(now)), payload)
+            raise PayloadValidationError(
+                f"simplify/{self.season_year(now)}: field '<root>' failed validation "
+                f"(expected a JSON list of listings); raw payload captured at {captured}"
+            )
+        return validate_rows(
+            SimplifyListing, payload, source=self.name, slug=str(self.season_year(now))
+        )
+
+    def _to_job(self, listing: SimplifyListing, now: datetime) -> Job:
+        posted = (
+            datetime.fromtimestamp(listing.date_posted, tz=now.tzinfo)
+            if listing.date_posted
+            else None
+        )
+        return Job(
+            id=job_id(self.name, listing.company_name, listing.id),
+            source=self.name,
+            company=collapse_whitespace(listing.company_name),
+            title_raw=listing.title,
+            title_normalized=collapse_whitespace(listing.title),
+            apply_url_raw=listing.url,
+            description=strip_html(""),
+            location_raw=collapse_whitespace(" / ".join(listing.locations)),
+            first_seen=now,
+            last_seen=now,
+            source_posted_at=posted,
+            signals=SourceSignals(
+                terms=tuple(listing.terms),
+                sponsorship=listing.sponsorship,
+                degrees=tuple(listing.degrees),
+                category=listing.category,
+            ),
+        )

stage/sources/smartrecruiters.py

@@ -0,0 +1,216 @@
+from collections.abc import Sequence
+from dataclasses import replace
+from datetime import datetime
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+from stage.domain import Company, DetailFetch, Job, Platform, board_key, job_id
+from stage.http import HttpClient, HttpError
+from stage.sources import register
+from stage.sources._text import collapse_whitespace, strip_html
+from stage.sources.base import (
+    FetchResult,
+    PayloadValidationError,
+    capture_payload,
+    malformed_note,
+    validate_rows,
+)
+from stage.sources.platforms import safe_slug
+
+BASE_URL = "https://api.smartrecruiters.com/v1/companies/{slug}/postings"
+DETAIL_URL = "https://api.smartrecruiters.com/v1/companies/{slug}/postings/{posting}"
+HOST = "api.smartrecruiters.com"
+PAGE_SIZE = 100
+MAX_PAGES = 30
+
+
+class SmartRecruitersLocation(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    city: str = ""
+    region: str = ""
+    country: str = ""
+    remote: bool = False
+    fullLocation: str = ""
+
+    def label(self) -> str:
+        if self.fullLocation:
+            return self.fullLocation
+        parts = [part for part in (self.city, self.region, self.country) if part]
+        return ", ".join(parts) or ("Remote" if self.remote else "")
+
+
+class SmartRecruitersPosting(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    id: str
+    name: str
+    releasedDate: datetime | None = None
+    location: SmartRecruitersLocation | None = None
+
+
+class SmartRecruitersPage(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    totalFound: int
+    content: list[Any]
+
+
+@register
+class SmartRecruitersAdapter:
+    name: ClassVar[str] = "smartrecruiters"
+    platform: ClassVar[Platform] = Platform.SMARTRECRUITERS
+    rate_profile: ClassVar[str] = "paginated"
+    hosts: ClassVar[frozenset[str]] = frozenset({HOST})
+    bucket_key: ClassVar[str] = ""
+    detail_budget: ClassVar[int] = 120
+    rotation_slice: ClassVar[int] = 0
+
+    max_requests_per_company: ClassVar[int] = MAX_PAGES
+
+    def hosts_for(self, companies: Sequence[Company]) -> frozenset[str]:
+        return self.hosts
+
+    def board_key(self, company: Company) -> str:
+        return board_key(self.name, company.slug)
+
+    def plan(self, company: Company) -> tuple[str, ...]:
+        return (f"{BASE_URL.format(slug=safe_slug(company.slug))}?limit={PAGE_SIZE}&offset=0",)
+
+    async def fetch(
+        self,
+        company: Company,
+        client: HttpClient,
+        now: datetime,
+        facets: object = None,
+        details: Sequence[str] = (),
+    ) -> FetchResult:
+        url = BASE_URL.format(slug=safe_slug(company.slug))
+        postings: list[SmartRecruitersPosting] = []
+        truncated = False
+        stale_page = False
+        malformed = 0
+
+        wanted = {job for job in details if job.startswith(f"{self.board_key(company)}:")}
+
+        for page in range(MAX_PAGES):
+            response = await client.get_json(
+                url,
+                params={"limit": str(PAGE_SIZE), "offset": str(page * PAGE_SIZE)},
+                revalidate=bool(wanted),
+            )
+            if response.not_modified:
+                if page == 0:
+                    return FetchResult(not_modified=True)
+                stale_page = True
+                break
+            rows, dropped, total = self._validate(company, response.payload)
+            malformed += dropped
+            if not rows and not dropped:
+                break
+            postings.extend(rows)
+            if len(postings) + malformed >= total:
+                break
+        else:
+            truncated = True
+
+        notes = []
+        if truncated:
+            notes.append(f"stopped at the {MAX_PAGES}-page cap")
+        if stale_page:
+            notes.append(
+                "a later page answered 304, so the walk ended early on an unchanged page "
+                "rather than on the end of the list"
+            )
+        if malformed:
+            notes.append(malformed_note(malformed))
+        paired = [(posting, self._to_job(company, posting, now)) for posting in postings]
+        fetched: list[DetailFetch] = []
+        if wanted:
+            paired, fetched = await self._attach_descriptions(company, client, paired, wanted)
+        jobs = [job for _, job in paired]
+
+        return FetchResult(
+            jobs=tuple(jobs),
+            degraded="; ".join(notes),
+            authoritative=not (truncated or stale_page or malformed),
+            detail_fetches=tuple(fetched),
+        )
+
+    async def _attach_descriptions(
+        self,
+        company: Company,
+        client: HttpClient,
+        paired: list[tuple[SmartRecruitersPosting, Job]],
+        wanted: set[str],
+    ) -> tuple[list[tuple[SmartRecruitersPosting, Job]], list[DetailFetch]]:
+        outcomes: list[DetailFetch] = []
+        merged: list[tuple[SmartRecruitersPosting, Job]] = []
+        for posting, job in paired:
+            if job.id not in wanted:
+                merged.append((posting, job))
+                continue
+            url = DETAIL_URL.format(slug=safe_slug(company.slug), posting=posting.id)
+            try:
+                response = await client.get_json(url)
+            except HttpError:
+                outcomes.append(DetailFetch(id=job.id, resolved=False, failed=True))
+                merged.append((posting, job))
+                continue
+            body = _description_from(response.payload)
+            outcomes.append(DetailFetch(id=job.id, resolved=bool(body)))
+            merged.append((posting, replace(job, description=body) if body else job))
+        return merged, outcomes
+
+    def _validate(
+        self, company: Company, payload: Any
+    ) -> tuple[list[SmartRecruitersPosting], int, int]:
+        page = self._validate_page(company, payload)
+        rows, dropped = validate_rows(
+            SmartRecruitersPosting, page.content, source=self.name, slug=company.slug
+        )
+        return rows, dropped, page.totalFound
+
+    def _validate_page(self, company: Company, payload: Any) -> SmartRecruitersPage:
+        try:
+            return SmartRecruitersPage.model_validate(payload)
+        except ValidationError as exc:
+            captured = capture_payload(self.name, company.slug, payload)
+            first = exc.errors()[0]
+            field = ".".join(str(part) for part in first["loc"]) or "<root>"
+            raise PayloadValidationError(
+                f"smartrecruiters/{company.slug}: field {field!r} failed validation "
+                f"({first['msg']}); raw payload captured at {captured}"
+            ) from exc
+
+    def _to_job(self, company: Company, posting: SmartRecruitersPosting, now: datetime) -> Job:
+        return Job(
+            id=job_id(self.name, company.slug, posting.id),
+            source=self.name,
+            company=company.name,
+            title_raw=posting.name,
+            title_normalized=collapse_whitespace(posting.name),
+            apply_url_raw=f"https://jobs.smartrecruiters.com/{company.slug}/{posting.id}",
+            description="",
+            location_raw=collapse_whitespace(posting.location.label() if posting.location else ""),
+            first_seen=now,
+            last_seen=now,
+            source_posted_at=posting.releasedDate,
+        )
+
+
+def _description_from(payload: Any) -> str:
+    if not isinstance(payload, dict):
+        return ""
+    ad = payload.get("jobAd")
+    sections = ad.get("sections") if isinstance(ad, dict) else None
+    if not isinstance(sections, dict):
+        return ""
+    parts: list[str] = []
+    for section in sections.values():
+        if isinstance(section, dict):
+            text = section.get("text")
+            if isinstance(text, str) and text.strip():
+                parts.append(strip_html(text))
+    return collapse_whitespace(" ".join(parts))