stage-cli 1.0.0__py3-none-any.whl

stage/domain/rotation.py

@@ -0,0 +1,46 @@
+from bisect import bisect_right
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class RotationMember:
+    key: str
+
+
+@dataclass(frozen=True, slots=True)
+class Rotation:
+    selected: tuple[str, ...]
+    deferred: tuple[str, ...]
+    cursor: str
+    wrapped: bool = False
+
+    @property
+    def rotating(self) -> bool:
+        return bool(self.deferred)
+
+
+def rotate(members: Sequence[RotationMember], *, cursor: str = "", budget: int = 0) -> Rotation:
+    if budget <= 0:
+        keys = tuple(member.key for member in members)
+        return Rotation(selected=keys, deferred=(), cursor=cursor)
+
+    ring = sorted(member.key for member in members)
+
+    if budget >= len(ring):
+        return Rotation(selected=tuple(ring), deferred=(), cursor="", wrapped=True)
+
+    start = bisect_right(ring, cursor) if cursor else 0
+    wrapped = start + budget > len(ring)
+    taken = [ring[(start + offset) % len(ring)] for offset in range(budget)]
+    remaining = [key for key in ring if key not in set(taken)]
+
+    return Rotation(
+        selected=tuple(taken),
+        deferred=tuple(remaining),
+        cursor=taken[-1] if taken else cursor,
+        wrapped=wrapped or start >= len(ring),
+    )
+
+
+__all__ = ["Rotation", "RotationMember", "rotate"]

stage/domain/signals.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True, slots=True)
+class SourceSignals:
+    terms: tuple[str, ...] = field(default_factory=tuple)
+    season: str = ""
+
+    sponsorship: str = ""
+    degrees: tuple[str, ...] = field(default_factory=tuple)
+    category: str = ""
+    employment_type: str = ""

stage/domain/sync_run.py

@@ -0,0 +1,35 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+
+from stage.domain.enums import SyncOutcome
+from stage.domain.health import UNRECORDED_VOLUME
+
+
+@dataclass(frozen=True, slots=True)
+class SourceRunStats:
+    source: str
+    fetched: int = 0
+    added: int = 0
+    updated: int = 0
+    closed: int = 0
+    quarantined: int = 0
+    errors: int = 0
+    requests: int = 0
+    not_modified: int = 0
+    retries: int = 0
+    tightenings: int = 0
+    latency_p50_ms: float = 0.0
+    latency_p95_ms: float = 0.0
+    elapsed_ms: float = 0.0
+    deferred: int = 0
+    stored: int = UNRECORDED_VOLUME
+
+    blocked: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class SyncRun:
+    started_at: datetime
+    finished_at: datetime
+    outcome: SyncOutcome
+    sources: tuple[SourceRunStats, ...] = field(default_factory=tuple)

stage/domain/text.py

@@ -0,0 +1,113 @@
+import json
+import re
+import socket
+import unicodedata
+from datetime import UTC, datetime
+from ipaddress import ip_address
+from urllib.parse import urlsplit
+
+_CONTROL = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]")
+_UNSAFE_IN_URL = re.compile(r"[\x00-\x20\x7f-\x9f]")
+WEB_SCHEMES = ("http", "https")
+
+
+def sanitize(value: str) -> str:
+    return _CONTROL.sub("", value)
+
+
+def web_url(raw: str) -> str | None:
+    candidate = raw.strip()
+    if not candidate or _UNSAFE_IN_URL.search(candidate):
+        return None
+    try:
+        parts = urlsplit(candidate)
+    except ValueError:
+        return None
+    if parts.scheme not in WEB_SCHEMES or not parts.netloc:
+        return None
+    return candidate
+
+
+def public_https_url(raw: str) -> str | None:
+    candidate = web_url(raw)
+    if candidate is None:
+        return None
+    parts = urlsplit(candidate)
+    if parts.scheme != "https" or parts.username or parts.password:
+        return None
+    host = (parts.hostname or "").rstrip(".").lower()
+    if not host or host == "localhost" or host.endswith(".localhost"):
+        return None
+    try:
+        address = ip_address(host)
+    except ValueError:
+        try:
+            address = ip_address(socket.inet_aton(host))
+        except OSError:
+            return candidate
+    return candidate if address.is_global else None
+
+
+def first_line(value: str) -> str:
+    lines = [line.strip() for line in sanitize(value).splitlines() if line.strip()]
+    return lines[0] if lines else ""
+
+
+def graphemes(value: str) -> list[str]:
+    clusters: list[str] = []
+    for char in value:
+        if unicodedata.combining(char) and clusters:
+            clusters[-1] += char
+        else:
+            clusters.append(char)
+    return clusters
+
+
+def truncate(value: str, width: int) -> str:
+    clean = sanitize(value)
+    clusters = graphemes(clean)
+    if len(clusters) <= width:
+        return clean
+    return "".join(clusters[: max(width - 1, 0)]) + "…"
+
+
+def summary(value: str, width: int) -> str:
+    return truncate(first_line(value), width)
+
+
+def json_default(value: object) -> object:
+    if isinstance(value, datetime):
+        return value.astimezone(UTC).isoformat()
+    raise TypeError(f"unserializable value of type {type(value).__name__}")
+
+
+def json_safe(value: object) -> object:
+    if isinstance(value, str):
+        return sanitize(value)
+    if isinstance(value, dict):
+        return {str(key): json_safe(item) for key, item in value.items()}
+    if isinstance(value, list | tuple):
+        return [json_safe(item) for item in value]
+    return value
+
+
+def dump(payload: object) -> str:
+    return json.dumps(json_safe(payload), indent=2, ensure_ascii=False, default=json_default)
+
+
+__all__ = [
+    "dump",
+    "first_line",
+    "graphemes",
+    "json_default",
+    "json_safe",
+    "public_https_url",
+    "sanitize",
+    "summary",
+    "truncate",
+    "web_url",
+]
+
+
+def escape_markup(value: str) -> str:
+    return sanitize(value).replace("\\", "\\\\").replace("[", "\\[")

stage/domain/validator.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+from datetime import datetime
+
+
+@dataclass(frozen=True, slots=True)
+class HttpValidator:
+    url: str
+    etag: str | None = None
+    last_modified: str | None = None
+    fetched_at: datetime | None = None
+
+    @property
+    def usable(self) -> bool:
+        return bool(self.etag or self.last_modified)

stage/domain/visits.py

@@ -0,0 +1,60 @@
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+
+REST_AFTER_FAILURES = 3
+REST_BASE_HOURS = 6
+REST_MAX_HOURS = 168
+_MAX_BACKOFF_STEPS = 8
+
+
+@dataclass(frozen=True, slots=True)
+class CompanyVisit:
+    board: str
+    succeeded: bool
+    error: str = ""
+    label: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class SourceVisit:
+    source: str
+    board: str
+    last_attempt_at: datetime
+    last_success_at: datetime | None = None
+    consecutive_failures: int = 0
+    last_error: str = ""
+    label: str = ""
+
+    @property
+    def never_succeeded(self) -> bool:
+        return self.last_success_at is None
+
+
+def rested_until(visit: SourceVisit) -> datetime | None:
+    if visit.consecutive_failures < REST_AFTER_FAILURES:
+        return None
+    steps = min(visit.consecutive_failures - REST_AFTER_FAILURES, _MAX_BACKOFF_STEPS)
+    hours = min(REST_BASE_HOURS * 2**steps, REST_MAX_HOURS)
+    return visit.last_attempt_at + timedelta(hours=hours)
+
+
+def is_resting(visit: SourceVisit, now: datetime) -> bool:
+    until = rested_until(visit)
+    return until is not None and now < until
+
+
+@dataclass(frozen=True, slots=True)
+class DetailFetch:
+    id: str
+    resolved: bool = False
+    failed: bool = False
+
+
+__all__ = [
+    "REST_AFTER_FAILURES",
+    "CompanyVisit",
+    "DetailFetch",
+    "SourceVisit",
+    "is_resting",
+    "rested_until",
+]

stage/domain/workday.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass
+from datetime import datetime
+
+
+@dataclass(frozen=True, slots=True)
+class WorkdayFacet:
+    tenant: str
+    site: str
+    parameter: str
+    facet_ids: tuple[str, ...]
+    descriptor: str = ""
+    resolved_at: datetime | None = None
+    pinned: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class WorkdayCrawl:
+    board: str
+    next_offset: int = 0
+    total: int | None = None
+    facet_parameter: str = ""
+    facet_ids: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class WorkdayCrawlStep:
+    board: str
+    next_offset: int = 0
+    total: int | None = None
+    facet_parameter: str = ""
+    facet_ids: tuple[str, ...] = ()
+    seen_ids: tuple[str, ...] = ()
+    complete: bool = False
+    reset: bool = False
+    discard: bool = False
+
+
+__all__ = ["WorkdayCrawl", "WorkdayCrawlStep", "WorkdayFacet"]

stage/http/__init__.py

@@ -0,0 +1,58 @@
+from stage.http.breaker import BreakerState, CircuitBreaker
+from stage.http.cache import ValidatorCache
+from stage.http.client import (
+    USER_AGENT,
+    BreakerOpenError,
+    BucketBlockedError,
+    ForbiddenError,
+    HostBudget,
+    HostBudgetExceededError,
+    HostMetrics,
+    HostNotAllowedError,
+    HttpClient,
+    HttpError,
+    HttpStatusError,
+    JsonResponse,
+    RedirectNotAllowedError,
+    RequestRecord,
+    ResponseTooLargeError,
+    RetryableStatusError,
+    UnreadablePayloadError,
+)
+from stage.http.profiles import (
+    CEILING_BACKSTOP,
+    PROFILES,
+    RatePosture,
+    UnknownProfileError,
+    profile,
+    resolve,
+)
+
+__all__ = [
+    "PROFILES",
+    "USER_AGENT",
+    "BreakerOpenError",
+    "BreakerState",
+    "BucketBlockedError",
+    "ForbiddenError",
+    "CircuitBreaker",
+    "HostBudget",
+    "HostBudgetExceededError",
+    "HostMetrics",
+    "HostNotAllowedError",
+    "HttpClient",
+    "HttpError",
+    "HttpStatusError",
+    "RedirectNotAllowedError",
+    "JsonResponse",
+    "CEILING_BACKSTOP",
+    "RatePosture",
+    "RequestRecord",
+    "ResponseTooLargeError",
+    "UnreadablePayloadError",
+    "RetryableStatusError",
+    "UnknownProfileError",
+    "ValidatorCache",
+    "profile",
+    "resolve",
+]

stage/http/breaker.py

@@ -0,0 +1,53 @@
+import time
+from dataclasses import dataclass, field
+from enum import StrEnum
+
+
+class BreakerState(StrEnum):
+    CLOSED = "closed"
+    OPEN = "open"
+    HALF_OPEN = "half-open"
+
+
+@dataclass(slots=True)
+class CircuitBreaker:
+    failure_threshold: int = 5
+    cooldown_s: float = 30.0
+    consecutive_failures: int = 0
+    opened_at: float | None = None
+    _probing: bool = field(default=False, repr=False)
+
+    def state(self, now: float | None = None) -> BreakerState:
+        if self.opened_at is None:
+            return BreakerState.CLOSED
+        moment = now if now is not None else time.monotonic()
+        if moment - self.opened_at >= self.cooldown_s:
+            return BreakerState.HALF_OPEN
+        return BreakerState.OPEN
+
+    def allows(self, now: float | None = None) -> bool:
+        current = self.state(now)
+        if current is BreakerState.CLOSED:
+            return True
+        if current is BreakerState.HALF_OPEN and not self._probing:
+            self._probing = True
+            return True
+        return False
+
+    def is_open(self, now: float | None = None) -> bool:
+        return self.state(now) is BreakerState.OPEN
+
+    def release_probe(self) -> None:
+        self._probing = False
+
+    def record_success(self) -> None:
+        self.consecutive_failures = 0
+        self.opened_at = None
+        self._probing = False
+
+    def record_failure(self, now: float | None = None) -> None:
+        self.consecutive_failures += 1
+        self._probing = False
+        tripped = self.consecutive_failures >= self.failure_threshold
+        if tripped or self.opened_at is not None:
+            self.opened_at = now if now is not None else time.monotonic()

stage/http/cache.py

@@ -0,0 +1,44 @@
+from collections.abc import Mapping
+from datetime import datetime
+
+import httpx
+
+from stage.domain import HttpValidator
+
+
+class ValidatorCache:
+    def __init__(self, seed: Mapping[str, HttpValidator] | None = None) -> None:
+        self._entries: dict[str, HttpValidator] = dict(seed or {})
+        self._dirty: dict[str, HttpValidator] = {}
+
+    def get(self, url: str) -> HttpValidator | None:
+        entry = self._entries.get(url)
+        return entry if entry is not None and entry.usable else None
+
+    def conditional_headers(self, url: str) -> dict[str, str]:
+        entry = self.get(url)
+        if entry is None:
+            return {}
+        headers: dict[str, str] = {}
+        if entry.etag:
+            headers["If-None-Match"] = entry.etag
+        if entry.last_modified:
+            headers["If-Modified-Since"] = entry.last_modified
+        return headers
+
+    def record(self, url: str, headers: httpx.Headers, fetched_at: datetime) -> None:
+        etag = headers.get("etag")
+        last_modified = headers.get("last-modified")
+        if not etag and not last_modified:
+            return
+        entry = HttpValidator(
+            url=url, etag=etag, last_modified=last_modified, fetched_at=fetched_at
+        )
+        if self._entries.get(url) == entry:
+            return
+        self._entries[url] = entry
+        self._dirty[url] = entry
+
+    @property
+    def pending(self) -> Mapping[str, HttpValidator]:
+        return dict(self._dirty)