stage-cli 1.0.0__py3-none-any.whl

stage/domain/events.py

@@ -0,0 +1,204 @@
+from dataclasses import dataclass
+from datetime import datetime
+
+from stage.domain.enums import SyncOutcome
+
+
+@dataclass(frozen=True, slots=True)
+class SyncStarted:
+    sources: tuple[str, ...]
+    companies: int
+    started_at: datetime
+
+
+@dataclass(frozen=True, slots=True)
+class SourceStarted:
+    source: str
+    companies: int
+
+
+@dataclass(frozen=True, slots=True)
+class CompanyStarted:
+    source: str
+    company: str
+
+
+@dataclass(frozen=True, slots=True)
+class CompanyFinished:
+    source: str
+    company: str
+    fetched: int
+    elapsed_ms: float
+    degraded: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class CompanyFailed:
+    source: str
+    company: str
+    error: str
+    elapsed_ms: float
+    unreachable: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class CompanyDeferred:
+    source: str
+    company: str
+    reason: str
+
+
+@dataclass(frozen=True, slots=True)
+class CompanyUnchanged:
+    source: str
+    company: str
+    elapsed_ms: float
+
+
+@dataclass(frozen=True, slots=True)
+class UnroutableCompanies:
+    companies: tuple[str, ...]
+    platforms: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class SourceBlocked:
+    source: str
+    bucket: str
+    blocked_until: datetime
+    remaining_s: float
+    reason: str
+    consecutive_failures: int
+
+
+@dataclass(frozen=True, slots=True)
+class SourceFailed:
+    source: str
+    error: str
+
+
+@dataclass(frozen=True, slots=True)
+class SourceRotated:
+    source: str
+    bucket: str
+    selected: int
+    deferred: int
+    cursor: str
+    wrapped: bool
+
+
+@dataclass(frozen=True, slots=True)
+class SourceFresh:
+    source: str
+    skipped: int
+    remaining: int
+    refresh_interval_h: float
+
+
+@dataclass(frozen=True, slots=True)
+class SourceResting:
+    source: str
+    skipped: int
+    remaining: int
+
+
+@dataclass(frozen=True, slots=True)
+class SourceCapped:
+    source: str
+    bucket: str
+    spent: int
+    allowance: int
+    ceiling: int
+
+
+@dataclass(frozen=True, slots=True)
+class PlannedRequest:
+    source: str
+    company: str
+    url: str
+    has_validator: bool
+    expectation: str
+
+
+@dataclass(frozen=True, slots=True)
+class BucketPlan:
+    bucket: str
+    sources: tuple[str, ...]
+    planned: int
+    worst_case: int
+    ceiling: int
+
+    @property
+    def exceeds_ceiling(self) -> bool:
+        return self.worst_case > self.ceiling
+
+
+@dataclass(frozen=True, slots=True)
+class RequestLogged:
+    source: str
+    method: str
+    url: str
+    status: int | None
+    elapsed_ms: float
+    attempt: int
+    error: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class SourceFinished:
+    source: str
+    fetched: int
+    added: int
+    updated: int
+    closed: int
+    failed_companies: int
+    elapsed_ms: float
+    fetch_ms: float = 0.0
+    normalize_ms: float = 0.0
+    write_ms: float = 0.0
+    quarantined: int = 0
+    requests: int = 0
+    not_modified: int = 0
+    retries: int = 0
+    tightenings: int = 0
+    latency_p50_ms: float = 0.0
+    latency_p95_ms: float = 0.0
+
+
+@dataclass(frozen=True, slots=True)
+class SyncFinished:
+    outcome: SyncOutcome
+    added: int
+    updated: int
+    closed: int
+    failed_sources: tuple[str, ...]
+    elapsed_ms: float
+    quarantined: int = 0
+    purged: int = 0
+    requests: int = 0
+    not_modified: int = 0
+    dry_run: bool = False
+    partial_reason: str = ""
+
+
+SyncEvent = (
+    SyncStarted
+    | UnroutableCompanies
+    | SourceBlocked
+    | SourceRotated
+    | SourceFresh
+    | SourceResting
+    | SourceCapped
+    | SourceStarted
+    | CompanyStarted
+    | CompanyFinished
+    | CompanyUnchanged
+    | CompanyFailed
+    | CompanyDeferred
+    | PlannedRequest
+    | BucketPlan
+    | RequestLogged
+    | SourceFailed
+    | SourceFinished
+    | SyncFinished
+)

stage/domain/filters.py

@@ -0,0 +1,27 @@
+from dataclasses import dataclass
+from datetime import datetime
+
+from stage.domain.enums import (
+    DegreeRequirement,
+    JobStatus,
+    Language,
+    LocationBucket,
+    RoleCategory,
+)
+
+DEFAULT_WINDOW_DAYS = 14
+DEFAULT_LIMIT = 50
+
+
+@dataclass(frozen=True, slots=True)
+class JobFilters:
+    location: LocationBucket | None = None
+    term: str | None = None
+    role: RoleCategory | None = None
+    degree: DegreeRequirement | None = None
+    language: Language | None = None
+    source: str | None = None
+    company: str | None = None
+    status: JobStatus | None = JobStatus.OPEN
+    first_seen_after: datetime | None = None
+    limit: int | None = DEFAULT_LIMIT

stage/domain/health.py

@@ -0,0 +1,169 @@
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from enum import StrEnum
+from statistics import median
+
+MIN_VOLUME_HISTORY = 3
+VOLUME_DROP_RATIO = 0.5
+STALE_AFTER_DAYS = 14
+UNRECORDED_VOLUME = -1
+
+
+class VolumeVerdict(StrEnum):
+    HEALTHY = "healthy"
+    DROPPED = "dropped"
+    COLLAPSED = "collapsed"
+    UNPROVEN = "unproven"
+
+
+class VisitState(StrEnum):
+    HEALTHY = "healthy"
+    STALE = "stale"
+    FAILING = "failing"
+
+
+@dataclass(frozen=True, slots=True)
+class VolumePoint:
+    stored: int
+    deferred: int = 0
+    blocked: bool = False
+
+    @property
+    def is_evidence(self) -> bool:
+        return self.stored > UNRECORDED_VOLUME and self.deferred == 0 and not self.blocked
+
+
+@dataclass(frozen=True, slots=True)
+class VolumeSignal:
+    source: str
+    verdict: VolumeVerdict
+    latest: int
+    baseline: float
+    samples: int
+    detail: str = ""
+
+    @property
+    def is_alert(self) -> bool:
+        return self.verdict in (VolumeVerdict.DROPPED, VolumeVerdict.COLLAPSED)
+
+
+def assess_volume(source: str, history: list[VolumePoint]) -> VolumeSignal:
+    evidence = [point for point in history if point.is_evidence]
+    if not evidence:
+        return VolumeSignal(
+            source=source,
+            verdict=VolumeVerdict.UNPROVEN,
+            latest=UNRECORDED_VOLUME,
+            baseline=0.0,
+            samples=0,
+            detail=_unproven_reason(history),
+        )
+
+    latest = evidence[0].stored
+    prior = [point.stored for point in evidence[1:]]
+    if len(prior) < MIN_VOLUME_HISTORY:
+        return VolumeSignal(
+            source=source,
+            verdict=VolumeVerdict.UNPROVEN,
+            latest=latest,
+            baseline=0.0,
+            samples=len(prior),
+            detail=f"{len(prior)} comparable run(s), {MIN_VOLUME_HISTORY} needed",
+        )
+
+    baseline = float(median(prior))
+    if baseline <= 0:
+        return VolumeSignal(
+            source=source,
+            verdict=VolumeVerdict.HEALTHY,
+            latest=latest,
+            baseline=baseline,
+            samples=len(prior),
+            detail="has never stored anything",
+        )
+
+    if latest == 0:
+        return VolumeSignal(
+            source=source,
+            verdict=VolumeVerdict.COLLAPSED,
+            latest=latest,
+            baseline=baseline,
+            samples=len(prior),
+            detail=f"stores nothing after a median of {baseline:.0f} across {len(prior)} runs",
+        )
+
+    if latest < baseline * VOLUME_DROP_RATIO:
+        return VolumeSignal(
+            source=source,
+            verdict=VolumeVerdict.DROPPED,
+            latest=latest,
+            baseline=baseline,
+            samples=len(prior),
+            detail=f"stores {latest} against a median of {baseline:.0f}",
+        )
+
+    return VolumeSignal(
+        source=source,
+        verdict=VolumeVerdict.HEALTHY,
+        latest=latest,
+        baseline=baseline,
+        samples=len(prior),
+    )
+
+
+def _unproven_reason(history: list[VolumePoint]) -> str:
+    if not history:
+        return "no runs on record"
+    if any(point.blocked for point in history):
+        return "every run was blocked, which is a throttle not a drop"
+    if any(point.deferred for point in history):
+        return "every run deferred members, so rotation explains it"
+    return "no run has recorded a stored count yet"
+
+
+def classify_visit(
+    last_success_at: datetime | None,
+    consecutive_failures: int,
+    now: datetime,
+    stale_after_days: int = STALE_AFTER_DAYS,
+) -> VisitState:
+    if last_success_at is None:
+        return VisitState.FAILING
+    if consecutive_failures > 0:
+        return VisitState.FAILING
+    if now - last_success_at > timedelta(days=stale_after_days):
+        return VisitState.STALE
+    return VisitState.HEALTHY
+
+
+@dataclass(frozen=True, slots=True)
+class IntegrityRepair:
+    check: str
+    repaired: int
+    detail: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class IntegrityFinding:
+    check: str
+    count: int
+    detail: str
+
+    @property
+    def is_clean(self) -> bool:
+        return self.count == 0
+
+
+__all__ = [
+    "MIN_VOLUME_HISTORY",
+    "STALE_AFTER_DAYS",
+    "UNRECORDED_VOLUME",
+    "VOLUME_DROP_RATIO",
+    "IntegrityFinding",
+    "VisitState",
+    "VolumePoint",
+    "VolumeSignal",
+    "VolumeVerdict",
+    "assess_volume",
+    "classify_visit",
+]

stage/domain/ids.py

@@ -0,0 +1,48 @@
+import re
+
+_SEPARATOR = ":"
+_UNSAFE = re.compile(r"[^a-z0-9._-]+")
+_CANONICAL = re.compile(r"^[a-z0-9._-]+$")
+
+
+def _slugify(value: str) -> str:
+    return _UNSAFE.sub("-", value.strip().lower()).strip("-")
+
+
+def _component(value: str) -> str:
+    clean = value.strip()
+    if not clean:
+        return ""
+    normalized = _slugify(clean)
+    if _CANONICAL.fullmatch(clean.lower()):
+        return normalized
+    return f"{normalized or 'id'}~{clean.encode().hex()}"
+
+
+def board_key(source: str, board_slug: str) -> str:
+    parts = (_slugify(source), _slugify(board_slug))
+    if not all(parts):
+        raise ValueError(f"cannot build a stable board key from {source!r}/{board_slug!r}")
+    return _SEPARATOR.join(parts)
+
+
+def board_of(identifier: str, fallback: str) -> str:
+    parts = identifier.split(_SEPARATOR)
+    if len(parts) < 3:
+        return fallback
+    return _SEPARATOR.join(parts[:2])
+
+
+def job_id(source: str, board_slug: str, native_id: str) -> str:
+    native = _component(native_id)
+    if not native:
+        raise ValueError(
+            f"cannot build a stable job id from {source!r}/{board_slug!r}/{native_id!r}"
+        )
+    try:
+        prefix = board_key(source, board_slug)
+    except ValueError as exc:
+        raise ValueError(
+            f"cannot build a stable job id from {source!r}/{board_slug!r}/{native_id!r}"
+        ) from exc
+    return _SEPARATOR.join((prefix, native))

stage/domain/job.py

@@ -0,0 +1,47 @@
+from dataclasses import dataclass
+from datetime import datetime
+
+from stage.domain.enums import (
+    UNKNOWN_TERM,
+    DegreeRequirement,
+    JobStatus,
+    Language,
+    LocationBucket,
+    RemoteScope,
+    RoleCategory,
+)
+from stage.domain.ids import board_of
+from stage.domain.signals import SourceSignals
+
+
+@dataclass(frozen=True, slots=True)
+class Job:
+    id: str
+    source: str
+    company: str
+    title_raw: str
+    title_normalized: str
+    apply_url_raw: str
+    description: str
+    first_seen: datetime
+    last_seen: datetime
+    location_raw: str = ""
+    title_canonical: str = ""
+    apply_url_canonical: str = ""
+    status: JobStatus = JobStatus.OPEN
+    language: Language = Language.UNKNOWN
+    location: LocationBucket = LocationBucket.UNKNOWN
+    remote_scope: RemoteScope | None = None
+    term: str = UNKNOWN_TERM
+    role: RoleCategory = RoleCategory.UNKNOWN
+    work_auth_flag: bool = False
+    degree_requirement: DegreeRequirement = DegreeRequirement.UNKNOWN
+    compensation: str | None = None
+    source_posted_at: datetime | None = None
+    duplicate_of: str | None = None
+
+    @property
+    def board_key(self) -> str:
+        return board_of(self.id, self.source)
+
+    signals: SourceSignals = SourceSignals()

stage/domain/matching.py

@@ -0,0 +1,15 @@
+from stage.domain.enums import UNKNOWN_TERM, LocationBucket
+
+_NON_EVIDENCE_BUCKETS = frozenset({LocationBucket.UNKNOWN, LocationBucket.INTERNATIONAL})
+
+
+def location_agrees(left: LocationBucket, right: LocationBucket) -> bool:
+    if left in _NON_EVIDENCE_BUCKETS or right in _NON_EVIDENCE_BUCKETS:
+        return False
+    return left is right
+
+
+def term_agrees(left: str, right: str) -> bool:
+    if left == UNKNOWN_TERM or right == UNKNOWN_TERM:
+        return False
+    return left == right

stage/domain/priority.py

@@ -0,0 +1,34 @@
+SOURCE_PRIORITY: tuple[str, ...] = (
+    "greenhouse",
+    "lever",
+    "smartrecruiters",
+    "ashby",
+    "workday",
+    "workable",
+    "bamboohr",
+    "recruitee",
+    "breezy",
+    "collage",
+    "jobvite",
+    "oracle_cloud",
+    "custom_json",
+    "quebec-emploi",
+    "jobbank",
+    "espresso-jobs",
+    "themuse",
+    "simplify",
+    "vanshb03",
+    "speedyapply",
+    "zshah101",
+    "hanzili",
+    "negar",
+    "northwestern-quant",
+)
+
+
+def source_rank(source: str, job_id: str = "") -> tuple[int, str]:
+    try:
+        index = SOURCE_PRIORITY.index(source)
+    except ValueError:
+        index = len(SOURCE_PRIORITY)
+    return index, job_id

stage/domain/quarantine.py

@@ -0,0 +1,39 @@
+from dataclasses import dataclass
+from datetime import datetime
+from enum import StrEnum
+
+from stage.domain.enums import LocationBucket, RemoteScope
+
+DEFAULT_QUARANTINE_LIMIT = 50
+
+
+class RejectionReason(StrEnum):
+    OUT_OF_SCOPE_LOCATION = "out-of-scope-location"
+    NOT_AN_INTERNSHIP = "not-an-internship"
+    OUT_OF_SCOPE_DEGREE = "out-of-scope-degree"
+    NOT_A_CS_ROLE = "not-a-cs-role"
+    UNKNOWN_CS_ROLE = "unknown-cs-role"
+
+
+@dataclass(frozen=True, slots=True)
+class QuarantinedJob:
+    id: str
+    source: str
+    company: str
+    title_raw: str
+    reason: RejectionReason
+    first_seen: datetime
+    last_seen: datetime
+    apply_url_raw: str = ""
+    location_raw: str = ""
+    location: LocationBucket = LocationBucket.UNKNOWN
+    remote_scope: RemoteScope | None = None
+    matched_phrase: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class QuarantineFilters:
+    reason: RejectionReason | None = None
+    source: str | None = None
+    company: str | None = None
+    limit: int | None = DEFAULT_QUARANTINE_LIMIT

stage/domain/rate_state.py

@@ -0,0 +1,78 @@
+from dataclasses import dataclass, replace
+from datetime import datetime, timedelta
+
+BREAKER_THRESHOLD = 5
+BASE_BLOCK_S = 300.0
+BLOCK_CAP_S = 24 * 3600.0
+DECAY_FACTOR = 0.7
+CLEAR_RATIO = 1.05
+
+
+def block_duration(consecutive_failures: int) -> float:
+    steps = max(0, consecutive_failures - BREAKER_THRESHOLD)
+    return min(BLOCK_CAP_S, BASE_BLOCK_S * float(2 ** min(steps, 16)))
+
+
+def decay(override: float, baseline: float) -> float | None:
+    if override <= baseline * CLEAR_RATIO:
+        return None
+    relaxed = max(baseline, override * DECAY_FACTOR)
+    return None if relaxed <= baseline * CLEAR_RATIO else relaxed
+
+
+@dataclass(frozen=True, slots=True)
+class RateState:
+    bucket: str
+    updated_at: datetime
+    blocked_until: datetime | None = None
+    min_interval_override: float | None = None
+    consecutive_failures: int = 0
+    last_failure_at: datetime | None = None
+    reason: str = ""
+    rotation_cursor: str = ""
+
+    def __post_init__(self) -> None:
+        if self.blocked_until is not None:
+            ceiling = self.updated_at + timedelta(seconds=BLOCK_CAP_S)
+            if self.blocked_until > ceiling:
+                object.__setattr__(self, "blocked_until", ceiling)
+
+    def is_blocked(self, now: datetime) -> bool:
+        return self.blocked_until is not None and now < self.blocked_until
+
+    def blocks_remaining_s(self, now: datetime) -> float:
+        if self.blocked_until is None:
+            return 0.0
+        return max(0.0, (self.blocked_until - now).total_seconds())
+
+    def cleared(self, now: datetime) -> "RateState":
+        return RateState(bucket=self.bucket, updated_at=now, rotation_cursor=self.rotation_cursor)
+
+    def with_cursor(self, cursor: str, now: datetime) -> "RateState":
+        return replace(self, rotation_cursor=cursor, updated_at=now)
+
+
+def blocked(
+    state: RateState, *, now: datetime, consecutive_failures: int, reason: str
+) -> RateState:
+    return replace(
+        state,
+        updated_at=now,
+        blocked_until=now + timedelta(seconds=block_duration(consecutive_failures)),
+        consecutive_failures=consecutive_failures,
+        last_failure_at=now,
+        reason=reason,
+    )
+
+
+__all__ = [
+    "BASE_BLOCK_S",
+    "BLOCK_CAP_S",
+    "BREAKER_THRESHOLD",
+    "CLEAR_RATIO",
+    "DECAY_FACTOR",
+    "RateState",
+    "block_duration",
+    "blocked",
+    "decay",
+]

stage/domain/retention.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass
+from datetime import datetime
+
+OPEN_RETENTION_DAYS = 14
+CLOSED_RETENTION_DAYS = 3
+
+
+@dataclass(frozen=True, slots=True)
+class PurgeResult:
+    purged: int = 0
+    tombstoned: int = 0
+    promoted: int = 0
+
+
+@dataclass(frozen=True, slots=True)
+class Tombstone:
+    id: str
+    source: str
+    first_seen: datetime
+    purged_at: datetime