stage-cli 1.0.0__py3-none-any.whl

stage/cli/schedule_state.py

@@ -0,0 +1,363 @@
+import json
+import os
+import tempfile
+import time
+import uuid
+from collections.abc import Mapping
+from contextlib import suppress
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from stage.domain import SyncOutcome
+from stage.domain.text import dump, summary
+from stage.paths import data_dir
+
+if TYPE_CHECKING:
+    from stage.domain import DiscoveryEvent, SyncEvent
+
+HEARTBEAT_INTERVAL_SECONDS = 1.0
+_HEARTBEAT_MISSES_BEFORE_UNRESPONSIVE = 3
+_STATE_RETRY_DELAYS_SECONDS = (0.0, 0.02, 0.1)
+_ACTIVE_PHASES = frozenset({"waiting", "syncing", "discovering", "checking"})
+_OUTCOMES = frozenset(outcome.value for outcome in SyncOutcome)
+
+
+def state_path(action: str) -> Path:
+    root = data_dir() / "schedule"
+    root.mkdir(parents=True, exist_ok=True)
+    return root / f"{action}.json"
+
+
+def read_state(action: str) -> dict[str, object] | None:
+    return read_state_path(state_path(action))
+
+
+def read_state_path(path: Path) -> dict[str, object] | None:
+    for delay in _STATE_RETRY_DELAYS_SECONDS:
+        if delay:
+            time.sleep(delay)
+        try:
+            with path.open(encoding="utf-8") as stream:
+                payload = json.load(stream)
+        except (OSError, json.JSONDecodeError):
+            continue
+        return payload if isinstance(payload, dict) else None
+    return None
+
+
+def state_for_status(
+    payload: dict[str, object] | None, *, now: datetime | None = None
+) -> dict[str, object] | None:
+    if payload is None or payload.get("phase") not in _ACTIVE_PHASES:
+        return payload
+    interval = _positive_number(payload, "heartbeat_interval_seconds")
+    heartbeat = _time(payload.get("heartbeat_at"))
+    if interval <= 0 or heartbeat is None:
+        return payload
+    age = (now or datetime.now(UTC)).astimezone(UTC) - heartbeat
+    if age < timedelta(seconds=interval * _HEARTBEAT_MISSES_BEFORE_UNRESPONSIVE):
+        return payload
+    status = dict(payload)
+    status["phase"] = "unresponsive"
+    status["heartbeat_age_seconds"] = round(age.total_seconds())
+    return status
+
+
+def is_active(payload: dict[str, object] | None) -> bool:
+    status = state_for_status(payload)
+    return bool(status and status.get("phase") in _ACTIVE_PHASES)
+
+
+class ScheduleStateWriter:
+    def __init__(self, path: Path, payload: dict[str, object]) -> None:
+        self.path = path
+        self._payload = payload
+        self._last_write = 0.0
+
+    @classmethod
+    def start(
+        cls,
+        action: str,
+        log_path: Path,
+        *,
+        destination: Path | None = None,
+        now: datetime | None = None,
+    ) -> "ScheduleStateWriter":
+        moment = now or datetime.now(UTC)
+        stamp = _stamp(moment)
+        return cls(
+            destination or state_path(action),
+            {
+                "schema_version": 2,
+                "action": action,
+                "run_id": uuid.uuid4().hex,
+                "phase": "waiting",
+                "triggered_at": stamp,
+                "updated_at": stamp,
+                "heartbeat_at": stamp,
+                "heartbeat_interval_seconds": HEARTBEAT_INTERVAL_SECONDS,
+                "log_path": str(log_path),
+                "progress": _progress(),
+            },
+        )
+
+    @classmethod
+    def open(cls, path: Path, action: str) -> "ScheduleStateWriter":
+        payload = read_state_path(path)
+        if payload is None:
+            return cls.start(action, Path(), destination=path)
+        return cls(path, payload)
+
+    @property
+    def path_is_blocked(self) -> bool:
+        self._reload()
+        return self._payload.get("phase") == "blocked"
+
+    def waiting(self, delay_seconds: float) -> None:
+        moment = datetime.now(UTC)
+        self._set(
+            force=True,
+            phase="waiting",
+            jitter_seconds=round(delay_seconds),
+            starts_after=_stamp(moment + timedelta(seconds=delay_seconds)),
+        )
+
+    def started(self, phase: str) -> None:
+        values: dict[str, object] = {"phase": phase}
+        if not self._payload.get("started_at"):
+            values["started_at"] = _stamp(datetime.now(UTC))
+        self._set(values, force=True)
+
+    def heartbeat(self) -> None:
+        self._reload()
+        self._set(force=True, heartbeat_at=_stamp(datetime.now(UTC)))
+
+    def checking(self) -> None:
+        self._reload()
+        self._set(force=True, phase="checking")
+
+    def complete(
+        self,
+        *,
+        sync_exit_code: int | None,
+        doctor_exit_code: int | None,
+        exit_code: int,
+    ) -> None:
+        self._reload()
+        values: dict[str, object] = {
+            "finished_at": _stamp(datetime.now(UTC)),
+            "sync_exit_code": sync_exit_code,
+            "doctor_exit_code": doctor_exit_code,
+            "exit_code": exit_code,
+        }
+        if self._payload.get("phase") != "blocked":
+            values["phase"] = "finished"
+            values["outcome"] = _outcome(
+                self._payload.get("sync_outcome"), exit_code, doctor_exit_code
+            )
+        self._set(values, force=True)
+
+    def failed(self, detail: str, *, exit_code: int = 2) -> None:
+        self._reload()
+        self._set(
+            force=True,
+            phase="finished",
+            outcome=SyncOutcome.FAILURE.value,
+            finished_at=_stamp(datetime.now(UTC)),
+            exit_code=exit_code,
+            error=summary(detail, 240),
+        )
+
+    def blocked(self, detail: str) -> None:
+        self._reload()
+        self._set(
+            force=True,
+            phase="blocked",
+            blocked_at=_stamp(datetime.now(UTC)),
+            error=summary(detail, 240),
+        )
+
+    def sync_event(self, event: "SyncEvent") -> None:
+        from stage.domain import (
+            CompanyDeferred,
+            CompanyFailed,
+            CompanyFinished,
+            CompanyStarted,
+            CompanyUnchanged,
+            SourceBlocked,
+            SourceFailed,
+            SourceStarted,
+            SyncFinished,
+            SyncStarted,
+        )
+
+        progress = _progress_from(self._payload)
+        values: dict[str, object] = {"phase": "syncing"}
+        if isinstance(event, SyncStarted):
+            progress["total"] = event.companies
+        elif isinstance(event, SourceStarted):
+            progress["source"] = event.source
+            progress["source_total"] = event.companies
+            progress["source_complete"] = 0
+        elif isinstance(event, CompanyStarted):
+            progress["source"] = event.source
+            progress["company"] = event.company
+        elif isinstance(event, (CompanyFinished, CompanyUnchanged, CompanyFailed, CompanyDeferred)):
+            _increment(progress, "complete")
+            _increment(progress, "source_complete")
+            progress["company"] = event.company
+            if isinstance(event, CompanyFinished):
+                _increment(progress, "fetched", event.fetched)
+                if event.degraded:
+                    _increment(progress, "warnings")
+            elif isinstance(event, (CompanyFailed, CompanyDeferred)):
+                _increment(progress, "warnings")
+        elif isinstance(event, (SourceBlocked, SourceFailed)):
+            progress["source"] = event.source
+            _increment(progress, "warnings")
+        elif isinstance(event, SyncFinished):
+            values["sync_outcome"] = event.outcome.value
+            progress["fetched"] = max(_number(progress, "fetched"), event.added + event.updated)
+            _increment(progress, "warnings", len(event.failed_sources))
+        values["progress"] = progress
+        self._set(values, force=isinstance(event, SyncFinished))
+
+    def discovery_event(self, event: "DiscoveryEvent") -> None:
+        from stage.domain import DiscoveryFinished, DiscoveryStarted, PlatformProbed, ProbeVerdict
+
+        progress = _progress_from(self._payload)
+        values: dict[str, object] = {"phase": "discovering"}
+        if isinstance(event, DiscoveryStarted):
+            progress["total"] = event.probes_planned
+        elif isinstance(event, PlatformProbed):
+            _increment(progress, "complete")
+            progress["company"] = event.result.company
+            progress["source"] = event.result.candidate.label
+            if event.result.verdict is ProbeVerdict.UNVERIFIED:
+                _increment(progress, "warnings")
+        elif isinstance(event, DiscoveryFinished):
+            progress["fetched"] = len(event.matched)
+            _increment(progress, "warnings", event.errors + len(event.unverified))
+        values["progress"] = progress
+        self._set(values, force=isinstance(event, DiscoveryFinished))
+
+    def _reload(self) -> None:
+        payload = read_state_path(self.path)
+        if payload is not None and payload.get("run_id") == self._payload.get("run_id"):
+            self._payload = payload
+
+    def _merge_heartbeat(self) -> None:
+        stored = read_state_path(self.path)
+        if stored is None or stored.get("run_id") != self._payload.get("run_id"):
+            return
+        theirs = _time(stored.get("heartbeat_at"))
+        mine = _time(self._payload.get("heartbeat_at"))
+        if theirs is not None and (mine is None or theirs > mine):
+            self._payload["heartbeat_at"] = stored["heartbeat_at"]
+
+    def _set(
+        self,
+        values: Mapping[str, object] | None = None,
+        *,
+        force: bool = False,
+        **extra: object,
+    ) -> None:
+        if values is not None:
+            self._payload.update(values)
+        self._payload.update(extra)
+        self._payload["updated_at"] = _stamp(datetime.now(UTC))
+        now = time.monotonic()
+        if not force and now - self._last_write < 0.5:
+            return
+        self._merge_heartbeat()
+        self._write()
+        self._last_write = now
+
+    def _write(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        staged: Path | None = None
+        try:
+            with suppress(OSError):
+                with tempfile.NamedTemporaryFile(
+                    mode="w",
+                    encoding="utf-8",
+                    dir=self.path.parent,
+                    prefix=f".{self.path.name}.",
+                    suffix=".tmp",
+                    delete=False,
+                ) as stream:
+                    staged = Path(stream.name)
+                    stream.write(dump(self._payload))
+                    stream.write("\n")
+                    stream.flush()
+                    os.fsync(stream.fileno())
+                for delay in _STATE_RETRY_DELAYS_SECONDS:
+                    if delay:
+                        time.sleep(delay)
+                    with suppress(OSError):
+                        staged.replace(self.path)
+                        return
+        finally:
+            if staged is not None:
+                with suppress(OSError):
+                    staged.unlink(missing_ok=True)
+
+
+def _outcome(sync_outcome: object, exit_code: int, doctor_exit_code: int | None) -> str:
+    if exit_code >= 2 or doctor_exit_code not in (None, 0):
+        return SyncOutcome.FAILURE.value
+    if isinstance(sync_outcome, str) and sync_outcome in _OUTCOMES:
+        return sync_outcome
+    return SyncOutcome.PARTIAL.value if exit_code else SyncOutcome.SUCCESS.value
+
+
+def _progress() -> dict[str, object]:
+    return {
+        "total": 0,
+        "complete": 0,
+        "source": "",
+        "source_total": 0,
+        "source_complete": 0,
+        "company": "",
+        "fetched": 0,
+        "warnings": 0,
+    }
+
+
+def _progress_from(payload: dict[str, object]) -> dict[str, object]:
+    progress = _progress()
+    stored = payload.get("progress")
+    if isinstance(stored, dict):
+        for key in progress:
+            if key in stored:
+                progress[key] = stored[key]
+    return progress
+
+
+def _increment(progress: dict[str, object], key: str, amount: int = 1) -> None:
+    progress[key] = _number(progress, key) + amount
+
+
+def _number(payload: dict[str, object], key: str) -> int:
+    value = payload.get(key)
+    return value if isinstance(value, int) else 0
+
+
+def _positive_number(payload: dict[str, object], key: str) -> float:
+    value = payload.get(key)
+    return float(value) if isinstance(value, int | float) and value > 0 else 0.0
+
+
+def _time(value: object) -> datetime | None:
+    if not isinstance(value, str):
+        return None
+    try:
+        moment = datetime.fromisoformat(value)
+    except ValueError:
+        return None
+    return moment.astimezone(UTC) if moment.tzinfo is not None else None
+
+
+def _stamp(moment: datetime) -> str:
+    return moment.astimezone(UTC).isoformat()

stage/cli/selection.py

@@ -0,0 +1,83 @@
+import json
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+MAX_REMEMBERED = 100_000
+
+
+class StaleSelectionError(Exception):
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class Selection:
+    ids: tuple[str, ...]
+    synced_at: str
+
+
+def selection_path() -> Path:
+    from stage.paths import data_dir
+
+    return data_dir() / "last-listing.json"
+
+
+def remember(ids: tuple[str, ...], synced_at: datetime | None, path: Path | None = None) -> int:
+    target = path or selection_path()
+    stored = list(ids[:MAX_REMEMBERED])
+    payload = {
+        "ids": stored,
+        "synced_at": synced_at.isoformat() if synced_at else "",
+    }
+    try:
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(json.dumps(payload), encoding="utf-8")
+    except OSError:
+        return 0
+    return len(stored)
+
+
+def read(path: Path | None = None) -> Selection | None:
+    target = path or selection_path()
+    try:
+        payload = json.loads(target.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None
+    if not isinstance(payload, dict):
+        return None
+    stored = payload.get("ids")
+    if not isinstance(stored, list):
+        return None
+    return Selection(
+        ids=tuple(str(value) for value in stored),
+        synced_at=str(payload.get("synced_at", "")),
+    )
+
+
+def resolve(row: int, synced_at: datetime | None, path: Path | None = None) -> str:
+    selection = read(path)
+    if selection is None or not selection.ids:
+        raise StaleSelectionError(
+            "No listing to number against. Run stage list or stage search first."
+        )
+    current = synced_at.isoformat() if synced_at else ""
+    if selection.synced_at != current:
+        raise StaleSelectionError(
+            "The database has changed since that listing. Run stage list again."
+        )
+    if row < 1 or row > len(selection.ids):
+        raise StaleSelectionError(
+            f"That listing had {len(selection.ids)} row(s), so {row} is out of range."
+        )
+    return selection.ids[row - 1]
+
+
+__all__ = [
+    "MAX_REMEMBERED",
+    "Selection",
+    "StaleSelectionError",
+    "read",
+    "remember",
+    "resolve",
+    "selection_path",
+]

stage/cli/serialize.py

@@ -0,0 +1,196 @@
+import sys
+from collections.abc import Sequence
+from dataclasses import asdict
+from typing import TYPE_CHECKING, Any, Protocol, cast
+
+if TYPE_CHECKING:
+    from stage.cli.schedule import ScheduleStatus
+    from stage.services.canary import CanaryReport
+    from stage.services.coverage import CoverageReport
+    from stage.services.health import DoctorReport, StatsReport
+    from stage.services.query import PostingDetail
+
+from stage.domain import Job, QuarantinedJob, RateState, VisitState
+from stage.domain.text import dump
+
+
+class _ReconfigurableOutput(Protocol):
+    def reconfigure(self, *, encoding: str) -> object: ...
+
+
+def configure_terminal_output(stdout: object, platform: str) -> None:
+    if platform != "win32" or not hasattr(stdout, "reconfigure"):
+        return
+    cast(_ReconfigurableOutput, stdout).reconfigure(encoding="utf-8")
+
+
+def emit(payload: str) -> None:
+    output = f"{payload}\n"
+    buffer = getattr(sys.stdout, "buffer", None)
+    if buffer is None:
+        sys.stdout.write(output)
+        return
+    buffer.write(output.encode("utf-8"))
+
+
+def fail(message: str) -> None:
+    from stage.domain.text import sanitize
+
+    sys.stderr.write(f"{sanitize(message)}\n")
+
+
+def jobs_to_json(jobs: Sequence[Job]) -> str:
+    return dump([asdict(job) for job in jobs])
+
+
+def _posting_payload(detail: "PostingDetail") -> dict[str, Any]:
+    return {
+        "job": asdict(detail.job),
+        "canonical": asdict(detail.canonical) if detail.canonical else None,
+        "duplicates": [asdict(job) for job in detail.duplicates],
+    }
+
+
+def posting_to_json(detail: "PostingDetail") -> str:
+    return dump(_posting_payload(detail))
+
+
+def postings_to_json(details: Sequence["PostingDetail"]) -> str:
+    return dump([_posting_payload(detail) for detail in details])
+
+
+def quarantine_to_json(entries: Sequence[QuarantinedJob]) -> str:
+    return dump([asdict(entry) for entry in entries])
+
+
+def coverage_to_json(report: "CoverageReport") -> str:
+    return dump(
+        {
+            "enabled": report.enabled,
+            "disabled": report.disabled,
+            "stale_after_days": report.stale_after_days,
+            "rows": [asdict(row) for row in report.rows],
+            "unregistered": [asdict(row) for row in report.unregistered],
+            "classifications": [asdict(entry) for entry in report.classifications],
+        }
+    )
+
+
+def health_to_json(report: "DoctorReport") -> str:
+    return dump(
+        {
+            "schema_version": report.schema_version,
+            "last_sync_at": report.last_sync_at,
+            "never_synced": report.never_synced,
+            "healthy": report.is_healthy,
+            "warnings": report.warnings,
+            "due_for_recheck": list(report.due_for_recheck),
+            "workday_crawls": [asdict(crawl) for crawl in report.workday_crawls],
+            "integrity": [asdict(finding) for finding in report.integrity],
+            "blocks": [asdict(state) for state in report.blocks],
+            "sources": [
+                {
+                    **asdict(source),
+                    "cache_hit_ratio": source.cache_hit_ratio,
+                    "success_rate": source.success_rate,
+                }
+                for source in report.sources
+            ],
+        }
+    )
+
+
+def sources_to_json(
+    report: "DoctorReport",
+    states: Sequence[RateState],
+    *,
+    include_boards: bool,
+    rate_states_cleared: int,
+    cache_validators_cleared: int,
+) -> str:
+    sources = []
+    for source in report.sources:
+        payload = asdict(source)
+        payload.pop("boards")
+        payload["cache_hit_ratio"] = source.cache_hit_ratio
+        payload["success_rate"] = source.success_rate
+        sources.append(payload)
+    boards = (
+        [
+            asdict(board)
+            for source in report.sources
+            for board in source.boards
+            if board.state is not VisitState.HEALTHY
+        ]
+        if include_boards
+        else []
+    )
+    return dump(
+        {
+            "healthy": report.is_healthy,
+            "stale_after_days": report.stale_after_days,
+            "sources": sources,
+            "rate_states": [asdict(state) for state in states],
+            "boards": boards,
+            "workday_crawls": [asdict(crawl) for crawl in report.workday_crawls],
+            "rate_states_cleared": rate_states_cleared,
+            "cache_validators_cleared": cache_validators_cleared,
+        }
+    )
+
+
+def stats_to_json(report: "StatsReport") -> str:
+    return dump(
+        {
+            "schema_version": report.schema_version,
+            "total_jobs": report.total_jobs,
+            "duplicates": report.duplicates,
+            "tombstones": report.tombstones,
+            "cached_urls": report.cached_urls,
+            "quarantined": report.quarantined,
+            "composition": report.composition,
+            "runs": [asdict(run) for run in report.runs],
+        }
+    )
+
+
+def schedule_to_json(report: "ScheduleStatus") -> str:
+    return dump(
+        {
+            "backend": report.backend,
+            "log_dir": str(report.log_dir),
+            "actions": [
+                {
+                    "key": action.key,
+                    "label": action.label,
+                    "cadence": action.cadence,
+                    "time": action.time,
+                    "enabled": enabled,
+                    "installed": installed,
+                    "needs_update": (
+                        report.needs_update[index] if index < len(report.needs_update) else False
+                    ),
+                    "stale_interpreter": (
+                        report.stale_interpreter[index]
+                        if index < len(report.stale_interpreter)
+                        else ""
+                    ),
+                    "run": report.states[index] if index < len(report.states) else None,
+                }
+                for index, (action, enabled, installed) in enumerate(report.actions)
+            ],
+        }
+    )
+
+
+def canary_to_json(report: "CanaryReport") -> str:
+    return dump(
+        {
+            "passed": report.passed,
+            "skipped_platforms": list(report.skipped_platforms),
+            "probes": [
+                {**asdict(probe), "failure": probe.is_failure, "empty": probe.is_empty}
+                for probe in report.probes
+            ],
+        }
+    )