sourcelock 0.1.0__py3-none-any.whl

hc_source/doctor.py

@@ -0,0 +1,472 @@
+"""``hc-source doctor``: run every canary and compare it to the lockfile.
+
+Exit codes, which are the contract with CI:
+
+* ``0`` -- everything matches the lockfile.
+* ``1`` -- drift or a schema change: upstream moved and the lockfile is stale.
+* ``2`` -- a source was unreachable, or an adapter module failed to load. This
+  outranks drift, because an unreachable source means the run does not know
+  whether drift happened.
+* ``3`` -- a canary was observed but nothing pinned it, so nothing was compared.
+  This outranks drift for the same reason ``2`` does. It used to be ``0``, which
+  meant a CI job that lost its ``source-lock.json`` reported "22 ok, 0 drift"
+  forever while real upstream regressions sailed past (round-1 finding H1).
+  ``cli._scoped_discovery`` already refuses an unknown ``--source`` with exactly
+  this reasoning: silently checking nothing would report a green run that
+  verified nothing.
+* ``4`` -- every canary matched its pin, but at least one said the thing it read
+  is out of date. The comparison succeeded and told you nothing, which is the
+  same failure as ``3`` wearing a friendlier face. It ranks below drift because
+  drift is a concrete finding and staleness is an absence of one.
+
+Precedence, highest first: adapter load failure or unreachable source (2),
+unpinned (3), drift or schema change (1), stale (4), ok (0). Everything above
+``1`` means "this run does not know", which is why they outrank a finding.
+
+**Severity is a separate axis from status.** A canary declared ``advisory``
+reports its real status, prints its remediation, and is annotated in CI -- and
+is left out of the exit code. That exists for the checks that cannot carry a
+build: an HTML scrape of a page CMS restyles at will, a cross-check whose
+disagreement means "somebody look" rather than "your data is wrong". A gate that
+cries wolf gets uninstalled, and then it protects nothing at all.
+
+**Transient failures are retried, deliberately narrowly.** A canary whose
+observation raises :class:`~hc_source.http.SourceUnreachable` for a transient
+reason -- a dropped connection, a 502, a 429 -- is observed again after an
+exponential backoff, up to the adapter's or the canary's declared retry count.
+Nothing else is retried: an error inside our own code returns the same error
+next time, and a 404 is an answer. The attempt count is reported, because "ok on
+the third try" is a different fact about a source than "ok".
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+from .adapters import AdapterLoadError, discover
+from .http import SourceUnreachable, is_transient, request_timeout
+from .interfaces import Canary, CanaryObservation, SourceAdapter
+from .schemas import CanaryResult, CanarySeverity, CanaryStatus, Lockfile, utcnow
+
+__all__ = [
+    "DoctorReport",
+    "EXIT_DRIFT",
+    "EXIT_OK",
+    "EXIT_STALE",
+    "EXIT_UNPINNED",
+    "EXIT_UNREACHABLE",
+    "MAX_RETRY_BACKOFF",
+    "RETRY_BACKOFF",
+    "run_doctor",
+]
+
+EXIT_OK = 0
+EXIT_DRIFT = 1
+EXIT_UNREACHABLE = 2
+EXIT_UNPINNED = 3
+EXIT_STALE = 4
+
+#: Base of the exponential backoff between observation attempts, in seconds.
+RETRY_BACKOFF = 0.5
+
+#: Ceiling on one backoff interval. A canary is supposed to be cheap; a build
+#: waiting minutes on a source that is plainly down is not cheap.
+MAX_RETRY_BACKOFF = 8.0
+
+#: Raise the retry floor for every canary without editing an adapter -- for the
+#: customer whose runners sit behind a proxy that drops one connection in fifty.
+RETRIES_ENV = "HC_SOURCE_CANARY_RETRIES"
+
+#: Per-request timeout for canary fetches, overriding what adapters declare.
+TIMEOUT_ENV = "HC_SOURCE_CANARY_TIMEOUT"
+
+#: Indirection so tests can assert the backoff schedule without waiting for it.
+_sleep = time.sleep
+
+_UNPINNED_WARNING = (
+    "not pinned in source-lock.json; run `hc-source lock init` to pin it so drift can be detected"
+)
+_UNPINNED_REMEDIATION = (
+    "This canary was observed but never compared: source-lock.json does not pin it, so this "
+    "run cannot tell you whether upstream moved. Pin it with `hc-source lock init` (and commit "
+    "the lockfile) before treating this run as a verification."
+)
+_FALLBACK_WARNING = (
+    "answered by the fallback mirror {name!r} because the pinned authority could not be read; "
+    "this compares the MIRROR against the pin, not the authority"
+)
+_STALE_REMEDIATION = (
+    "This canary matched its pin, but it reported that what it read is out of date -- see "
+    "the note below. A match against stale data is not evidence that upstream is unchanged; "
+    "it is evidence that the thing being read stopped moving. Check the source's publication "
+    "schedule before treating this run as a verification."
+)
+_ERROR_REMEDIATION = (
+    "This canary raised an error that is not a transport failure, so upstream may be fine and "
+    "SourceLock's own adapter is what broke. Re-run with the adapter's fixture override to "
+    "reproduce it locally, and fix the adapter -- re-pinning will not help."
+)
+
+
+@dataclass
+class DoctorReport:
+    results: list[CanaryResult] = field(default_factory=list)
+    load_errors: list[AdapterLoadError] = field(default_factory=list)
+    lockfile_present: bool = False
+
+    @property
+    def summary(self) -> dict[str, int]:
+        counts = {s.value: 0 for s in CanaryStatus}
+        for r in self.results:
+            counts[r.status.value] += 1
+        # Not a status: a canary can be `ok` AND have been answered by a mirror.
+        # Surfaced in the summary so CI can gate on it without walking results.
+        counts["fallback"] = sum(1 for r in self.results if r.fallback_used)
+        # Nor is this: a finding that was reported and did not fail the build.
+        # It belongs in the summary line so an advisory canary that has been
+        # drifting for six months is visible rather than merely not-failing.
+        counts["advisory"] = sum(1 for r in self.advisory_findings)
+        counts["retried"] = sum(1 for r in self.results if r.attempts > 1)
+        return counts
+
+    @property
+    def advisory_findings(self) -> list[CanaryResult]:
+        """Non-ok results that are excluded from the verdict by declaration."""
+        return [
+            r
+            for r in self.results
+            if r.severity is CanarySeverity.ADVISORY and r.status is not CanaryStatus.OK
+        ]
+
+    @property
+    def exit_code(self) -> int:
+        if self.load_errors:
+            return EXIT_UNREACHABLE
+        # Advisory canaries report their status and stay out of the verdict.
+        # An adapter that fails to LOAD is never advisory: nothing declared
+        # anything, so there is nothing to have declared it harmless.
+        statuses = {
+            r.status for r in self.results if r.severity is CanarySeverity.BLOCKING
+        }
+        # ERROR is never advisory, whatever the canary declared.
+        #
+        # Severity is a statement about UPSTREAM -- "this source restyles its
+        # HTML at will, do not fail a build over it". ERROR is the one status
+        # that is a statement about US: the observation raised something that is
+        # not a transport failure, so SourceLock's own adapter is what broke.
+        # Filtering advisory results out BEFORE looking at status meant an
+        # advisory canary could crash our code and leave doctor and the action
+        # green -- a bug in this product silenced by a declaration about
+        # somebody else's website.
+        errored = any(r.status is CanaryStatus.ERROR for r in self.results)
+        # `error` rides with `unreachable`: both mean this run produced no
+        # verdict. They are separate STATUSES because the fix differs, and one
+        # exit code because CI's decision -- stop -- is the same.
+        if errored or CanaryStatus.UNREACHABLE in statuses:
+            return EXIT_UNREACHABLE
+        if CanaryStatus.UNPINNED in statuses or not self.lockfile_present:
+            return EXIT_UNPINNED
+        if statuses & {CanaryStatus.DRIFT, CanaryStatus.SCHEMA_CHANGED}:
+            return EXIT_DRIFT
+        if CanaryStatus.STALE in statuses:
+            return EXIT_STALE
+        return EXIT_OK
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "exit_code": self.exit_code,
+            "lockfile_present": self.lockfile_present,
+            "summary": self.summary,
+            # as_dict, not two hand-picked fields: a load error now carries which
+            # discovery route it came from, and "the adapter I pip-installed is
+            # broken" must not read the same as "the one you ship is broken".
+            "load_errors": [e.as_dict() for e in self.load_errors],
+            "results": [r.model_dump(mode="json") for r in self.results],
+        }
+
+
+def run_doctor(
+    lockfile: Lockfile | None,
+    adapters: list[SourceAdapter] | None = None,
+    *,
+    load_errors: list[AdapterLoadError] | None = None,
+) -> DoctorReport:
+    """Observe every canary of every adapter and grade it against the lockfile.
+
+    ``adapters`` scopes the run (e.g. ``adapters=[get_adapter("demo")]``); when
+    omitted, every discovered adapter is checked. ``load_errors`` lets a caller
+    that already ran discovery carry its load failures into the report.
+    """
+    load_errors = list(load_errors) if load_errors is not None else []
+    if adapters is None:
+        found = discover()
+        adapters = found.adapters
+        load_errors.extend(found.errors)
+
+    report = DoctorReport(load_errors=load_errors, lockfile_present=lockfile is not None)
+    seen_ids: set[str] = set()
+
+    for adapter in adapters:
+        retries = _effective_retries(adapter)
+        timeout = _effective_timeout(adapter)
+        for canary in adapter.canaries():
+            seen_ids.add(canary.canary_id)
+            report.results.append(
+                _check(
+                    canary,
+                    lockfile,
+                    retries=canary.retries if canary.retries is not None else retries,
+                    timeout=canary.timeout if canary.timeout is not None else timeout,
+                )
+            )
+
+    if lockfile is not None:
+        for source_id, entry in lockfile.sources.items():
+            live_source = any(a.source_id == source_id for a in adapters)
+            for canary_id in entry.expected_canaries:
+                if canary_id in seen_ids:
+                    continue
+                if not live_source:
+                    # The whole adapter is absent from this install; that is a
+                    # packaging question, not upstream drift.
+                    continue
+                report.results.append(
+                    CanaryResult(
+                        canary_id=canary_id,
+                        source_id=source_id,
+                        status=CanaryStatus.DRIFT,
+                        observed=None,
+                        expected=entry.expected_canaries[canary_id].value,
+                        checked_at=utcnow(),
+                        remediation=(
+                            f"source-lock.json pins canary {canary_id!r} but no adapter provides "
+                            "it. Either restore the canary in the adapter, or drop the entry with "
+                            "`hc-source lock init`."
+                        ),
+                    )
+                )
+
+    report.results.sort(key=lambda r: r.canary_id)
+    return report
+
+
+def _effective_retries(adapter: SourceAdapter) -> int:
+    """The adapter's declared retries, raised (never lowered) by the environment."""
+    declared = max(0, int(getattr(adapter, "canary_retries", 0) or 0))
+    return max(declared, _env_int(RETRIES_ENV, 0))
+
+
+def _effective_timeout(adapter: SourceAdapter) -> float | None:
+    env = os.environ.get(TIMEOUT_ENV)
+    if env is not None:
+        try:
+            value = float(env)
+        except ValueError:
+            value = 0.0
+        if value > 0:
+            return value
+    return getattr(adapter, "canary_timeout", None)
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    try:
+        return max(0, int(raw))
+    except ValueError:
+        return default
+
+
+def _observe(canary: Canary, retries: int, timeout: float | None):
+    """Observe once, or a few times if the failure looks transient.
+
+    Returns ``(observation, attempts)`` or raises the last failure. The retry is
+    deliberately narrow: only :class:`SourceUnreachable`, only when
+    :func:`hc_source.http.is_transient` says trying again could plausibly get a
+    different answer. A bug in our own parsing raises something else and is not
+    retried -- three identical TypeErrors are not more informative than one, and
+    a build that eventually goes green on re-run teaches people to press re-run.
+    """
+    attempts = max(0, retries) + 1
+    last: Exception | None = None
+    for attempt in range(attempts):
+        try:
+            with request_timeout(timeout):
+                return canary.observe(), attempt + 1
+        except SourceUnreachable as exc:
+            last = exc
+            if attempt == attempts - 1 or not is_transient(exc):
+                raise
+            _sleep(min(RETRY_BACKOFF * (2**attempt), MAX_RETRY_BACKOFF))
+    raise last  # pragma: no cover - the loop either returns or raises
+
+
+def _check(
+    canary: Canary,
+    lockfile: Lockfile | None,
+    *,
+    retries: int = 0,
+    timeout: float | None = None,
+) -> CanaryResult:
+    checked_at = utcnow()
+    warnings: list[str] = []
+    attempts = 1
+
+    try:
+        observation, attempts = _observe(canary, retries, timeout)
+    except SourceUnreachable as exc:
+        return _result(
+            canary,
+            CanaryStatus.UNREACHABLE,
+            observed=None,
+            expected=_expected_value(canary, lockfile),
+            checked_at=checked_at,
+            warnings=[
+                exc.reason
+                + (f" (after {retries + 1} attempts)" if retries else "")
+            ],
+            attempts=retries + 1 if is_transient(exc) else 1,
+        )
+    except Exception as exc:  # noqa: BLE001 - a bad canary must not end the run
+        # Not `unreachable`: nothing says the source is down. A TypeError in our
+        # own parsing was reported for a year as "CMS could not be read", which
+        # sent people to check a network that was never the problem.
+        return _result(
+            canary,
+            CanaryStatus.ERROR,
+            observed=None,
+            expected=_expected_value(canary, lockfile),
+            checked_at=checked_at,
+            warnings=[f"canary raised {type(exc).__name__}"],
+        )
+
+    if attempts > 1:
+        warnings.append(
+            f"observed on attempt {attempts} of {retries + 1}: the first "
+            f"{attempts - 1} failed transiently. This source is answering, but not "
+            "reliably on the first try"
+        )
+
+    if observation.fallback_used:
+        warnings.append(
+            _FALLBACK_WARNING.format(name=observation.fallback_name or "unnamed mirror")
+        )
+    # The canary's own note is the only channel an adapter has for "I saw
+    # something you should know about". It used to be recorded on the
+    # observation and then dropped: coverage's stale-snapshot signal and leie's
+    # skipped waiver cross-check never reached the CLI, the JSON, or CI.
+    if observation.note:
+        warnings.append(observation.note)
+
+    expectation = lockfile.expectation(canary.source_id, canary.canary_id) if lockfile else None
+    if expectation is None:
+        warnings.append(_UNPINNED_WARNING)
+        return _result(
+            canary,
+            CanaryStatus.UNPINNED,
+            observed=observation.value,
+            expected=None,
+            checked_at=checked_at,
+            warnings=warnings,
+            observation=observation,
+            attempts=attempts,
+        )
+
+    status = _classify(observation, expectation, warnings)
+
+    return _result(
+        canary,
+        status,
+        observed=observation.value,
+        expected=expectation.value,
+        checked_at=checked_at,
+        warnings=warnings,
+        observation=observation,
+        attempts=attempts,
+    )
+
+
+def _classify(observation, expectation, warnings: list[str]) -> CanaryStatus:
+    """Grade one observation against its pin. Shape first, then value.
+
+    The old order asked "did the value move?" first and only looked at the
+    schema hash when BOTH hashes were non-null. So a source that stopped
+    publishing its shape entirely -- the hash going from a real digest to
+    ``None`` -- came back ``ok``, which is the loudest possible upstream event
+    reported as the quietest possible verdict. Shape loss is graded first, and
+    when the value moved too, that fact is recorded rather than discarded: two
+    facets, one status, no silent facts.
+    """
+    schema_lost = expectation.schema_hash is not None and observation.schema_hash is None
+    schema_moved = (
+        expectation.schema_hash is not None
+        and observation.schema_hash is not None
+        and observation.schema_hash != expectation.schema_hash
+    )
+    value_moved = observation.value != expectation.value
+
+    if schema_lost or schema_moved:
+        if schema_lost:
+            warnings.append(
+                "the pin records a schema hash but this observation carries none: the source "
+                "stopped exposing its shape, so nothing about the shape was verified"
+            )
+        if value_moved:
+            warnings.append("the observed value moved as well as the shape")
+        return CanaryStatus.SCHEMA_CHANGED
+
+    if value_moved:
+        return CanaryStatus.DRIFT
+
+    # Matched. Whether that is assurance depends on what the canary read.
+    if observation.stale:
+        return CanaryStatus.STALE
+    return CanaryStatus.OK
+
+
+def _expected_value(canary: Canary, lockfile: Lockfile | None) -> str | None:
+    expectation = lockfile.expectation(canary.source_id, canary.canary_id) if lockfile else None
+    return expectation.value if expectation else None
+
+
+def _result(
+    canary: Canary,
+    status: CanaryStatus,
+    *,
+    observed: str | None,
+    expected: str | None,
+    checked_at,
+    warnings: list[str],
+    observation: CanaryObservation | None = None,
+    attempts: int = 1,
+) -> CanaryResult:
+    if status is CanaryStatus.OK:
+        remediation = ""
+    elif status is CanaryStatus.UNPINNED:
+        # The canary's own remediation text is about DRIFT ("upstream moved,
+        # re-pin"). Unpinned is a different problem with a different fix.
+        remediation = _UNPINNED_REMEDIATION
+    elif status is CanaryStatus.STALE:
+        remediation = _STALE_REMEDIATION
+    elif status is CanaryStatus.ERROR:
+        remediation = _ERROR_REMEDIATION
+    else:
+        remediation = canary.resolve_remediation(status, observed, expected)
+    return CanaryResult(
+        canary_id=canary.canary_id,
+        source_id=canary.source_id,
+        status=status,
+        severity=canary.severity,
+        attempts=max(1, attempts),
+        observed=observed,
+        expected=expected,
+        checked_at=checked_at,
+        remediation=remediation,
+        warnings=warnings,
+        fallback_used=bool(observation and observation.fallback_used),
+        fallback_name=(observation.fallback_name if observation else None),
+    )