sourcelock 0.1.0__py3-none-any.whl

hc_source/interfaces.py

@@ -0,0 +1,395 @@
+"""The adapter interface.
+
+This module is the contract between SourceLock's core and the route adapters.
+It is deliberately small: four types and one base class.
+
+    from hc_source.adapters import (
+        Canary, CanaryObservation, SourceAdapter, ToolResult, ToolSpec,
+    )
+
+A route adapter is a single module in ``hc_source/adapters/`` that ends with::
+
+    ADAPTER = MyAdapter()
+
+Nothing else in the repository needs to change for it to be discovered, listed,
+called, canaried, locked, and served over MCP.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Mapping
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from pydantic import ValidationError
+
+from .guard import PHI_NON_CLAIM, ParamsRejected, assert_public_params, safe_validation_reasons
+from .schemas import CanarySeverity, CanaryStatus, Receipt, SourceContract
+
+__all__ = [
+    "AdapterError",
+    "Canary",
+    "CanaryObservation",
+    "CanarySeverity",
+    "SourceAdapter",
+    "ToolResult",
+    "ToolSpec",
+    "validate_adapter",
+]
+
+
+class AdapterError(Exception):
+    """An adapter module is malformed. Raised at discovery time, never later."""
+
+
+class CanaryObservation(BaseModel):
+    """What a canary saw upstream, reduced to something comparable.
+
+    ``value`` is the token the lockfile pins -- a release id, a row count, a
+    version string. Keep it short, deterministic, and free of payload data.
+    ``schema_hash`` lets doctor distinguish "the data moved" (drift) from "the
+    shape moved" (schema_changed), which need different fixes.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    value: str = Field(min_length=1, description="Comparable observation token.")
+    schema_hash: str | None = Field(
+        default=None, description="Stable hash of the upstream shape, if the source exposes one."
+    )
+    upstream_status: int | None = None
+    note: str | None = Field(
+        default=None,
+        description=(
+            "One short line for humans. No payloads. Doctor copies this onto the "
+            "result's warnings, so it reaches the CLI, the JSON report, and the "
+            "GitHub annotations -- it used to be dropped on the floor."
+        ),
+    )
+    stale: bool = Field(
+        default=False,
+        description=(
+            "True when the canary read something real but out of date: a weekly "
+            "snapshot that has not moved in a month, a cross-check it had to skip. "
+            "Doctor downgrades an otherwise-matching canary to STALE, because an "
+            "`ok` computed from data the canary itself calls stale is not "
+            "assurance. Say why in `note`."
+        ),
+    )
+    fallback_used: bool = Field(
+        default=False,
+        description=(
+            "True when this observation came from a mirror because the pinned authority "
+            "was unreachable. Pass `FetchResult.fallback_used` straight through: doctor "
+            "cannot tell 'the authority agrees with the pin' from 'the authority is dark "
+            "and a mirror agrees with the pin' unless the canary says so."
+        ),
+    )
+    fallback_name: str | None = Field(
+        default=None, description="Which mirror answered, when fallback_used is True."
+    )
+
+
+@dataclass(frozen=True)
+class Canary:
+    """One cheap, deterministic, unauthenticated check against a source.
+
+    ``observe`` must either return a :class:`CanaryObservation` or raise
+    :class:`hc_source.http.SourceUnreachable`. Any other exception is caught by
+    doctor and reported as unreachable with a warning -- it is never allowed to
+    take the run down.
+    """
+
+    canary_id: str
+    source_id: str
+    description: str
+    observe: Callable[[], CanaryObservation]
+    #: Exact human-readable fix, shown on any non-ok status. Required.
+    remediation: str
+    #: Optional override producing status-specific remediation text.
+    remediation_for: Callable[[CanaryStatus, str | None, str | None], str] | None = None
+    #: Whether a finding here may fail somebody's build.
+    #:
+    #: ``BLOCKING`` (the default) is the honest default: a canary exists because
+    #: a change upstream would make an answer wrong. ``ADVISORY`` is for the
+    #: checks that cannot support that weight -- an HTML scrape of a page CMS
+    #: restyles at will, a cross-check whose disagreement means "look at this",
+    #: not "your data is wrong". An advisory finding is reported with its real
+    #: status, annotated in CI, and left out of the exit code. Do not reach for
+    #: it to quiet a canary that is telling the truth: a gate that cries wolf
+    #: gets uninstalled, and a gate that never fires gets uninstalled too.
+    severity: CanarySeverity = CanarySeverity.BLOCKING
+    #: Retries for a TRANSIENT upstream failure, or ``None`` for the adapter's
+    #: default. Never retries a bug in our own code, a body over the ceiling, or
+    #: a 4xx that is not 408/425/429 -- retrying those just spends the build's
+    #: time to reach the same answer.
+    retries: int | None = None
+    #: Per-request timeout in seconds for fetches made during this observation,
+    #: or ``None`` for the adapter's default. A canary is supposed to be cheap;
+    #: this is where you say what cheap means for a source that is slow.
+    timeout: float | None = None
+
+    def resolve_remediation(
+        self, status: CanaryStatus, observed: str | None, expected: str | None
+    ) -> str:
+        if self.remediation_for is not None:
+            text = self.remediation_for(status, observed, expected)
+            if text and text.strip():
+                return text
+        return self.remediation
+
+
+class ToolResult(BaseModel):
+    """A tool's answer and the evidence behind it."""
+
+    model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
+
+    data: Any = Field(description="The answer. JSON-serialisable.")
+    receipt: Receipt
+
+
+@dataclass(frozen=True)
+class ToolSpec:
+    """One callable route.
+
+    ``params_model`` must be a pydantic model with ``extra='forbid'``: SourceLock
+    accepts public typed parameters only, and silently ignoring an unexpected
+    parameter is how a caller ends up believing it filtered something.
+
+    ``handler`` receives the VALIDATED model instance and returns a
+    :class:`ToolResult`. Call the tool through :meth:`invoke`, never the handler
+    directly -- ``invoke`` is where the zero-PHI guard runs.
+    """
+
+    name: str
+    description: str
+    params_model: type[BaseModel]
+    handler: Callable[[Any], ToolResult]
+    tags: tuple[str, ...] = field(default=())
+
+    @property
+    def source_id(self) -> str:
+        return self.name.split(".", 1)[0]
+
+    def params_schema(self) -> dict[str, Any]:
+        schema = self.params_model.model_json_schema()
+        schema.setdefault("type", "object")
+        return schema
+
+    def invoke(self, raw_params: Mapping[str, Any] | None = None) -> ToolResult:
+        """Guard, validate, call, and stamp the receipt.
+
+        Raises :class:`hc_source.guard.PHIRejected` for patient-shaped input and
+        :class:`hc_source.guard.ParamsRejected` (a ``ValueError``) for anything
+        that does not fit the declared parameter model.
+
+        The ``ValidationError`` is deliberately not allowed to escape. Its
+        ``str()`` contains ``input_value=<the raw value>``, and a caller that
+        logs the exception would then have written the value it asked us to
+        refuse. Converting it here means every consumer of ``invoke`` -- CLI,
+        MCP server, or someone else's agent framework -- is value-safe by
+        construction rather than by remembering to be careful.
+        """
+        params = dict(raw_params or {})
+        assert_public_params(params, tool=self.name)
+        try:
+            model = self.params_model.model_validate(params)
+        except ValidationError as exc:
+            # `from None`: chaining would put the raw ValidationError, and with
+            # it the input value, into any printed traceback.
+            raise ParamsRejected(self.name, safe_validation_reasons(exc)) from None
+
+        # The result cache sits AFTER the guard and the model, never before. A
+        # cache consulted on raw input would answer a question nobody validated,
+        # and a patient-shaped parameter would become a cache key.
+        cached = _cache_lookup(self.name, model)
+        if cached is not None:
+            return cached
+
+        result = self.handler(model)
+
+        if not isinstance(result, ToolResult):
+            raise AdapterError(f"{self.name}: handler returned {type(result).__name__}, expected ToolResult")
+        receipt = result.receipt
+        _assert_receipt_is_bound(self.name, result.receipt)
+        if PHI_NON_CLAIM not in receipt.non_claims:
+            receipt.non_claims.insert(0, PHI_NON_CLAIM)
+        _cache_store(self.name, model, result)
+        return result
+
+
+def _assert_receipt_is_bound(tool_name: str, receipt: Receipt) -> None:
+    """A receipt may only speak for the route and the source that produced it.
+
+    ``route`` was checked from the start. ``source_id`` was not, and it is the
+    field a reader trusts to know WHOSE data an answer came from: the lockfile
+    pins by source id, doctor grades by source id, and every "this came from
+    CMS" claim in the product is that string. An installed plugin's
+    ``acme.lookup`` could return a receipt saying ``source_id='codes'`` and the
+    answer would be filed under a source it never touched -- silently, since
+    the two fields are read in different places. Discovery already refuses to
+    let a plugin CLAIM a built-in source id; this refuses to let one borrow the
+    name on the way out.
+    """
+    if receipt.route != tool_name:
+        raise AdapterError(
+            f"{tool_name}: receipt.route is {receipt.route!r}; it must equal the tool name"
+        )
+    expected = tool_name.split(".", 1)[0]
+    if receipt.source_id != expected:
+        raise AdapterError(
+            f"{tool_name}: receipt.source_id is {receipt.source_id!r}, but this tool "
+            f"belongs to {expected!r}. A receipt names the source an answer came from; "
+            "one adapter may not issue evidence under another's name."
+        )
+
+
+#: Warning added to a receipt served out of the result cache. It is a warning
+#: rather than a silent flag because the result cache, unlike the HTTP cache,
+#: did not ask upstream anything -- the operator opted into that tradeoff by
+#: setting a TTL, and every answer that took it says so.
+CACHE_HIT_WARNING = (
+    "CACHED_RESULT: this answer was replayed from the local result cache "
+    "(HC_SOURCE_TOOL_CACHE_TTL) and no source was contacted. retrieved_at, "
+    "source_version and raw_sha256 describe the ORIGINAL read; clear the cache "
+    "with `hc-source cache clear` to force a fresh one."
+)
+
+
+def _cache_lookup(tool: str, model: BaseModel) -> "ToolResult | None":
+    """A stored result for this exact call, or ``None``. Off unless a TTL is set."""
+    from . import cache
+    from .lockfile import lockfile_fingerprint
+
+    ttl = cache.tool_ttl()
+    if ttl <= 0:
+        return None
+    key = cache.tool_key(
+        tool, model.model_dump(mode="json"), lockfile_hash=lockfile_fingerprint()
+    )
+    stored = cache.read_tool(key, ttl)
+    if stored is None:
+        return None
+    data, receipt_payload = stored
+    try:
+        receipt = Receipt.model_validate(receipt_payload)
+    except ValidationError:
+        # An entry we cannot parse is a miss. Serving a receipt we could not
+        # validate would be handing back evidence nobody checked.
+        return None
+    try:
+        # The same binding the live path enforces. A stored entry passed it once
+        # when it was written, so reaching this is a cache directory that is not
+        # what this build thinks it is -- and the safe reading of that is "miss",
+        # not "raise": the caller asked a question that has a perfectly good
+        # fresh answer.
+        _assert_receipt_is_bound(tool, receipt)
+    except AdapterError:
+        return None
+    receipt.cache_hit = True
+    if CACHE_HIT_WARNING not in receipt.warnings:
+        receipt.warnings.append(CACHE_HIT_WARNING)
+    return ToolResult(data=data, receipt=receipt)
+
+
+def _cache_store(tool: str, model: BaseModel, result: "ToolResult") -> None:
+    from . import cache
+    from .lockfile import lockfile_fingerprint
+
+    if cache.tool_ttl() <= 0:
+        return
+    key = cache.tool_key(
+        tool, model.model_dump(mode="json"), lockfile_hash=lockfile_fingerprint()
+    )
+    cache.write_tool(key, result.data, result.receipt.model_dump(mode="json"))
+
+
+class SourceAdapter:
+    """Base class for a route adapter.
+
+    Subclasses set ``source_id`` and ``contract`` and override ``canaries`` and
+    ``tools``. Both are methods rather than attributes so an adapter can build
+    them lazily from configuration without import-time work.
+    """
+
+    source_id: str = ""
+    contract: SourceContract
+
+    #: Retries doctor allows each of this adapter's canaries on a TRANSIENT
+    #: upstream failure. Zero is right for a source read from vendored data --
+    #: there is no network to be flaky. Set 2 when the canaries talk to a live
+    #: service: CMS returns the occasional 502, and a red build that goes green
+    #: on re-run teaches people to press re-run.
+    canary_retries: int = 0
+
+    #: Per-request timeout in seconds for this adapter's canaries. ``None``
+    #: means the transport default.
+    canary_timeout: float | None = None
+
+    def canaries(self) -> list[Canary]:
+        return []
+
+    def tools(self) -> list[ToolSpec]:
+        return []
+
+    def __repr__(self) -> str:  # pragma: no cover - debugging aid
+        return f"<{type(self).__name__} source_id={self.source_id!r}>"
+
+
+def validate_adapter(adapter: Any, *, origin: str = "<unknown>") -> SourceAdapter:
+    """Check an ADAPTER object against the interface. Raises :class:`AdapterError`.
+
+    Discovery runs this on every adapter, so a malformed adapter fails at load
+    with a message naming its module -- not halfway through a doctor run.
+    """
+    where = f"{origin}: ADAPTER"
+
+    if not isinstance(adapter, SourceAdapter):
+        raise AdapterError(f"{where} must subclass hc_source.adapters.SourceAdapter")
+
+    source_id = getattr(adapter, "source_id", "")
+    if not isinstance(source_id, str) or not source_id:
+        raise AdapterError(f"{where} must set a non-empty source_id")
+
+    contract = getattr(adapter, "contract", None)
+    if not isinstance(contract, SourceContract):
+        raise AdapterError(f"{where}.contract must be a SourceContract")
+    if contract.source_id != source_id:
+        raise AdapterError(
+            f"{where}.contract.source_id is {contract.source_id!r} but source_id is {source_id!r}"
+        )
+
+    canaries = adapter.canaries()
+    if not isinstance(canaries, list) or not all(isinstance(c, Canary) for c in canaries):
+        raise AdapterError(f"{where}.canaries() must return a list[Canary]")
+    seen_canaries: set[str] = set()
+    for c in canaries:
+        if c.source_id != source_id:
+            raise AdapterError(f"{where}: canary {c.canary_id!r} has source_id {c.source_id!r}")
+        if not c.canary_id.startswith(f"{source_id}."):
+            raise AdapterError(f"{where}: canary id {c.canary_id!r} must start with '{source_id}.'")
+        if c.canary_id in seen_canaries:
+            raise AdapterError(f"{where}: duplicate canary id {c.canary_id!r}")
+        if not (c.remediation.strip() or c.remediation_for):
+            raise AdapterError(f"{where}: canary {c.canary_id!r} needs remediation text")
+        seen_canaries.add(c.canary_id)
+
+    tools = adapter.tools()
+    if not isinstance(tools, list) or not all(isinstance(t, ToolSpec) for t in tools):
+        raise AdapterError(f"{where}.tools() must return a list[ToolSpec]")
+    seen_tools: set[str] = set()
+    for t in tools:
+        if not t.name.startswith(f"{source_id}."):
+            raise AdapterError(f"{where}: tool name {t.name!r} must start with '{source_id}.'")
+        if t.name in seen_tools:
+            raise AdapterError(f"{where}: duplicate tool name {t.name!r}")
+        if not (isinstance(t.params_model, type) and issubclass(t.params_model, BaseModel)):
+            raise AdapterError(f"{where}: tool {t.name!r} params_model must be a pydantic model")
+        if t.params_model.model_config.get("extra") != "forbid":
+            raise AdapterError(
+                f"{where}: tool {t.name!r} params_model must set ConfigDict(extra='forbid')"
+            )
+        seen_tools.add(t.name)
+
+    return adapter

hc_source/lockfile.py

@@ -0,0 +1,236 @@
+"""Reading, writing, and building ``source-lock.json``.
+
+The lockfile is what turns "the data moved" from a silent production surprise
+into a failing build. It is meant to be committed and reviewed like any other
+lockfile.
+
+Two rules govern everything in this module, both learned the hard way:
+
+* **A failed observation never erases a pin.** ``build_lockfile`` used to
+  ``continue`` past any canary that raised, then emit whatever it managed to
+  see. An offline ``hc-source lock init --source codes`` therefore pinned the
+  three vendored canaries, silently dropped the two live release-train pins, and
+  exited 0 -- destroying the only artifact that can detect drift, in the one
+  situation (no network) where the operator is least likely to notice.
+* **Nothing is written until the whole build validates.** The write is a
+  temp-file-plus-rename, so a crash or a full disk leaves the previous lockfile
+  intact rather than a truncated one.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .interfaces import SourceAdapter
+from .schemas import ExpectedCanary, Lockfile, SourceLockEntry, utcnow
+
+__all__ = [
+    "DEFAULT_LOCK_FILENAME",
+    "CanaryFailure",
+    "LockfileBuild",
+    "build_lockfile",
+    "default_lock_path",
+    "load_lockfile",
+    "lockfile_fingerprint",
+    "save_lockfile",
+]
+
+DEFAULT_LOCK_FILENAME = "source-lock.json"
+
+
+def default_lock_path() -> Path:
+    return Path.cwd() / DEFAULT_LOCK_FILENAME
+
+
+def lockfile_fingerprint(path: Path | None = None) -> str:
+    """SHA-256 of the lockfile on disk, or ``""`` when there is none.
+
+    Used as part of the tool-result cache key. Re-pinning a source must not
+    leave yesterday's answers reachable under today's pins: a cached answer
+    citing a release it was never derived from is precisely the failure this
+    product exists to prevent, and a cache is a fast way to build one.
+    """
+    path = Path(path) if path is not None else default_lock_path()
+    try:
+        return hashlib.sha256(path.read_bytes()).hexdigest()
+    except OSError:
+        return ""
+
+
+def load_lockfile(path: Path) -> Lockfile:
+    """Load and validate a lockfile. Raises FileNotFoundError with the fix."""
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(
+            f"no lockfile at {path}. Create one with `hc-source lock init`."
+        )
+    return Lockfile.model_validate_json(path.read_text(encoding="utf-8"))
+
+
+def save_lockfile(lockfile: Lockfile, path: Path) -> Path:
+    """Write a lockfile as stable, diff-friendly JSON, atomically.
+
+    Serialise first, then write a temp file in the destination directory, then
+    ``os.replace``. A half-written ``source-lock.json`` is worse than a missing
+    one: it fails validation on the next run, which reads as "regenerate it",
+    which is how a partial write becomes a total loss.
+    """
+    path = Path(path)
+    payload = lockfile.model_dump_json(indent=2) + "\n"
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    handle, tmp_name = tempfile.mkstemp(
+        dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp"
+    )
+    try:
+        with os.fdopen(handle, "w", encoding="utf-8") as fh:
+            fh.write(payload)
+            fh.flush()
+            os.fsync(fh.fileno())
+        os.replace(tmp_name, path)
+    except BaseException:
+        try:
+            os.unlink(tmp_name)
+        except OSError:  # pragma: no cover - best effort cleanup
+            pass
+        raise
+    return path
+
+
+@dataclass(frozen=True)
+class CanaryFailure:
+    """One canary that could not be observed during a ``lock init``."""
+
+    canary_id: str
+    source_id: str
+    reason: str
+    #: True when a previous pin for this canary was carried forward untouched.
+    pin_preserved: bool = False
+
+    def __str__(self) -> str:
+        kept = "previous pin kept" if self.pin_preserved else "NOT PINNED"
+        return f"{self.canary_id}: {self.reason} ({kept})"
+
+
+@dataclass
+class LockfileBuild:
+    """The result of observing every selected canary once.
+
+    ``lockfile`` is only safe to write when ``complete`` is true, or when the
+    operator has explicitly accepted a partial pin.
+    """
+
+    lockfile: Lockfile
+    failures: list[CanaryFailure] = field(default_factory=list)
+    #: Canary ids observed fresh from upstream in this run.
+    observed: list[str] = field(default_factory=list)
+
+    @property
+    def complete(self) -> bool:
+        return not self.failures
+
+    @property
+    def preserved(self) -> list[str]:
+        return [f.canary_id for f in self.failures if f.pin_preserved]
+
+    @property
+    def unpinned(self) -> list[str]:
+        return [f.canary_id for f in self.failures if not f.pin_preserved]
+
+
+def build_lockfile(
+    adapters: list[SourceAdapter], previous: Lockfile | None = None
+) -> LockfileBuild:
+    """Observe every canary once and pin what it saw.
+
+    A canary that cannot be observed is recorded as a :class:`CanaryFailure` and
+    its PREVIOUS pin, if there is one, is carried through unchanged. The caller
+    decides what to do about it; this function never silently narrows a lockfile.
+
+    When ``previous`` is supplied and a source's expectations are unchanged, its
+    timestamps are carried over so that re-running ``lock init`` on an unchanged
+    world produces a byte-identical file. A lockfile that churns on every run is
+    a lockfile nobody reads in review.
+    """
+    now = utcnow()
+    sources: dict[str, SourceLockEntry] = {}
+    failures: list[CanaryFailure] = []
+    observed_ids: list[str] = []
+
+    for adapter in adapters:
+        prior = previous.sources.get(adapter.source_id) if previous else None
+        canaries = adapter.canaries()
+        expected: dict[str, ExpectedCanary] = {}
+        #: Canary ids in declaration order, so release_id/schema_hash stay stable.
+        order: list[str] = []
+
+        for canary in canaries:
+            order.append(canary.canary_id)
+            try:
+                observation = canary.observe()
+            except Exception as exc:  # noqa: BLE001 - unreachable or broken; do not pin a guess
+                kept = prior.expected_canaries.get(canary.canary_id) if prior else None
+                if kept is not None:
+                    expected[canary.canary_id] = kept
+                failures.append(
+                    CanaryFailure(
+                        canary_id=canary.canary_id,
+                        source_id=adapter.source_id,
+                        reason=_reason(exc),
+                        pin_preserved=kept is not None,
+                    )
+                )
+                continue
+            expected[canary.canary_id] = ExpectedCanary(
+                value=observation.value, schema_hash=observation.schema_hash
+            )
+            observed_ids.append(canary.canary_id)
+
+        if not expected:
+            # Nothing observed and nothing to preserve. Keep the previous entry
+            # rather than deleting a source because today's run could not reach it.
+            if prior is not None:
+                sources[adapter.source_id] = prior
+            continue
+
+        release_value: str | None = None
+        schema_hash: str | None = None
+        for canary_id in order:
+            pin = expected.get(canary_id)
+            if pin is None:
+                continue
+            if release_value is None:
+                release_value = pin.value
+            if schema_hash is None and pin.schema_hash:
+                schema_hash = pin.schema_hash
+
+        expected = dict(sorted(expected.items()))
+        unchanged = prior is not None and prior.expected_canaries == expected
+
+        sources[adapter.source_id] = SourceLockEntry(
+            source_id=adapter.source_id,
+            pinned_version=release_value or "unknown",
+            release_id=release_value,
+            schema_hash=schema_hash,
+            checked_at=prior.checked_at if unchanged else now,
+            expected_canaries=expected,
+        )
+
+    sources = dict(sorted(sources.items()))
+    if previous is not None and previous.sources == sources:
+        lockfile = Lockfile(generated_at=previous.generated_at, sources=sources)
+    else:
+        lockfile = Lockfile(generated_at=now, sources=sources)
+    return LockfileBuild(lockfile=lockfile, failures=failures, observed=observed_ids)
+
+
+def _reason(exc: BaseException) -> str:
+    """A short, payload-free reason. Never the exception's own str() blindly."""
+    reason = getattr(exc, "reason", None)
+    if isinstance(reason, str) and reason.strip():
+        return reason.strip()
+    return type(exc).__name__