sourcelock 0.1.0__py3-none-any.whl

hc_source/http.py

@@ -0,0 +1,541 @@
+"""The one transport every adapter uses.
+
+Adapters do not create their own httpx clients. Going through :func:`fetch`
+buys three things the receipt depends on: the SHA-256 of the exact bytes an
+answer was derived from, a uniform authority/fallback policy, and the guarantee
+that no response body is ever logged.
+
+``file://`` URLs are supported so that fixtures and locally-mirrored releases
+travel the same code path as live HTTP.
+
+Response bodies are STREAMED under a byte ceiling and hashed incrementally.
+Round-1 finding H6: this module read ``response.content`` whole with no cap
+while explicitly asking for gzip, so a 300 MB body at a canary URL took the
+process to 651 MB RSS and a ~200 KB gzip bomb inflated to 200 MB -- about 1000x
+amplification, from the one side an attacker or a broken CDN actually controls.
+The guard had capped INBOUND parameters at 64 KiB since day one and there was
+no equivalent for the response.
+
+The default ceiling comes from live measurement of every URL the shipped
+adapters fetch:
+
+===========================================  ============
+``current_lcd.zip`` (coverage bulk crosswalk)  31.8 MB
+``UPDATED.csv`` (OIG LEIE full file)           14.8 MB
+local-coverage-articles report                  1.02 MB
+every other full-body fetch                    < 300 KB
+NPPES bulk files                               never fetched (7 KB index only)
+===========================================  ============
+
+So the default is deliberately tight and the two large fetches raise it for
+themselves. That way the two dozen small routes -- including the NPPES canary
+that H6 was demonstrated against -- get a ceiling proportionate to what they
+actually read, instead of one sized for the biggest file in the product.
+"""
+
+from __future__ import annotations
+
+import contextvars
+import hashlib
+import json
+import os
+import zlib
+from contextlib import contextmanager
+from dataclasses import dataclass, field, replace
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Iterator, Mapping
+from urllib.parse import unquote, urlparse
+
+import httpx
+
+from . import cache
+
+__all__ = [
+    "DEFAULT_MAX_BYTES",
+    "DEFAULT_TIMEOUT",
+    "FetchResult",
+    "RETRYABLE_STATUSES",
+    "ResponseTooLarge",
+    "SourceUnreachable",
+    "USER_AGENT",
+    "as_cache_hit",
+    "fetch",
+    "fetch_json",
+    "is_transient",
+    "request_timeout",
+]
+
+DEFAULT_TIMEOUT = 20.0
+USER_AGENT = "sourcelock/0.1 (+https://github.com/sourcelock)"
+
+#: Statuses worth trying again. Everything else -- 401, 403, 404, a 200 whose
+#: body was malformed -- returns the same answer on the second attempt and just
+#: spends the build's time getting there. Retrying is doctor's decision, but
+#: WHICH failures are transient is knowledge about the transport, so it lives
+#: here beside the exception that carries the status.
+RETRYABLE_STATUSES = frozenset({408, 425, 429, 500, 502, 503, 504})
+
+#: Ambient timeout, set by doctor around one canary's observation. A ContextVar
+#: rather than a module global because it is scoped to a call: a global would
+#: leak the last canary's budget into every tool call that followed it.
+_TIMEOUT_VAR: contextvars.ContextVar[float | None] = contextvars.ContextVar(
+    "hc_source_http_timeout", default=None
+)
+
+
+@contextmanager
+def request_timeout(seconds: float | None) -> Iterator[None]:
+    """Apply a per-request timeout to every fetch made inside this block.
+
+    This is how a canary's declared budget reaches a transport it never passes
+    arguments to: the canary's ``observe`` takes no parameters by design, so
+    doctor sets the policy around the call instead of threading it through every
+    adapter. An explicit ``timeout=`` at a call site still wins -- the coverage
+    bulk zip needs 300 seconds and should not inherit a canary's twenty.
+    """
+    token = _TIMEOUT_VAR.set(seconds if seconds is not None else _TIMEOUT_VAR.get())
+    try:
+        yield
+    finally:
+        _TIMEOUT_VAR.reset(token)
+
+
+def _env_float(name: str) -> float | None:
+    raw = os.environ.get(name)
+    if raw is None:
+        return None
+    try:
+        value = float(raw)
+    except ValueError:
+        return None
+    return value if value > 0 else None
+
+
+def _resolve_timeout(explicit: float | None) -> float:
+    """Call site, then ambient policy, then environment, then the default."""
+    if explicit is not None:
+        return explicit
+    ambient = _TIMEOUT_VAR.get()
+    if ambient is not None:
+        return ambient
+    return _env_float("HC_SOURCE_HTTP_TIMEOUT") or DEFAULT_TIMEOUT
+
+
+def is_transient(exc: SourceUnreachable) -> bool:
+    """Would trying again plausibly get a different answer?
+
+    A body over the ceiling is excluded on purpose: it will be over the ceiling
+    again, and downloading it twice to find that out is the opposite of cheap.
+    A ``file://`` failure is excluded for the same reason -- a local file that
+    could not be read will not read differently in 500 milliseconds.
+    """
+    if isinstance(exc, ResponseTooLarge):
+        return False
+    if urlparse(exc.url).scheme == "file":
+        return False
+    return exc.status is None or exc.status in RETRYABLE_STATUSES
+
+
+#: Ceiling on the DECOMPRESSED size of one response, in bytes. Sized against
+#: the measured payloads in the module docstring: 15x the largest routine fetch
+#: (the 1.02 MB articles report) and comfortably below anything that threatens
+#: a 2-7 GB CI container. The two genuinely large fetches -- the LEIE full file
+#: and the MCD bulk zip -- pass their own ``max_bytes``.
+DEFAULT_MAX_BYTES = 16 * 1024 * 1024
+
+#: Chunk size for streaming. Small enough that the ceiling is enforced promptly,
+#: large enough not to make a 30 MB download a syscall storm.
+_CHUNK = 64 * 1024
+
+#: Content encodings we will inflate ourselves. Anything else is refused rather
+#: than handed to a decoder we cannot bound.
+_SUPPORTED_ENCODINGS = {"", "identity", "gzip", "x-gzip", "deflate"}
+
+
+class SourceUnreachable(RuntimeError):
+    """The upstream source could not be read.
+
+    Carries no response body: doctor output and logs stay payload-free.
+    """
+
+    def __init__(self, url: str, reason: str, *, status: int | None = None) -> None:
+        self.url = url
+        self.reason = reason
+        self.status = status
+        super().__init__(f"{url} unreachable: {reason}" + (f" (HTTP {status})" if status else ""))
+
+
+class ResponseTooLarge(SourceUnreachable):
+    """The upstream body exceeded this call's byte ceiling.
+
+    A subclass of :class:`SourceUnreachable` on purpose. Every caller that
+    already handles an unreachable source -- doctor, the CLI, the MCP server --
+    handles this correctly without changing, and the classification is honest:
+    the source was not read, so this run does not know what it says.
+
+    Never truncate-and-continue. A hash of the first N bytes with a receipt
+    attesting to it is a confident wrong answer, which is the one thing this
+    product exists not to produce.
+    """
+
+    def __init__(self, url: str, limit: int, *, status: int | None = None) -> None:
+        self.limit = limit
+        super().__init__(
+            url,
+            f"response exceeded the {limit}-byte ceiling for this call. If this source "
+            "legitimately grew, raise max_bytes at the call site; otherwise the upstream "
+            "is serving something it should not",
+            status=status,
+        )
+
+
+def _utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+@dataclass(frozen=True)
+class FetchResult:
+    """Raw upstream bytes plus the provenance a receipt needs."""
+
+    url: str
+    status: int | None
+    content: bytes
+    sha256: str
+    headers: dict[str, str]
+    fallback_used: bool = False
+    fallback_name: str | None = None
+    #: When THESE bytes were read from upstream. Stamped here, at the fetch
+    #: boundary, and never re-stamped -- round-1 finding: ``build_receipt`` used
+    #: to call ``utcnow()`` when the receipt was assembled, so a route served
+    #: out of a process cache produced a receipt claiming it had just been to
+    #: CMS. The receipt is the product; a retrieval time it invented is the one
+    #: field that must never be a guess. A cached ``FetchResult`` carries the
+    #: timestamp of the fetch that created it, because the object is built once.
+    retrieved_at: datetime = field(default_factory=_utcnow)
+    #: True when these bytes came out of the on-disk cache rather than off the
+    #: wire. Never inferred: the receipt carries it, so a reader can tell an
+    #: answer computed from a fresh download from one computed from bytes stored
+    #: last week and re-confirmed a moment ago.
+    cache_hit: bool = False
+    #: When upstream last confirmed these bytes are current -- the moment of the
+    #: 304 on a cache hit, the moment of the download otherwise. Kept apart from
+    #: ``retrieved_at`` because "you still have the right bytes" and "these bytes
+    #: are new" are different statements, and a receipt that merges them is
+    #: back to inventing a retrieval time.
+    revalidated_at: datetime | None = None
+
+    def json(self) -> Any:
+        try:
+            return json.loads(self.content)
+        except ValueError as exc:
+            raise SourceUnreachable(self.url, f"response was not valid JSON: {type(exc).__name__}")
+
+
+#: Sentinel for "leave ``revalidated_at`` alone", distinct from ``None``, which
+#: means "nobody upstream has ever confirmed these bytes".
+_KEEP = object()
+
+
+def as_cache_hit(result: FetchResult, *, revalidated_at: Any = _KEEP) -> FetchResult:
+    """The same bytes, served again out of an adapter's own in-process cache.
+
+    Adapters keep parsed snapshots in memory -- LEIE's 15 MB exclusion file, the
+    coverage bulk crosswalk -- and re-answer from them for minutes at a time.
+    Those answers used to carry ``cache_hit=false`` because the flag came
+    straight off the original :class:`FetchResult`, so the receipt's prose said
+    "served from an in-process cache" while its structured field said the
+    opposite. A machine reading the receipt believed the field.
+
+    ``retrieved_at``, ``sha256`` and ``status`` are preserved: they describe the
+    bytes, and the bytes did not change. ``revalidated_at`` defaults to whatever
+    upstream last confirmed -- pass a timestamp ONLY when upstream just
+    confirmed these bytes again, and pass ``None`` when it never has. A reuse
+    that contacted nobody must not advance it; that is the difference between
+    "still current as of a moment ago" and "still what we downloaded on Tuesday".
+    """
+    return replace(
+        result,
+        cache_hit=True,
+        revalidated_at=(
+            result.revalidated_at if revalidated_at is _KEEP else revalidated_at
+        ),
+    )
+
+
+def fetch(
+    url: str,
+    *,
+    fallback_url: str | None = None,
+    fallback_name: str | None = None,
+    params: Mapping[str, Any] | None = None,
+    headers: Mapping[str, str] | None = None,
+    timeout: float | None = None,
+    raise_for_status: bool = True,
+    max_bytes: int = DEFAULT_MAX_BYTES,
+    cache_scope: str = "",
+) -> FetchResult:
+    """Read ``url``, falling back to ``fallback_url`` only if the authority fails.
+
+    Raises :class:`SourceUnreachable` when neither location can be read, when
+    the status is not 2xx and ``raise_for_status`` is set, or -- as
+    :class:`ResponseTooLarge` -- when the body exceeds ``max_bytes`` decompressed.
+    Raise ``max_bytes`` at the call site for a route that legitimately reads a
+    large file; do not raise the default for one route's sake.
+
+    ``timeout`` defaults to the ambient policy (see :func:`request_timeout`),
+    then to ``HC_SOURCE_HTTP_TIMEOUT``, then to :data:`DEFAULT_TIMEOUT`. An
+    explicit argument always wins.
+    """
+    timeout = _resolve_timeout(timeout)
+    try:
+        return _read(url, params=params, headers=headers, timeout=timeout,
+                     raise_for_status=raise_for_status, max_bytes=max_bytes,
+                     cache_scope=cache_scope)
+    except SourceUnreachable as primary:
+        if not fallback_url:
+            raise
+        try:
+            result = _read(fallback_url, params=params, headers=headers, timeout=timeout,
+                           raise_for_status=raise_for_status, max_bytes=max_bytes,
+                           cache_scope=cache_scope)
+        except SourceUnreachable as secondary:
+            raise SourceUnreachable(
+                url,
+                f"authority failed ({primary.reason}) and fallback failed ({secondary.reason})",
+                status=primary.status,
+            ) from secondary
+        return FetchResult(
+            url=result.url,
+            status=result.status,
+            content=result.content,
+            sha256=result.sha256,
+            headers=result.headers,
+            fallback_used=True,
+            fallback_name=fallback_name or fallback_url,
+            # The mirror read is still the read that produced these bytes.
+            retrieved_at=result.retrieved_at,
+            cache_hit=result.cache_hit,
+            revalidated_at=result.revalidated_at,
+        )
+
+
+def fetch_json(url: str, **kwargs: Any) -> tuple[FetchResult, Any]:
+    """``fetch`` plus JSON parsing. Invalid JSON is an unreachable source."""
+    result = fetch(url, **kwargs)
+    return result, result.json()
+
+
+def _read(
+    url: str,
+    *,
+    params: Mapping[str, Any] | None,
+    headers: Mapping[str, str] | None,
+    timeout: float,
+    raise_for_status: bool,
+    max_bytes: int = DEFAULT_MAX_BYTES,
+    cache_scope: str = "",
+) -> FetchResult:
+    if urlparse(url).scheme == "file":
+        return _read_file(url, max_bytes=max_bytes)
+
+    # One header object is the source of truth for both the cache signature and
+    # the transport. HTTPX applies the case-insensitive replacement semantics
+    # that a plain dict.update cannot: ``user-agent`` replaces ``User-Agent``
+    # instead of inventing a second occurrence beside the default. It also
+    # preserves genuine repeated occurrences from an ``httpx.Headers`` caller.
+    request_headers = httpx.Headers(
+        {"User-Agent": USER_AGENT, "Accept-Encoding": "gzip"}
+    )
+    request_headers.update(headers or {})
+
+    # Conditional revalidation. The cache never answers on its own: it supplies
+    # a validator, and only a 304 from upstream turns into a hit. So a cached
+    # answer is one the source confirmed a moment ago, and the receipt still
+    # reports when the BYTES were retrieved rather than when they were reused.
+    #
+    # The key covers the ACTUAL merged headers and this call's byte ceiling, not
+    # the caller's pre-merge object and not just the URL. The distinction is the
+    # contract: a signature more precise than the wire costs one cache miss; a
+    # signature less precise than the wire can return a wrong body after a 304.
+    #
+    # The cache's own validators are the only headers not present yet. They come
+    # out of the entry we are about to look up, so including them would make the
+    # key depend on its own answer and every revalidation would miss. A caller's
+    # conditional header is already in request_headers and remains key material.
+    signature = cache.request_signature(
+        url,
+        params=params,
+        scope=cache_scope,
+        headers=request_headers,
+        max_bytes=max_bytes,
+    )
+    key = cache.http_key(
+        url,
+        params=params,
+        scope=cache_scope,
+        headers=request_headers,
+        max_bytes=max_bytes,
+    )
+    entry = cache.read_http(key, signature=signature)
+    if entry is not None:
+        request_headers.update(entry.conditional_headers())
+
+    try:
+        with httpx.Client(timeout=timeout, follow_redirects=True) as client:
+            # Streamed, not buffered: `response.content` decompresses the whole
+            # body before anyone can object to its size, which is exactly how a
+            # 200 KB gzip bomb became 200 MB of resident memory.
+            with client.stream(
+                "GET", url, params=params, headers=request_headers
+            ) as response:
+                if response.status_code == 304 and entry is not None:
+                    # The ceiling is re-enforced on the way out of the cache, not
+                    # only on the way in. Otherwise a body stored when the call
+                    # allowed 30 MB would be replayed whole to a call that
+                    # declared 16 -- the limit would hold against upstream and
+                    # quietly not hold against ourselves.
+                    if len(entry.body) > max_bytes:
+                        raise ResponseTooLarge(url, max_bytes, status=response.status_code)
+                    confirmed = _utcnow()
+                    cache.touch_http(key, entry, confirmed)
+                    response.close()
+                    return FetchResult(
+                        url=entry.url,
+                        status=entry.status,
+                        content=entry.body,
+                        sha256=entry.sha256,
+                        headers=entry.headers,
+                        retrieved_at=entry.retrieved_at,
+                        cache_hit=True,
+                        revalidated_at=confirmed,
+                    )
+                if raise_for_status and not (200 <= response.status_code < 300):
+                    raise SourceUnreachable(
+                        url, "non-success status", status=response.status_code
+                    )
+
+                buffer, hasher = _read_bounded(response, url, max_bytes)
+                final_url = str(response.request.url)
+                status = response.status_code
+                response_headers = {k.lower(): v for k, v in response.headers.items()}
+    except httpx.HTTPError as exc:
+        raise SourceUnreachable(url, type(exc).__name__) from exc
+
+    retrieved_at = _utcnow()
+    result = FetchResult(
+        url=final_url,
+        status=status,
+        content=bytes(buffer),
+        sha256=hasher.hexdigest(),
+        headers=response_headers,
+        retrieved_at=retrieved_at,
+        revalidated_at=retrieved_at,
+    )
+    if 200 <= status < 300:
+        # write_http decides what is storable -- a 206 is not, whatever the key
+        # says, and neither is a 2xx without a validator.
+        cache.write_http(
+            key,
+            cache.CachedResponse(
+                url=result.url,
+                status=result.status,
+                body=result.content,
+                sha256=result.sha256,
+                headers=result.headers,
+                retrieved_at=retrieved_at,
+                revalidated_at=retrieved_at,
+                request_signature=signature,
+            ),
+        )
+    return result
+
+
+def _read_bounded(
+    response: httpx.Response, url: str, max_bytes: int
+) -> tuple[bytearray, "hashlib._Hash"]:
+    """Stream and inflate a response without ever holding more than the ceiling.
+
+    We iterate ``iter_raw`` and run the decompressor ourselves rather than using
+    ``iter_bytes``. httpx's decoder calls ``zlib.decompress`` on a whole raw
+    chunk with no length bound, so a single 64 KiB chunk of a gzip bomb inflates
+    to tens of megabytes BEFORE any caller gets to object -- measured at 221 MB
+    peak RSS on the report's 200 KB / 200 MB bomb even with an outer 16 MiB cap.
+    Driving ``decompressobj.decompress(data, max_length)`` ourselves makes the
+    ceiling real: the bomb now peaks near the ceiling instead of near the
+    payload.
+    """
+    encoding = response.headers.get("content-encoding", "").lower().strip()
+    if encoding not in _SUPPORTED_ENCODINGS:
+        raise SourceUnreachable(
+            url,
+            f"unsupported content-encoding {encoding!r}; SourceLock will not hand a "
+            "response to a decoder whose output size it cannot bound",
+            status=response.status_code,
+        )
+    if encoding in ("gzip", "x-gzip"):
+        decoder = zlib.decompressobj(16 + zlib.MAX_WBITS)
+    elif encoding == "deflate":
+        decoder = zlib.decompressobj()
+    else:
+        decoder = None
+
+    buffer = bytearray()
+    hasher = hashlib.sha256()
+    wire = 0
+
+    def _absorb(chunk: bytes) -> None:
+        if len(buffer) + len(chunk) > max_bytes:
+            raise ResponseTooLarge(url, max_bytes, status=response.status_code)
+        buffer.extend(chunk)
+        hasher.update(chunk)
+
+    for raw in response.iter_raw(_CHUNK):
+        wire += len(raw)
+        # A compressed body bigger than the ceiling is over the ceiling too,
+        # whatever it inflates to.
+        if wire > max_bytes:
+            raise ResponseTooLarge(url, max_bytes, status=response.status_code)
+        if decoder is None:
+            _absorb(raw)
+            continue
+        pending = raw
+        while pending:
+            # +1 so that overshooting the ceiling is observable rather than
+            # silently clipped to exactly the limit.
+            try:
+                _absorb(decoder.decompress(pending, max_bytes - len(buffer) + 1))
+            except zlib.error as exc:
+                raise SourceUnreachable(
+                    url, f"malformed {encoding} body: {type(exc).__name__}",
+                    status=response.status_code,
+                ) from exc
+            pending = decoder.unconsumed_tail
+
+    if decoder is not None:
+        try:
+            _absorb(decoder.flush())
+        except zlib.error as exc:
+            raise SourceUnreachable(
+                url, f"malformed {encoding} body: {type(exc).__name__}",
+                status=response.status_code,
+            ) from exc
+
+    return buffer, hasher
+
+
+def _read_file(url: str, *, max_bytes: int = DEFAULT_MAX_BYTES) -> FetchResult:
+    path = Path(unquote(urlparse(url).path))
+    try:
+        if path.stat().st_size > max_bytes:
+            raise ResponseTooLarge(url, max_bytes)
+        content = path.read_bytes()
+    except OSError as exc:
+        raise SourceUnreachable(url, type(exc).__name__) from exc
+    return FetchResult(
+        url=url,
+        status=None,
+        content=content,
+        sha256=hashlib.sha256(content).hexdigest(),
+        headers={},
+    )