mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/hosted_crawler_fetch.py

@@ -0,0 +1,423 @@
+"""Trusted fixed public-source retrieval for the hosted crawler coordinator."""
+
+from __future__ import annotations
+
+import time
+from concurrent.futures import ThreadPoolExecutor
+from urllib.parse import urlsplit
+
+from mostlyright.data_harness.acquisition.http import (
+    AcquisitionLimiter,
+    ByteRange,
+    PinnedHttpsRetriever,
+    RetrievalHop,
+    RetrievalLimits,
+    RetrievedBytes,
+    StdlibPinnedTransport,
+    require_range_plan_within_budget,
+)
+from mostlyright.data_harness.acquisition.url_policy import (
+    AcquisitionSecurityError,
+    EgressPolicy,
+    SystemResolver,
+    canonical_public_hostname,
+)
+from mostlyright.data_harness.canonical import canonical_sha256, sha256_bytes
+from mostlyright.data_harness.hosted_crawler import (
+    HostedCrawlerError,
+    _public_expected_media_types,
+    validate_crawler_request,
+)
+from mostlyright.data_harness.hosted_crawler_protocol import HostedCrawlerRequest
+
+_OPENLIGADB_LIMITER = AcquisitionLimiter(
+    max_concurrency=2,
+    min_interval_seconds=1.0,
+)
+_PUBLIC_HTTPS_LIMITER = AcquisitionLimiter(max_concurrency=2, min_interval_seconds=1.0)
+
+# The ranged bulk-fetch envelope for capacity-constrained public origins (EPA AirData is
+# the motivating case: a 60-80 MB archive on a single-IP Apache origin that advertises
+# ``Accept-Ranges: bytes`` and stalls whole-file streams mid-transfer under load).  Every
+# figure here is restated, in milliseconds, inside the ``ranged_fetch`` envelope of
+# ``PUBLIC_HTTPS_EGRESS_POLICY`` in hosted_crawler_protocol.py, and a test holds the two
+# spellings together: the attestation must never claim an envelope this path does not run.
+#
+# An object whose 0-0 probe names a total above the threshold is fetched as consecutive
+# ranges instead of one stream, so a mid-transfer stall costs one range and its retries
+# rather than the whole file.  Below the threshold the ranged machinery would only add
+# requests to a transfer the single-shot path already survives.
+RANGED_FETCH_THRESHOLD_BYTES = 32 * 1024 * 1024
+RANGE_FETCH_BYTES = 16 * 1024 * 1024
+RANGE_FETCH_PARALLELISM = 2
+RANGE_FETCH_ATTEMPTS = 4
+# Per-range stall detection is deliberately much tighter than the single-shot 120-second
+# read / 600-second total budget: a 16 MiB range that makes no progress for 30 seconds is
+# a stalled range worth retrying, not a slow origin worth waiting out.
+RANGE_FETCH_READ_TIMEOUT_SECONDS = 30.0
+RANGE_FETCH_TOTAL_TIMEOUT_SECONDS = 120.0
+# The aggregate wall-clock bound for the whole ranged fetch, probe included, equal to the
+# attested single-shot total (total_timeout_milliseconds = 600_000): splitting a fetch
+# into ranges must not quietly widen how long one acquisition may hold the origin.
+# Admission-bounded, not a hard wall: the deadline is checked before each range attempt
+# is admitted, and one attempt already admitted may then run to its own per-range total,
+# so the wall-clock worst case is this bound plus one RANGE_FETCH_TOTAL_TIMEOUT_SECONDS
+# attempt (~720 s).  The probe-failure fallback composes the bounded probe (at most the
+# per-range total) with one single-shot fetch (at most 600 s), so it shares the same
+# ~720 s worst case.
+RANGED_FETCH_DEADLINE_SECONDS = 600.0
+# Headroom over the exact range size so the transport's byte ceiling refuses a response
+# that overruns its ordered span without refusing the span itself.
+RANGE_FETCH_RESPONSE_SLACK_BYTES = 64 * 1024
+# The only per-range failures worth a fresh attempt: a peer that stalled or disconnected
+# (TRANSPORT_FAILURE) and a range that timed out its own per-range budget (TOTAL_TIMEOUT).
+# Every other refusal is a policy or integrity verdict that a retry cannot change and
+# must stay loud.
+_RETRYABLE_RANGE_FAILURE_CODES = frozenset({"TRANSPORT_FAILURE", "TOTAL_TIMEOUT"})
+
+
+def production_retriever() -> PinnedHttpsRetriever:
+    """Return the single reviewed OpenLigaDB HTTPS retriever."""
+
+    return PinnedHttpsRetriever(
+        resolver=SystemResolver(),
+        egress_policy=EgressPolicy(
+            allowed_hostnames=("api.openligadb.de",),
+            allowed_ports=frozenset({443}),
+        ),
+        transport=StdlibPinnedTransport(),
+        limits=RetrievalLimits(
+            # The reviewed OpenLigaDB policy caps one response at 8 MiB; its egress-policy
+            # attestation states exactly this value and does not move with the public-HTTPS
+            # ceiling in MAX_RAW_RESPONSE_BYTES.
+            max_response_bytes=8 * 1024 * 1024,
+            max_redirects=0,
+            max_requests=1,
+            connect_timeout_seconds=5.0,
+            read_timeout_seconds=15.0,
+            total_timeout_seconds=20.0,
+            max_concurrency=2,
+            min_interval_seconds=1.0,
+            allowed_media_types=("application/json",),
+            user_agent="MostlyRightDataHarness-OpenLigaDB/1.0",
+        ),
+        limiter=_OPENLIGADB_LIMITER,
+    )
+
+
+def _public_https_retriever(
+    request: HostedCrawlerRequest,
+    hostname: str,
+    *,
+    max_response_bytes: int,
+    read_timeout_seconds: float,
+    total_timeout_seconds: float,
+) -> PinnedHttpsRetriever:
+    """The one reviewed public-HTTPS retriever construction.
+
+    The single-shot fetch and the per-range fetch share every term of this construction --
+    resolver, egress pin, transport, redirect and request budgets, media-type allowlist,
+    limiter -- and differ only in the three explicit caps a caller names: the byte ceiling
+    and the two stall timeouts.  Stated once so the ranged path cannot drift into a second,
+    subtly different transport policy.
+    """
+
+    return PinnedHttpsRetriever(
+        resolver=SystemResolver(),
+        egress_policy=EgressPolicy(
+            allowed_hostnames=(hostname,),
+            allowed_ports=frozenset({443}),
+        ),
+        transport=StdlibPinnedTransport(),
+        limits=RetrievalLimits(
+            max_response_bytes=max_response_bytes,
+            max_redirects=3,
+            max_requests=4,
+            connect_timeout_seconds=5.0,
+            read_timeout_seconds=read_timeout_seconds,
+            total_timeout_seconds=total_timeout_seconds,
+            max_concurrency=2,
+            min_interval_seconds=1.0,
+            allowed_media_types=_public_expected_media_types(request.query),
+            user_agent="MostlyRightDataHarness-PublicHttps/1.0",
+        ),
+        limiter=_PUBLIC_HTTPS_LIMITER,
+    )
+
+
+def fetch_crawler_source(
+    request: HostedCrawlerRequest,
+    *,
+    expected_egress_policy_attestation: str,
+    retriever: PinnedHttpsRetriever | None = None,
+) -> RetrievedBytes:
+    """Fetch one exact reviewed URL for the crawler child to normalize.
+
+    The retrieved bytes are framed by encode_normalization_input and handed to the
+    mr-data-crawler child, which hosted_crawler_job spawns under a seccomp filter that
+    denies socket and connect. That child is not acquisition.sandbox.CrawlerSandbox.
+    """
+
+    coordinate = validate_crawler_request(
+        request,
+        expected_egress_policy_attestation=expected_egress_policy_attestation,
+    )
+    if request.adapter_id == "public.https":
+        requested_url = request.query["url"]
+        hostname = canonical_public_hostname(urlsplit(requested_url))
+        retrieved = _retrieve_public_https(
+            request,
+            requested_url=requested_url,
+            hostname=hostname,
+            injected_retriever=retriever,
+        )
+        if (
+            retrieved.source_url != requested_url
+            or canonical_public_hostname(urlsplit(retrieved.final_url)) != hostname
+        ):
+            raise HostedCrawlerError(
+                "CRAWLER_ENDPOINT_MISMATCH",
+                "retrieval changed the reviewed public HTTPS source authority",
+            )
+        return retrieved
+    league, season = coordinate
+    requested_url = f"https://api.openligadb.de/getmatchdata/{league}/{season}"
+    retrieved = (retriever or production_retriever()).retrieve(requested_url)
+    if retrieved.source_url != requested_url or retrieved.final_url != requested_url:
+        raise HostedCrawlerError(
+            "CRAWLER_ENDPOINT_MISMATCH",
+            "retrieval changed the exact reviewed OpenLigaDB endpoint",
+        )
+    if retrieved.media_type != "application/json":
+        raise HostedCrawlerError(
+            "CRAWLER_MEDIA_TYPE_MISMATCH",
+            "retrieval changed the exact reviewed OpenLigaDB media type",
+        )
+    return retrieved
+
+
+def _retrieve_public_https(
+    request: HostedCrawlerRequest,
+    *,
+    requested_url: str,
+    hostname: str,
+    injected_retriever: PinnedHttpsRetriever | None,
+) -> RetrievedBytes:
+    """Retrieve one public HTTPS source, ranged when the origin supports it and size warrants.
+
+    Look-then-lock, applied to bulk transfer: the existing one-byte presence-style probe
+    asks the origin whether it answers ranges and how large the object is.  The probe runs
+    under the ranged-tight retriever construction, not the single-shot one, so an origin
+    that ignores the range and streams its whole body costs at most one range-sized
+    response before the refusal surfaces, never the whole object.  Only an origin that
+    answered 206 with a parseable complete size above RANGED_FETCH_THRESHOLD_BYTES *and*
+    named at least one validator (ETag or Last-Modified) on the probe takes the ranged
+    strategy: without a validator there is no way to prove every range came from the same
+    edition of the object, so a validator-less origin cannot be ranged safely and keeps
+    the single-shot path.  Every other outcome -- a 200 that ignored the range, a refused
+    or unparseable probe, an unknown ``*`` total, an object at or below the threshold --
+    also falls back to the exact single-shot path this module always had.
+
+    Whatever the probe transferred is never dropped from the sealed evidence: a fallback
+    result carries the probe's measured hops -- from the probe itself when it succeeded,
+    or from the typed refusal's own hop evidence when it did not -- ahead of the
+    single-shot hops, with the single-shot 200 as the terminal hop, so the composed
+    ``total_response_body_size_bytes`` accounts every response byte the origin shipped.
+    """
+
+    max_source_bytes = request.query["limits"]["max_source_bytes"]
+    range_retriever = injected_retriever or _public_https_retriever(
+        request,
+        hostname,
+        max_response_bytes=RANGE_FETCH_BYTES + RANGE_FETCH_RESPONSE_SLACK_BYTES,
+        read_timeout_seconds=RANGE_FETCH_READ_TIMEOUT_SECONDS,
+        total_timeout_seconds=RANGE_FETCH_TOTAL_TIMEOUT_SECONDS,
+    )
+
+    def single_shot(probe_hops: tuple[RetrievalHop, ...]) -> RetrievedBytes:
+        retriever = injected_retriever or _public_https_retriever(
+            request,
+            hostname,
+            max_response_bytes=max_source_bytes,
+            read_timeout_seconds=120.0,
+            total_timeout_seconds=600.0,
+        )
+        return _seal_probe_evidence(probe_hops, retriever.retrieve(requested_url))
+
+    started = float(time.monotonic())
+    # A transport-class probe failure means an unlucky connection, not a
+    # range-incapable origin: retrying the one-byte probe is far cheaper than
+    # surrendering to the single-shot path this feature exists to replace.
+    # Structural answers (a 200, a redirect on Range) still mean single-shot.
+    probe = None
+    probe_error: AcquisitionSecurityError | None = None
+    for _ in range(RANGE_FETCH_ATTEMPTS):
+        try:
+            probe = range_retriever.retrieve_presence_evidence(requested_url)
+            break
+        except AcquisitionSecurityError as error:
+            probe_error = error
+            if error.code not in _RETRYABLE_RANGE_FAILURE_CODES:
+                break
+            if float(time.monotonic()) - started >= RANGED_FETCH_DEADLINE_SECONDS:
+                break
+    if probe is None:
+        # The refusal's own hop evidence carries whatever the failed probe consumed
+        # (a range-ignoring origin's 200 body, for instance) into the sealed result.
+        return single_shot(tuple(getattr(probe_error, "hops", ())))
+    total = probe.complete_size_bytes
+    if total is None or total <= RANGED_FETCH_THRESHOLD_BYTES:
+        return single_shot(tuple(probe.hops))
+    if total > max_source_bytes:
+        # The single-shot path would stream up to the cap and then refuse; the probe already
+        # proved the object cannot fit, so refuse with the same code before moving a byte.
+        raise AcquisitionSecurityError(
+            "RESPONSE_LIMIT",
+            "probed complete object size exceeds the reviewed source byte budget",
+        )
+    probe_terminal = probe.hops[-1]
+    validators = (probe_terminal.etag, probe_terminal.last_modified)
+    if validators == (None, None):
+        return single_shot(tuple(probe.hops))
+    return _fetch_ranged(
+        requested_url,
+        hostname=hostname,
+        probe=probe,
+        total=total,
+        validators=validators,
+        retriever=range_retriever,
+        deadline=started + RANGED_FETCH_DEADLINE_SECONDS,
+    )
+
+
+def _seal_probe_evidence(
+    probe_hops: tuple[RetrievalHop, ...],
+    retrieved: RetrievedBytes,
+) -> RetrievedBytes:
+    """Seal the probe's measured hops ahead of a single-shot result's own evidence."""
+
+    if not probe_hops:
+        return retrieved
+    hops = probe_hops + retrieved.hops
+    return RetrievedBytes(
+        source_url=retrieved.source_url,
+        final_url=retrieved.final_url,
+        media_type=retrieved.media_type,
+        content=retrieved.content,
+        content_sha256=retrieved.content_sha256,
+        hops=hops,
+        transport_evidence_digest=canonical_sha256([hop.to_dict() for hop in hops]),
+        complete_size_bytes=retrieved.complete_size_bytes,
+        rate_observation=retrieved.rate_observation,
+    )
+
+
+def _fetch_ranged(
+    url: str,
+    *,
+    hostname: str,
+    probe: RetrievedBytes,
+    total: int,
+    validators: tuple[str | None, str | None],
+    retriever: PinnedHttpsRetriever,
+    deadline: float,
+) -> RetrievedBytes:
+    """Fetch [0, total) as consecutive ranges and compose one RetrievedBytes.
+
+    The transport layer already proves each delivered slice: exactly 206, a Content-Range
+    naming exactly the ordered span, and span, declared length and body agreeing byte for
+    byte (RANGE_MISMATCH / RANGE_LENGTH / CONTENT_RANGE in acquisition.http).  This
+    function adds only what a single-range call cannot know: every range must stay on the
+    reviewed hostname, answer with exactly the validators the probe pinned, name the same
+    complete size the probe named, and assemble in offset order to exactly that size.
+
+    The validator pin is what makes ranged assembly of a *mutable* origin sound.  Three
+    lengths agreeing proves each slice is internally consistent; it cannot prove two
+    slices came from the same edition of the object, and a same-size origin update
+    mid-fetch would otherwise assemble interleaved bytes of two editions into one sealed
+    digest.  Comparing the recorded validators, rather than sending ``If-Range``, keeps
+    the attested outgoing header set exactly as it is: the refusal is decided from the
+    same hop evidence the result seals, not from origin-side conditional behavior.
+
+    ``deadline`` is admission-bounded: it is checked before each attempt is admitted, and
+    an attempt already admitted may run to the retriever's own per-range total, so the
+    wall clock may overrun the deadline by at most one RANGE_FETCH_TOTAL_TIMEOUT_SECONDS.
+    """
+
+    ranges = tuple(
+        ByteRange(first_byte=first_byte, last_byte=min(first_byte + RANGE_FETCH_BYTES, total) - 1)
+        for first_byte in range(0, total, RANGE_FETCH_BYTES)
+    )
+    require_range_plan_within_budget(len(ranges), max_ranges=retriever.limits.max_ranges)
+
+    def fetch_one(byte_range: ByteRange) -> RetrievedBytes:
+        failure: AcquisitionSecurityError | None = None
+        for _attempt in range(RANGE_FETCH_ATTEMPTS):
+            if float(time.monotonic()) > deadline:
+                raise AcquisitionSecurityError(
+                    "TOTAL_TIMEOUT",
+                    "ranged retrieval exceeded its aggregate time budget",
+                )
+            try:
+                piece = retriever.retrieve(url, byte_range=byte_range)
+            except AcquisitionSecurityError as error:
+                if error.code not in _RETRYABLE_RANGE_FAILURE_CODES:
+                    raise
+                failure = error
+                continue
+            if canonical_public_hostname(urlsplit(piece.final_url)) != hostname:
+                raise HostedCrawlerError(
+                    "CRAWLER_ENDPOINT_MISMATCH",
+                    "retrieval changed the reviewed public HTTPS source authority",
+                )
+            terminal = piece.hops[-1]
+            if (terminal.etag, terminal.last_modified) != validators:
+                # An integrity verdict, not a stall: the origin changed the object under
+                # the fetch, and no retry of the same range can make two editions one.
+                raise AcquisitionSecurityError(
+                    "RANGE_VALIDATOR_MISMATCH",
+                    "range response validators differ from the probed object edition",
+                )
+            if piece.complete_size_bytes != total:
+                raise AcquisitionSecurityError(
+                    "CONTENT_RANGE",
+                    "range response names a complete size different from the probed object",
+                )
+            return piece
+        # A range that exhausted its attempts fails the whole fetch with the last
+        # transport failure exactly as a stalled single-shot fetch would fail.
+        assert failure is not None
+        raise failure
+
+    # Wrapper-side worker threads in the coordinator process; nothing here runs in the
+    # seccomp-filtered normalizer child.
+    with ThreadPoolExecutor(max_workers=RANGE_FETCH_PARALLELISM) as pool:
+        futures = [pool.submit(fetch_one, byte_range) for byte_range in ranges]
+        try:
+            pieces = [future.result() for future in futures]
+        except BaseException:
+            for future in futures:
+                future.cancel()
+            raise
+    content = b"".join(piece.content for piece in pieces)
+    if len(content) != total:
+        raise AcquisitionSecurityError(
+            "RANGE_LENGTH",
+            "assembled ranged content does not match the probed complete size",
+        )
+    # One composed retrieval, shaped exactly like a single-shot RetrievedBytes so every
+    # downstream consumer (normalization framing, receipts, sealing, CLI validation) is
+    # untouched.  Identity fields come from the probe and the first range; the evidence is
+    # the probe hop followed by every per-range hop in offset order, and the digest is the
+    # same canonical hash over that exact hop list that RetrievedBytes itself re-enforces.
+    hops = tuple(probe.hops) + tuple(hop for piece in pieces for hop in piece.hops)
+    return RetrievedBytes(
+        source_url=url,
+        final_url=probe.final_url,
+        media_type=pieces[0].media_type,
+        content=content,
+        content_sha256=sha256_bytes(content),
+        hops=hops,
+        transport_evidence_digest=canonical_sha256([hop.to_dict() for hop in hops]),
+        complete_size_bytes=total,
+    )