mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/thin/stream.py

@@ -0,0 +1,501 @@
+"""The run-event stream a hosted ``mr-data watch`` consumes, and how it survives a cut.
+
+Studio publishes one run's append-only log as ``text/event-stream`` (ADR 0021). Three
+properties of that endpoint decide the shape of everything below:
+
+* **The SSE ``id`` is the log cursor.** Resume is exclusive and happens in Studio's own cursor
+  space, so a reconnect sends ``Last-Event-ID`` and never a sequence number it invented.
+* **The stream closes itself.** Studio sends ``event: end`` before the Cloud Run request timeout
+  rather than letting the platform truncate the response, and names why: ``run_terminal`` means
+  the run will never append again and reconnecting would be a lie; anything else is a deadline and
+  the client reconnects from its last cursor. A dropped connection with no ``end`` is the same
+  case as a deadline.
+* **Facts live only in the unsealed adjunct.** A durable run event is a sealed record -- type,
+  time, sequence, digest -- and carries no facts by design. What the worker measured rides beside
+  it in ``unsealed.payload``. A renderer that reads only the sealed half draws a correctly ordered,
+  correctly framed, entirely factless timeline, which is the failure this module exists to avoid.
+
+The frame NAME is upstream's two-way partition and is preserved rather than collapsed:
+``run_progress`` is an incremental tick, ``run_event`` is a bulk-append proof or a lifecycle
+transition. Five of the six producer command types collapse onto one durable event type, so the
+partition is not recoverable from the sealed record and a consumer that discards the name has
+destroyed information nothing downstream can rebuild.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import Callable, Iterable, Iterator, Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from mostlyright.data_harness.progress_events import PROGRESS_EVENT_FACTS, PROGRESS_SCHEMA_VERSION
+from mostlyright.data_harness.thin.runs import RUN_EVENTS_STREAM_PATH
+from mostlyright.data_harness.thin.session import StudioSession
+from mostlyright.data_harness.thin.transport import ThinLaneError, Transport, UrlLibTransport
+
+#: Upstream's frame names. The first two are the tick/proof partition; ``end`` is its
+#: end-of-stream notice.
+RUN_EVENT_FRAME = "run_event"
+RUN_PROGRESS_FRAME = "run_progress"
+RUN_END_FRAME = "end"
+
+#: The one ``end`` reason that means "do not reconnect".
+RUN_END_REASON_TERMINAL = "run_terminal"
+
+#: The four producer command types Studio frames as ``run_progress``. Used only as a fallback when
+#: a frame arrived with no name this build knows -- when upstream named the frame, that name wins.
+PROGRESS_PRODUCER_EVENT_TYPES = frozenset(
+    {"attempt_progress", "run_stage_reported", "source_probe_completed", "diagnostic_emitted"}
+)
+
+#: Statuses after which a run appends nothing further. Mirrors Studio's own terminal set.
+TERMINAL_RUN_STATUSES = frozenset({"released", "failed", "cancelled", "abandoned"})
+
+#: One SSE line may not exceed this. A stream is untrusted input like any other network body, and
+#: ``readline`` with no bound is how a hostile or broken producer turns a watch into an OOM.
+MAX_STREAM_LINE_BYTES = 1024 * 1024
+
+#: Studio closes at 3300 s and expects the client to come back; the socket read timeout has to
+#: exceed the 15 s heartbeat by enough that a slow hop is not mistaken for a cut.
+STREAM_READ_TIMEOUT_SECONDS = 90.0
+
+#: How long a reconnect waits, and the ceiling the doubling stops at.
+FIRST_RECONNECT_DELAY_SECONDS = 1.0
+MAX_RECONNECT_DELAY_SECONDS = 30.0
+
+#: Consecutive failures to re-establish the stream before the watch gives up and says so. A
+#: successful reconnect resets the count, so a long run that is cut hourly never exhausts it.
+MAX_CONSECUTIVE_RECONNECTS = 8
+
+
+@dataclass(frozen=True)
+class SseFrame:
+    """One dispatched Server-Sent-Events frame: its name, its data, and its id."""
+
+    name: str
+    data: str
+    event_id: str | None = None
+
+    def json(self) -> dict[str, Any]:
+        try:
+            parsed = json.loads(self.data)
+        except ValueError as error:
+            raise ThinLaneError(
+                "THIN_STREAM_FRAME_INVALID", "a stream frame carried a body that is not JSON"
+            ) from error
+        if not isinstance(parsed, dict):
+            raise ThinLaneError(
+                "THIN_STREAM_FRAME_INVALID", "a stream frame carried a body that is not an object"
+            )
+        return parsed
+
+
+def parse_sse(lines: Iterable[bytes]) -> Iterator[SseFrame]:
+    """Turn a byte-line stream into dispatched frames, dropping comments and heartbeats.
+
+    Implements only the part of the SSE grammar this endpoint uses: ``id``, ``event``, ``data``
+    (possibly repeated, joined with newlines) and comment lines, dispatched on a blank line. A
+    ``retry:`` field is read and discarded -- the reconnect policy here is the client's, and a
+    server-supplied delay is advice this lane does not need. A frame with no ``event:`` defaults to
+    ``message`` per the specification; Studio always names its frames, so such a frame is a signal
+    that something between us rewrote the stream, and it is passed through under that name rather
+    than being guessed at.
+    """
+
+    name: str | None = None
+    event_id: str | None = None
+    data: list[str] = []
+    for raw in lines:
+        if len(raw) > MAX_STREAM_LINE_BYTES:
+            raise ThinLaneError(
+                "THIN_STREAM_LINE_TOO_LONG", "a stream line exceeded its byte ceiling"
+            )
+        line = raw.rstrip(b"\n").rstrip(b"\r").decode("utf-8", "replace")
+        if not line:
+            if name is not None or data:
+                yield SseFrame(name=name or "message", data="\n".join(data), event_id=event_id)
+            name, data = None, []
+            continue
+        if line.startswith(":"):
+            continue
+        field_name, _, value = line.partition(":")
+        if value.startswith(" "):
+            value = value[1:]
+        if field_name == "event":
+            name = value
+        elif field_name == "data":
+            data.append(value)
+        elif field_name == "id" and "\x00" not in value:
+            event_id = value
+    if name is not None or data:
+        yield SseFrame(name=name or "message", data="\n".join(data), event_id=event_id)
+
+
+def frame_name_for(frame: Mapping[str, Any], upstream_name: str) -> str:
+    """The tick/proof partition for one event body.
+
+    Precedence: upstream's own name when it is one of the two the partition defines, then the
+    producer command type in the adjunct applying Studio's rule, then ``run_event`` -- the honest
+    answer for a record with neither, because a lifecycle transition is not a tick.
+    """
+
+    if upstream_name in {RUN_EVENT_FRAME, RUN_PROGRESS_FRAME}:
+        return upstream_name
+    unsealed = frame.get("unsealed")
+    if isinstance(unsealed, Mapping):
+        producer_event_type = unsealed.get("producer_event_type")
+        if producer_event_type in PROGRESS_PRODUCER_EVENT_TYPES:
+            return RUN_PROGRESS_FRAME
+    return RUN_EVENT_FRAME
+
+
+@dataclass
+class RunProgress:
+    """What one watch has learned so far: the last cursor, the stage rail, and the last facts."""
+
+    last_event_id: str | None = None
+    last_sequence: int = 0
+    events_seen: int = 0
+    progress_seen: int = 0
+    stage: str | None = None
+    stage_index: int | None = None
+    stage_total: int | None = None
+    status: str | None = None
+    end_reason: str | None = None
+    reconnects: int = 0
+    #: The follow stopped because the caller said so -- its observer saw the frame it was waiting
+    #: for, or its ``stop`` check answered yes between frames. Distinct from ``end_reason``, which
+    #: is what STUDIO said: a follow halted here left the stream open on Studio's side, and the
+    #: run is still appending.
+    halted: bool = False
+    #: The follow stopped because the caller's deadline passed first. The run is unaffected and the
+    #: frame being waited for may still arrive; a caller reads the resource itself to find out.
+    expired: bool = False
+    lines: list[str] = field(default_factory=list)
+
+    def to_receipt(self) -> dict[str, Any]:
+        return {
+            "last_event_id": self.last_event_id,
+            "last_sequence": self.last_sequence,
+            "events_seen": self.events_seen,
+            "progress_seen": self.progress_seen,
+            "stage": self.stage,
+            "stage_index": self.stage_index,
+            "stage_total": self.stage_total,
+            "status": self.status,
+            "end_reason": self.end_reason,
+            "reconnects": self.reconnects,
+            "halted": self.halted,
+            "expired": self.expired,
+        }
+
+
+def progress_facts(frame: Mapping[str, Any]) -> tuple[str, dict[str, Any]] | None:
+    """The progress event name and its facts, from the unsealed adjunct, or ``None``.
+
+    ⚠ NEVER EVIDENCE. ``payload_digest`` on a record whose adjunct carries a payload is the content
+    address of THAT payload: it says the facts were not rewritten in transit and says nothing about
+    the Build. Nothing here is verified against a candidate, and nothing here reaches a receipt
+    that claims to be attestation.
+
+    An unknown event name is rendered rather than dropped -- the vocabulary is the harness's and it
+    grows, and a client that has not been taught a name still knows the facts came with it.
+    """
+
+    unsealed = frame.get("unsealed")
+    if not isinstance(unsealed, Mapping):
+        return None
+    payload = unsealed.get("payload")
+    if not isinstance(payload, Mapping):
+        return None
+    if payload.get("schema_version") != PROGRESS_SCHEMA_VERSION:
+        return None
+    name = payload.get("event")
+    facts = payload.get("facts")
+    if not isinstance(name, str) or not isinstance(facts, Mapping):
+        return None
+    known = PROGRESS_EVENT_FACTS.get(name)
+    ordered = known if known is not None else tuple(sorted(str(key) for key in facts))
+    return name, {key: facts[key] for key in ordered if key in facts}
+
+
+def render_frame(frame: Mapping[str, Any], upstream_name: str) -> str:
+    """One timeline line for one stream frame.
+
+    A tick renders its facts; a proof renders its durable event type. Both carry the sequence
+    number, which is the only ordering authority the log has and the thing a person reads back to
+    us when something looks wrong.
+    """
+
+    sequence = frame.get("sequence")
+    marker = f"{sequence:>5}" if isinstance(sequence, int) else "    ?"
+    named = progress_facts(frame)
+    if frame_name_for(frame, upstream_name) == RUN_PROGRESS_FRAME and named is not None:
+        name, facts = named
+        rendered = " ".join(f"{key}={_scalar(value)}" for key, value in facts.items())
+        return f"{marker}  {name}{'  ' + rendered if rendered else ''}"
+    event_type = frame.get("event_type")
+    return f"{marker}  {event_type if isinstance(event_type, str) else 'run_event'}"
+
+
+def _scalar(value: Any) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, int | float | str):
+        return str(value)
+    return json.dumps(value, separators=(",", ":"), sort_keys=True)
+
+
+class RunEventStream:
+    """Replay-then-tail one run's event log, reconnecting across Cloud Run stream cuts."""
+
+    def __init__(
+        self,
+        session: StudioSession,
+        run_id: str,
+        *,
+        transport: Transport | None = None,
+        sleep: Callable[[float], None] = time.sleep,
+        max_reconnects: int = MAX_CONSECUTIVE_RECONNECTS,
+    ) -> None:
+        self._session = session
+        self._run_id = run_id
+        self._transport = transport or UrlLibTransport()
+        self._sleep = sleep
+        self._max_reconnects = max_reconnects
+
+    def follow(
+        self,
+        *,
+        from_seq: int = 0,
+        last_event_id: str | None = None,
+        on_line: Callable[[str], None] | None = None,
+        observe: Callable[[Mapping[str, Any], str], bool] | None = None,
+        stop: Callable[[], bool] | None = None,
+        deadline: float | None = None,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> RunProgress:
+        """Consume the stream to its terminal end, or until reconnects are exhausted.
+
+        Returns the accumulated :class:`RunProgress` in every case that is not a refusal, so a
+        caller can report how far it got even when the run outlived the watch.
+
+        Three optional ways to stop before the run does, all written for a caller that is waiting
+        for ONE record on a run that will keep appending -- a probe settling on a research
+        session's run, whose stream only ends when the session closes. ``observe`` sees every new
+        frame body with its frame name, after it is folded into ``progress``, and answers whether
+        the follow is over. ``stop`` is asked between frames -- after every frame boundary and
+        every heartbeat on a quiet stream, and before every reconnect -- so a caller whose answer
+        may arrive somewhere OTHER than this stream can go and look there; it is never asked in
+        the middle of a frame. ``deadline`` is a ``clock()`` reading past which the follow gives
+        up. The receipt says what happened under ``halted`` (the first two) and ``expired`` (the
+        third), and none of them is an ``end_reason``: Studio did not end anything, the caller
+        stopped listening.
+        """
+
+        progress = RunProgress(last_event_id=last_event_id)
+        delay = FIRST_RECONNECT_DELAY_SECONDS
+        consecutive = 0
+        while True:
+            delivered = self._one_connection(
+                progress,
+                from_seq=from_seq,
+                on_line=on_line,
+                observe=observe,
+                stop=stop,
+                deadline=deadline,
+                clock=clock,
+            )
+            if progress.end_reason == RUN_END_REASON_TERMINAL:
+                return progress
+            if progress.status in TERMINAL_RUN_STATUSES:
+                return progress
+            if progress.halted:
+                return progress
+            if deadline is not None and clock() >= deadline:
+                progress.expired = True
+                return progress
+            if stop is not None and stop():
+                # Asked before the reconnect wait rather than after it: a connection that just
+                # closed is exactly the gap in which an answer can have landed elsewhere.
+                progress.halted = True
+                return progress
+            if delivered:
+                consecutive = 0
+                delay = FIRST_RECONNECT_DELAY_SECONDS
+            else:
+                consecutive += 1
+                if consecutive > self._max_reconnects:
+                    raise ThinLaneError(
+                        "THIN_STREAM_UNAVAILABLE",
+                        f"the run event stream could not be re-established after {consecutive} "
+                        "attempts; the run is unaffected and watch can be resumed",
+                    )
+            progress.reconnects += 1
+            self._sleep(delay)
+            delay = min(delay * 2, MAX_RECONNECT_DELAY_SECONDS)
+
+    def _one_connection(
+        self,
+        progress: RunProgress,
+        *,
+        from_seq: int,
+        on_line: Callable[[str], None] | None,
+        observe: Callable[[Mapping[str, Any], str], bool] | None = None,
+        stop: Callable[[], bool] | None = None,
+        deadline: float | None = None,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> bool:
+        """Hold one connection until it ends. Returns whether it delivered any frame."""
+
+        def _between_frames() -> bool:
+            # The deadline first, because it costs nothing; the caller's check second, because it
+            # may cost a round trip and a deadline that has passed makes the answer moot.
+            if deadline is not None and clock() >= deadline:
+                return True
+            if stop is not None and stop():
+                progress.halted = True
+                return True
+            return False
+
+        headers = {"Accept": "text/event-stream", **self._session.authorization()}
+        # Resume in Studio's own cursor space when we have one. `from_seq` is only ever the
+        # starting position of a watch that has not yet seen a frame.
+        if progress.last_event_id:
+            headers["Last-Event-ID"] = progress.last_event_id
+            query = ""
+        else:
+            query = f"?from_seq={int(from_seq)}" if from_seq else ""
+        url = (
+            f"{self._session.studio_base_url}"
+            f"{RUN_EVENTS_STREAM_PATH.format(run_id=self._run_id)}{query}"
+        )
+        delivered = False
+        try:
+            with self._transport.stream(url, headers, STREAM_READ_TIMEOUT_SECONDS) as response:
+                status = int(getattr(response, "status", 200))
+                if status in {401, 403}:
+                    raise ThinLaneError(
+                        "THIN_AUTHORIZATION_DENIED",
+                        "the Studio token does not carry authority to read this run's events",
+                    )
+                if status == 404:
+                    raise ThinLaneError("THIN_NOT_FOUND", "Studio has no run with that identifier")
+                if status != 200:
+                    raise ThinLaneError(
+                        "THIN_STREAM_REFUSED",
+                        f"Studio answered the event stream with HTTP {status}",
+                    )
+                for frame in parse_sse(_lines_until(response, _between_frames)):
+                    delivered = True
+                    if self._consume(frame, progress, on_line, observe):
+                        return delivered
+        except ThinLaneError:
+            raise
+        except Exception:
+            # A cut connection is the ordinary case this endpoint is designed around, not an
+            # error: Studio closes at its own deadline and every hop between us may close sooner.
+            # The caller reconnects from the last cursor.
+            return delivered
+        return delivered
+
+    def _consume(
+        self,
+        frame: SseFrame,
+        progress: RunProgress,
+        on_line: Callable[[str], None] | None,
+        observe: Callable[[Mapping[str, Any], str], bool] | None = None,
+    ) -> bool:
+        """Fold one frame into ``progress``. Returns whether this connection is over."""
+
+        if frame.name == RUN_END_FRAME:
+            body = frame.json()
+            reason = body.get("reason")
+            progress.end_reason = reason if isinstance(reason, str) else None
+            status = body.get("status")
+            if isinstance(status, str):
+                progress.status = status
+            trailing = body.get("last_event_id")
+            if isinstance(trailing, str) and trailing:
+                progress.last_event_id = trailing
+            return True
+        body = frame.json()
+        if frame.event_id:
+            progress.last_event_id = frame.event_id
+        sequence = body.get("sequence")
+        if isinstance(sequence, int):
+            if sequence <= progress.last_sequence:
+                # Replay is exclusive by contract, so a repeat means a retried delivery rather
+                # than a new record. Counting it twice would make the receipt lie.
+                return False
+            progress.last_sequence = sequence
+        progress.events_seen += 1
+        named = progress_facts(body)
+        if named is not None:
+            progress.progress_seen += 1
+            name, facts = named
+            if name in {"stage_started", "stage_completed"}:
+                progress.stage = facts.get("stage") if isinstance(facts.get("stage"), str) else None
+                progress.stage_index = _as_int(facts.get("index"))
+                progress.stage_total = _as_int(facts.get("total"))
+        line = render_frame(body, frame.name)
+        progress.lines.append(line)
+        if on_line is not None:
+            on_line(line)
+        if observe is not None and observe(body, frame.name):
+            progress.halted = True
+            return True
+        return False
+
+
+def _lines_until(response: Iterable[bytes], should_stop: Callable[[], bool]) -> Iterator[bytes]:
+    """The response's lines, stopping where the caller says rather than at the stream's end.
+
+    Asked per LINE rather than per frame on purpose. Studio writes a heartbeat comment at most
+    fifteen seconds apart on a quiet stream, and a comment is a line the frame parser drops without
+    dispatching anything -- so a check made only when a frame arrived would, on a quiet session,
+    be made at Studio's own stream deadline and not before. A line is the finest grain the socket
+    offers, and the heartbeat is what makes it arrive.
+
+    ⚠ NEVER BETWEEN A FRAME'S OWN LINES. A frame is ``id``, ``event`` and ``data`` lines closed by
+    a blank one, and stopping after the ``event`` line would hand the parser a frame with no body
+    -- which it refuses as a malformed frame, so a deadline would have reported itself as a broken
+    stream. The check is made only at a blank line, which closes a frame, or at a comment that
+    arrived while no frame was open.
+    """
+
+    mid_frame = False
+    for raw in response:
+        yield raw
+        line = raw.rstrip(b"\r\n")
+        if not line:
+            mid_frame = False
+        elif not line.startswith(b":"):
+            mid_frame = True
+        if not mid_frame and should_stop():
+            return
+
+
+def _as_int(value: Any) -> int | None:
+    return value if isinstance(value, int) and not isinstance(value, bool) else None
+
+
+__all__ = [
+    "MAX_CONSECUTIVE_RECONNECTS",
+    "MAX_STREAM_LINE_BYTES",
+    "PROGRESS_PRODUCER_EVENT_TYPES",
+    "RUN_END_FRAME",
+    "RUN_END_REASON_TERMINAL",
+    "RUN_EVENT_FRAME",
+    "RUN_PROGRESS_FRAME",
+    "TERMINAL_RUN_STATUSES",
+    "RunEventStream",
+    "RunProgress",
+    "SseFrame",
+    "frame_name_for",
+    "parse_sse",
+    "progress_facts",
+    "render_frame",
+]

mostlyright/data_harness/thin/transport.py

@@ -0,0 +1,187 @@
+"""One bounded HTTPS transport for the thin lane, and the origin check every URL passes first.
+
+``urllib`` rather than a client library, for the reason the rest of this repository already uses
+it: the thin profile must install with no third-party runtime dependency at all, so the transport
+that reaches Cloud and Studio has to come out of the standard library. The hardening is the same
+hardening ``hosted_deploy.UrlLibTokenExchangeTransport`` applies -- ambient proxies off, redirects
+off, a byte ceiling on every response, a request timeout -- because a thin client is not a reason
+to relax any of it.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import ssl
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from typing import Any, Protocol
+from urllib.error import HTTPError
+from urllib.parse import urlsplit
+from urllib.request import HTTPRedirectHandler, HTTPSHandler, ProxyHandler, Request, build_opener
+
+#: The same thirty seconds ``hosted_deploy.REQUEST_TIMEOUT_SECONDS`` gives an ordinary request. A
+#: held-open event stream is not an ordinary request and passes its own, much larger, timeout.
+REQUEST_TIMEOUT_SECONDS = 30.0
+
+#: A JSON answer from Cloud or Studio that exceeds this is refused unread. Every response this
+#: lane parses is a small object; a signed download session is the largest of them.
+MAX_JSON_RESPONSE_BYTES = 1024 * 1024
+
+
+class ThinLaneError(RuntimeError):
+    """A typed, credential-safe refusal from the hosted lane.
+
+    ``code`` is a stable token a script may branch on and ``detail`` is one sentence a person
+    reads. Neither ever carries a credential: the raw ``mr_cli_`` key and the short-lived Studio
+    token are the two secrets this lane holds, and no refusal below is built from either.
+    """
+
+    def __init__(self, code: str, detail: str) -> None:
+        self.code = code
+        self.detail = detail
+        super().__init__(f"{code}: {detail}")
+
+
+class Transport(Protocol):
+    """The one seam every test substitutes. Returns status, body bytes, and lowercased headers."""
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        headers: Mapping[str, str],
+        body: bytes | None,
+        maximum: int,
+    ) -> tuple[int, bytes, Mapping[str, str]]: ...
+
+    def stream(
+        self,
+        url: str,
+        headers: Mapping[str, str],
+        timeout: float,
+    ) -> Any: ...
+
+
+class _NoRedirect(HTTPRedirectHandler):
+    """Refuse every redirect. A bearer token must not be replayed at an origin we did not choose."""
+
+    def redirect_request(
+        self,
+        request: Request,
+        file_pointer: Any,
+        code: int,
+        message: str,
+        headers: Any,
+        new_url: str,
+    ) -> None:
+        return None
+
+
+class UrlLibTransport:
+    """Bounded HTTPS with ambient proxies and redirects disabled."""
+
+    def __init__(self) -> None:
+        self._opener = build_opener(
+            ProxyHandler({}), HTTPSHandler(context=ssl.create_default_context()), _NoRedirect()
+        )
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        headers: Mapping[str, str],
+        body: bytes | None,
+        maximum: int,
+    ) -> tuple[int, bytes, Mapping[str, str]]:
+        request = Request(url, data=body, headers=dict(headers), method=method)
+        try:
+            response = self._opener.open(request, timeout=REQUEST_TIMEOUT_SECONDS)
+        except HTTPError as error:
+            response = error
+        with response:
+            raw = response.read(maximum + 1)
+            status = int(response.status)
+            response_headers = {key.lower(): value for key, value in response.headers.items()}
+        if len(raw) > maximum:
+            raise ThinLaneError(
+                "THIN_RESPONSE_TOO_LARGE", "the response exceeded the byte ceiling for its route"
+            )
+        return status, raw, response_headers
+
+    @contextmanager
+    def stream(self, url: str, headers: Mapping[str, str], timeout: float) -> Iterator[Any]:
+        """Open a long-lived response whose body the caller reads incrementally.
+
+        Separate from :meth:`request` because everything about it differs: no byte ceiling (the
+        caller bounds each line instead), a timeout measured in the length of a run rather than of
+        a request, and a body that must never be read whole.
+        """
+
+        request = Request(url, headers=dict(headers), method="GET")
+        try:
+            response = self._opener.open(request, timeout=timeout)
+        except HTTPError as error:
+            response = error
+        try:
+            yield response
+        finally:
+            response.close()
+
+
+def service_origin(value: str, label: str, *, allow_loopback_http: bool) -> str:
+    """Return ``value`` when it is a bare HTTPS origin, and refuse it otherwise.
+
+    Mirrors ``hosted_deploy._service_url``: no user information, no path, no query, no fragment,
+    and plain HTTP only for a loopback host, which is what a developer running Studio locally
+    needs and what nothing in production is allowed to be.
+    """
+
+    if not isinstance(value, str) or not 1 <= len(value) <= 2048 or value.endswith("/"):
+        raise ThinLaneError("THIN_CONFIG_INVALID", f"{label} is not a usable origin")
+    parsed = urlsplit(value)
+    loopback = False
+    if parsed.hostname:
+        try:
+            loopback = ipaddress.ip_address(parsed.hostname).is_loopback
+        except ValueError:
+            loopback = parsed.hostname == "localhost"
+    if (
+        parsed.scheme not in ({"https", "http"} if allow_loopback_http else {"https"})
+        or (parsed.scheme == "http" and not loopback)
+        or not parsed.hostname
+        or parsed.username is not None
+        or parsed.password is not None
+        or parsed.fragment
+        or parsed.path not in {"", "/"}
+        or parsed.query
+    ):
+        raise ThinLaneError(
+            "THIN_CONFIG_INVALID",
+            f"{label} must be an HTTPS origin without user information, path, query, or fragment",
+        )
+    return value
+
+
+def signed_transfer_url(value: str, label: str) -> str:
+    """Return ``value`` when it is an HTTPS URL a signed transfer may be performed against.
+
+    A signed download URL is the one URL in this lane that legitimately carries a path and a
+    query, so it cannot pass :func:`service_origin`. Everything else stays refused: no plain HTTP,
+    no embedded credentials, no fragment.
+    """
+
+    if not isinstance(value, str) or not 1 <= len(value) <= 8192:
+        raise ThinLaneError("THIN_RESPONSE_INVALID", f"{label} is not a usable URL")
+    parsed = urlsplit(value)
+    if (
+        parsed.scheme != "https"
+        or not parsed.hostname
+        or parsed.username is not None
+        or parsed.password is not None
+        or parsed.fragment
+    ):
+        raise ThinLaneError(
+            "THIN_RESPONSE_INVALID",
+            f"{label} must be an HTTPS URL without user information or a fragment",
+        )
+    return value