mostlyright-data 0.17.0__tar.gz → 0.17.1__tar.gz

{mostlyright_data-0.17.0 → mostlyright_data-0.17.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.17.0
+Version: 0.17.1
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/

{mostlyright_data-0.17.0 → mostlyright_data-0.17.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.17.0"
+version = "0.17.1"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
 requires-python = ">=3.13,<3.14"

{mostlyright_data-0.17.0 → mostlyright_data-0.17.1}/src/mostlyright/data_harness/acquisition/http.py

@@ -6,6 +6,7 @@ import contextlib
 import errno
 import hashlib
 import http.client
+import ipaddress
 import json
 import math
 import os
@@ -1127,6 +1128,10 @@ class TransportResponse:
     connected_peer: str
     tls_verified: bool
     hostname_verified: bool
+    proxy_endpoint_id: str | None = None
+    proxy_config_digest: str | None = None
+    gateway_peer: str | None = None
+    tunnel_target: str | None = None
 
     def header(self, name: str) -> str | None:
         wanted = name.lower()
@@ -1171,6 +1176,10 @@ class StreamedTransportResponse:
     connected_peer: str
     tls_verified: bool
     hostname_verified: bool
+    proxy_endpoint_id: str | None = None
+    proxy_config_digest: str | None = None
+    gateway_peer: str | None = None
+    tunnel_target: str | None = None
 
     def header(self, name: str) -> str | None:
         return TransportResponse(
@@ -1183,6 +1192,85 @@ class StreamedTransportResponse:
         ).header(name)
 
 
+@dataclass(frozen=True)
+class ConnectProxyMember:
+    """One trusted internal CONNECT gateway selected for a public acquisition."""
+
+    endpoint_id: str
+    host: str
+    port: int
+
+    def __post_init__(self) -> None:
+        if (
+            not isinstance(self.endpoint_id, str)
+            or re.fullmatch(r"[a-z][a-z0-9-]{0,63}", self.endpoint_id) is None
+        ):
+            raise AcquisitionSecurityError("CONNECT_PROXY_CONFIG", "proxy endpoint id is invalid")
+        if not isinstance(self.host, str):
+            raise AcquisitionSecurityError(
+                "CONNECT_PROXY_CONFIG", "proxy host must be an IP literal"
+            )
+        try:
+            address = ipaddress.ip_address(self.host)
+        except ValueError:
+            raise AcquisitionSecurityError(
+                "CONNECT_PROXY_CONFIG", "proxy host must be an IP literal"
+            ) from None
+        private_networks = (
+            ipaddress.ip_network("10.0.0.0/8"),
+            ipaddress.ip_network("172.16.0.0/12"),
+            ipaddress.ip_network("192.168.0.0/16"),
+        )
+        if address.version != 4 or not any(address in network for network in private_networks):
+            raise AcquisitionSecurityError(
+                "CONNECT_PROXY_CONFIG", "proxy host must be a private IPv4 address"
+            )
+        if type(self.port) is not int or self.port != 3128:
+            raise AcquisitionSecurityError("CONNECT_PROXY_CONFIG", "proxy port must be 3128")
+
+
+@dataclass(frozen=True)
+class ConnectProxyPool:
+    """Closed, trusted runtime configuration for the V4 public-fetch gateway pool."""
+
+    pool_id: str
+    members: tuple[ConnectProxyMember, ...]
+    config_digest: str
+
+    @classmethod
+    def from_mapping(cls, value: Mapping[str, object]) -> ConnectProxyPool:
+        expected = {"schema_version", "pool_id", "members", "selection"}
+        if (
+            set(value) != expected
+            or value.get("schema_version") != "mostlyright-fetch-egress-pool.v1"
+        ):
+            raise AcquisitionSecurityError(
+                "CONNECT_PROXY_CONFIG", "proxy pool configuration is invalid"
+            )
+        if (
+            value.get("pool_id") != "fetch-egress-v1"
+            or value.get("selection") != "per-attempt-sticky"
+        ):
+            raise AcquisitionSecurityError(
+                "CONNECT_PROXY_CONFIG", "proxy pool identity or selection is invalid"
+            )
+        raw_members = value.get("members")
+        if not isinstance(raw_members, list) or not 1 <= len(raw_members) <= 16:
+            raise AcquisitionSecurityError("CONNECT_PROXY_CONFIG", "proxy pool members are invalid")
+        members: list[ConnectProxyMember] = []
+        for raw in raw_members:
+            if not isinstance(raw, dict) or set(raw) != {"id", "host", "port"}:
+                raise AcquisitionSecurityError("CONNECT_PROXY_CONFIG", "proxy member is invalid")
+            members.append(ConnectProxyMember(raw["id"], raw["host"], raw["port"]))
+        if len({member.endpoint_id for member in members}) != len(members):
+            raise AcquisitionSecurityError(
+                "CONNECT_PROXY_CONFIG", "proxy member ids must be unique"
+            )
+        if len({(member.host, member.port) for member in members}) != len(members):
+            raise AcquisitionSecurityError("CONNECT_PROXY_CONFIG", "proxy endpoints must be unique")
+        return cls("fetch-egress-v1", tuple(members), canonical_sha256(dict(value)))
+
+
 class PinnedTransport(Protocol):
     def request(
         self,
@@ -1191,6 +1279,8 @@ class PinnedTransport(Protocol):
         approved_ip: str,
         headers: Mapping[str, str],
         limits: RetrievalLimits,
+        connect_proxy: ConnectProxyMember | None = None,
+        connect_proxy_config_digest: str | None = None,
         byte_range: ByteRange | None = None,
         method: str = "GET",
         body: bytes | None = None,
@@ -1348,12 +1438,16 @@ class RetrievalHop:
     # set to rederive ``resolution_digest`` without either repeating the request or receiving the
     # credential, so the pinned retriever retains it alongside (but not inside) ``to_dict``.
     approved_ips: tuple[str, ...] = ()
+    proxy_endpoint_id: str | None = None
+    proxy_config_digest: str | None = None
+    gateway_peer: str | None = None
+    tunnel_target: str | None = None
 
     def to_dict(self) -> dict[str, object]:
         # Both range keys are always present, ``None`` when the hop ordered no slice, so a hop
         # has exactly one shape rather than two and the digest below covers the same fields on
         # every acquisition.
-        return {
+        result: dict[str, object] = {
             "url": self.url,
             "resolution_digest": self.resolution_digest,
             "connected_peer": self.connected_peer,
@@ -1367,6 +1461,14 @@ class RetrievalHop:
             "last_modified": self.last_modified,
             "peer_attempts": [attempt.to_dict() for attempt in self.peer_attempts],
         }
+        if self.proxy_endpoint_id is not None:
+            result["connect_proxy"] = {
+                "endpoint_id": self.proxy_endpoint_id,
+                "config_digest": self.proxy_config_digest,
+                "gateway_peer": self.gateway_peer,
+                "tunnel_target": self.tunnel_target,
+            }
+        return result
 
     def authenticated_attestation_dict(self) -> dict[str, object]:
         """Return the complete secret-free facts proved by the pinned HTTPS boundary."""
@@ -1689,6 +1791,8 @@ class PinnedHttpsRetriever:
         egress_policy: EgressPolicy,
         transport: PinnedTransport,
         limits: RetrievalLimits,
+        connect_proxy: ConnectProxyMember | None = None,
+        connect_proxy_config_digest: str | None = None,
         limiter: AcquisitionLimiter | None = None,
         clock: Callable[[], float] = time.monotonic,
         sleep: Callable[[float], None] = time.sleep,
@@ -1707,6 +1811,12 @@ class PinnedHttpsRetriever:
         self._egress_policy = egress_policy
         self._transport = transport
         self._limits = limits
+        if (connect_proxy is None) != (connect_proxy_config_digest is None):
+            raise AcquisitionSecurityError(
+                "CONNECT_PROXY_CONFIG", "proxy member and digest must be paired"
+            )
+        self._connect_proxy = connect_proxy
+        self._connect_proxy_config_digest = connect_proxy_config_digest
         self._limiter = limiter or AcquisitionLimiter(
             max_concurrency=limits.max_concurrency,
             min_interval_seconds=limits.min_interval_seconds,
@@ -1794,6 +1904,8 @@ class PinnedHttpsRetriever:
             egress_policy=self._egress_policy,
             transport=self._transport,
             limits=effective,
+            connect_proxy=self._connect_proxy,
+            connect_proxy_config_digest=self._connect_proxy_config_digest,
             limiter=self._limiter,
             clock=self._clock,
         )
@@ -1877,7 +1989,8 @@ class PinnedHttpsRetriever:
             }
             response: StreamedTransportResponse | None = None
             attempts: list[PeerAttempt] = []
-            for ip in target.approved_ips:
+            peer_candidates = self._peer_candidates(target)
+            for ip in peer_candidates:
                 if requests >= self._limits.max_requests:
                     raise PeerAttemptError(
                         "REQUEST_LIMIT",
@@ -1917,7 +2030,7 @@ class PinnedHttpsRetriever:
                     )
                     if (
                         exc.code == "TRANSPORT_FAILURE"
-                        and ip != target.approved_ips[-1]
+                        and ip != peer_candidates[-1]
                         and aggregate < self._limits.max_aggregate_response_bytes
                     ):
                         # A failed peer may already have copied an unauthenticated prefix to
@@ -1966,7 +2079,7 @@ class PinnedHttpsRetriever:
                     tuple(attempts),
                     hops=tuple(hops),
                 )
-            peer = require_approved_peer(target, response.connected_peer)
+            peer = self._require_response_peer(target, response, ip)
             normalized = _validate_response_headers(response.headers)
             content_length = response.header("content-length")
             if content_length is not None and (
@@ -1999,6 +2112,10 @@ class PinnedHttpsRetriever:
                 last_modified=response.header("last-modified"),
                 peer_attempts=tuple(attempts),
                 approved_ips=target.approved_ips,
+                proxy_endpoint_id=response.proxy_endpoint_id,
+                proxy_config_digest=response.proxy_config_digest,
+                gateway_peer=response.gateway_peer,
+                tunnel_target=response.tunnel_target,
             )
             hops.append(hop)
             if response.status in REDIRECT_STATUSES:
@@ -2264,7 +2381,8 @@ class PinnedHttpsRetriever:
                 headers[name] = value
             peer_attempts: list[PeerAttempt] = []
             response: TransportResponse | None = None
-            for approved_ip in target.approved_ips:
+            peer_candidates = self._peer_candidates(target)
+            for approved_ip in peer_candidates:
                 if requests >= self._limits.max_requests:
                     raise PeerAttemptError(
                         "REQUEST_LIMIT",
@@ -2381,7 +2499,7 @@ class PinnedHttpsRetriever:
                     if (
                         failure_code == "TRANSPORT_FAILURE"
                         and not aggregate_exhausted
-                        and approved_ip != target.approved_ips[-1]
+                        and approved_ip != peer_candidates[-1]
                     ):
                         continue
                     if failure_code == "TRANSPORT_FAILURE" and aggregate_exhausted:
@@ -2451,7 +2569,7 @@ class PinnedHttpsRetriever:
                     hops,
                 )
             try:
-                peer = require_approved_peer(target, response.connected_peer)
+                peer = self._require_response_peer(target, response, approved_ip)
                 normalized_headers = _validate_response_headers(response.headers)
             except AcquisitionSecurityError as exc:
                 self._raise_peer_failure(exc.code, exc.detail, peer_attempts, hops)
@@ -2543,6 +2661,10 @@ class PinnedHttpsRetriever:
                         ),
                         peer_attempts=tuple(peer_attempts),
                         approved_ips=target.approved_ips,
+                        proxy_endpoint_id=response.proxy_endpoint_id,
+                        proxy_config_digest=response.proxy_config_digest,
+                        gateway_peer=response.gateway_peer,
+                        tunnel_target=response.tunnel_target,
                     )
                 )
                 raise NotModified(
@@ -2609,6 +2731,10 @@ class PinnedHttpsRetriever:
                 last_modified=self._response_header(response, "last-modified", peer_attempts, hops),
                 peer_attempts=tuple(peer_attempts),
                 approved_ips=target.approved_ips,
+                proxy_endpoint_id=response.proxy_endpoint_id,
+                proxy_config_digest=response.proxy_config_digest,
+                gateway_peer=response.gateway_peer,
+                tunnel_target=response.tunnel_target,
             )
             hops.append(hop)
             # The delivery check.  A slice is delivered only when the server answered 206,
@@ -2800,6 +2926,49 @@ class PinnedHttpsRetriever:
             total_timeout_seconds=remaining,
         )
 
+    def _require_response_peer(
+        self,
+        target: ValidatedTarget,
+        response: TransportResponse | StreamedTransportResponse,
+        approved_ip: str,
+    ) -> str:
+        """Bind direct peers normally and gateway tunnels to the selected trusted member."""
+
+        if self._connect_proxy is None:
+            if response.proxy_endpoint_id is not None:
+                raise AcquisitionSecurityError(
+                    "CONNECT_PROXY_EVIDENCE", "direct transport reported proxy evidence"
+                )
+            return require_approved_peer(target, response.connected_peer)
+        member = self._connect_proxy
+        if (
+            response.proxy_endpoint_id != member.endpoint_id
+            or response.proxy_config_digest != self._connect_proxy_config_digest
+            or response.gateway_peer != member.host
+            or response.connected_peer != member.host
+            or response.tunnel_target != approved_ip
+        ):
+            raise AcquisitionSecurityError(
+                "CONNECT_PROXY_EVIDENCE",
+                "gateway tunnel evidence does not bind the selected proxy and target",
+            )
+        return member.host
+
+    def _peer_candidates(self, target: ValidatedTarget) -> tuple[str, ...]:
+        """The pinned peers this concrete transport can actually tunnel to."""
+
+        if self._connect_proxy is None:
+            return target.approved_ips
+        candidates = tuple(
+            address for address in target.approved_ips if ipaddress.ip_address(address).version == 4
+        )
+        if not candidates:
+            raise AcquisitionSecurityError(
+                "CONNECT_PROXY_TARGET_FAMILY",
+                "the selected IPv4 gateway cannot tunnel an IPv6-only target",
+            )
+        return candidates
+
     @staticmethod
     def _raise_peer_failure(
         code: str,
@@ -2885,10 +3054,60 @@ class _PinnedHTTPSConnection(http.client.HTTPSConnection):
             raise
 
 
+class _PinnedProxyHTTPSConnection(http.client.HTTPSConnection):
+    """CONNECT through one private gateway to an exact resolver-approved IPv4 target."""
+
+    def __init__(
+        self,
+        *,
+        hostname: str,
+        approved_ip: str,
+        proxy: ConnectProxyMember,
+        timeout: float,
+        context: ssl.SSLContext,
+    ) -> None:
+        super().__init__(host=hostname, port=443, timeout=timeout, context=context)
+        self._approved_ip = approved_ip
+        self._proxy = proxy
+        self.gateway_peer: str | None = None
+
+    def connect(self) -> None:
+        raw_socket = socket.create_connection(
+            (self._proxy.host, self._proxy.port), timeout=self.timeout
+        )
+        try:
+            peer = ipaddress.ip_address(raw_socket.getpeername()[0]).compressed
+            if peer != self._proxy.host:
+                raise AcquisitionSecurityError(
+                    "CONNECT_PROXY_PEER", "gateway peer did not match selected proxy"
+                )
+            authority = f"{self._approved_ip}:443"
+            raw_socket.sendall(
+                f"CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n".encode("ascii")
+            )
+            response = http.client.HTTPResponse(raw_socket)
+            response.begin()
+            if response.status != 200:
+                raise AcquisitionSecurityError(
+                    "CONNECT_PROXY_STATUS", "gateway refused the pinned tunnel"
+                )
+            self.gateway_peer = peer
+            self.sock = self._context.wrap_socket(raw_socket, server_hostname=self.host)
+        except BaseException:
+            raw_socket.close()
+            raise
+
+
 class StdlibPinnedTransport:
     """Production transport with no environment-proxy integration or connection pooling."""
 
-    def __init__(self, *, ssl_context: ssl.SSLContext | None = None) -> None:
+    def __init__(
+        self,
+        *,
+        ssl_context: ssl.SSLContext | None = None,
+        connect_proxy: ConnectProxyMember | None = None,
+        connect_proxy_config_digest: str | None = None,
+    ) -> None:
         self._context = ssl_context or ssl.create_default_context()
         if (
             self._context.check_hostname is not True
@@ -2898,6 +3117,54 @@ class StdlibPinnedTransport:
                 "TLS_CONTEXT",
                 "TLS context must require certificate and hostname verification",
             )
+        if (connect_proxy is None) != (connect_proxy_config_digest is None):
+            raise AcquisitionSecurityError(
+                "CONNECT_PROXY_CONFIG", "proxy member and digest must be paired"
+            )
+        self._connect_proxy = connect_proxy
+        self._connect_proxy_config_digest = connect_proxy_config_digest
+
+    def _connection(
+        self, target: ValidatedTarget, approved_ip: str, timeout: float
+    ) -> http.client.HTTPSConnection:
+        if self._connect_proxy is None:
+            return _PinnedHTTPSConnection(
+                hostname=target.hostname,
+                approved_ip=approved_ip,
+                port=target.port,
+                timeout=timeout,
+                context=self._context,
+            )
+        return _PinnedProxyHTTPSConnection(
+            hostname=target.hostname,
+            approved_ip=approved_ip,
+            proxy=self._connect_proxy,
+            timeout=timeout,
+            context=self._context,
+        )
+
+    def _evidence(
+        self, connection: http.client.HTTPSConnection, approved_ip: str
+    ) -> tuple[str, str | None, str | None, str | None, str | None]:
+        if self._connect_proxy is None:
+            peer = getattr(connection, "connected_peer", None)
+            if peer is None:
+                raise AcquisitionSecurityError(
+                    "PEER_ADDRESS", "transport did not record its connected peer"
+                )
+            return peer, None, None, None, None
+        peer = getattr(connection, "gateway_peer", None)
+        if peer is None:
+            raise AcquisitionSecurityError(
+                "CONNECT_PROXY_PEER", "transport did not record its gateway peer"
+            )
+        return (
+            peer,
+            self._connect_proxy.endpoint_id,
+            self._connect_proxy_config_digest,
+            peer,
+            approved_ip,
+        )
 
     def request(
         self,
@@ -2914,12 +3181,8 @@ class StdlibPinnedTransport:
         if byte_range is not None:
             outgoing["Range"] = byte_range.header_value
         attempt_deadline = time.monotonic() + limits.total_timeout_seconds
-        connection = _PinnedHTTPSConnection(
-            hostname=target.hostname,
-            approved_ip=approved_ip,
-            port=target.port,
-            timeout=min(limits.connect_timeout_seconds, limits.total_timeout_seconds),
-            context=self._context,
+        connection = self._connection(
+            target, approved_ip, min(limits.connect_timeout_seconds, limits.total_timeout_seconds)
         )
         total = 0
         try:
@@ -2971,12 +3234,9 @@ class StdlibPinnedTransport:
                         "response exceeded the configured byte limit",
                     )
                 chunks.append(chunk)
-            peer = connection.connected_peer
-            if peer is None:
-                raise AcquisitionSecurityError(
-                    "PEER_ADDRESS",
-                    "transport did not record its connected peer",
-                )
+            peer, proxy_id, proxy_digest, gateway_peer, tunnel_target = self._evidence(
+                connection, approved_ip
+            )
             return TransportResponse(
                 status=response.status,
                 headers=raw_headers,
@@ -2984,6 +3244,10 @@ class StdlibPinnedTransport:
                 connected_peer=peer,
                 tls_verified=True,
                 hostname_verified=True,
+                proxy_endpoint_id=proxy_id,
+                proxy_config_digest=proxy_digest,
+                gateway_peer=gateway_peer,
+                tunnel_target=tunnel_target,
             )
         except ssl.SSLError as exc:
             raise AcquisitionSecurityError(
@@ -3030,12 +3294,8 @@ class StdlibPinnedTransport:
         carried by ``TransportFailureError`` for retry accounting.
         """
         deadline = time.monotonic() + limits.total_timeout_seconds
-        connection = _PinnedHTTPSConnection(
-            hostname=target.hostname,
-            approved_ip=approved_ip,
-            port=target.port,
-            timeout=min(limits.connect_timeout_seconds, limits.total_timeout_seconds),
-            context=self._context,
+        connection = self._connection(
+            target, approved_ip, min(limits.connect_timeout_seconds, limits.total_timeout_seconds)
         )
         total = 0
         digest = hashlib.sha256()
@@ -3079,13 +3339,21 @@ class StdlibPinnedTransport:
                     )
                 digest.update(chunk)
                 sink(chunk)
-            peer = connection.connected_peer
-            if peer is None:
-                raise AcquisitionSecurityError(
-                    "PEER_ADDRESS", "transport did not record its connected peer"
-                )
+            peer, proxy_id, proxy_digest, gateway_peer, tunnel_target = self._evidence(
+                connection, approved_ip
+            )
             return StreamedTransportResponse(
-                response.status, raw_headers, total, digest.hexdigest(), peer, True, True
+                response.status,
+                raw_headers,
+                total,
+                digest.hexdigest(),
+                peer,
+                True,
+                True,
+                proxy_id,
+                proxy_digest,
+                gateway_peer,
+                tunnel_target,
             )
         except http.client.IncompleteRead as exc:
             partial = exc.partial if isinstance(exc.partial, bytes) else bytes(exc.partial)