minimalcode 1.5.2__py3-none-any.whl

minimalcode/provider/transport.py

@@ -0,0 +1,241 @@
+"""HTTP request construction and retry policy for provider calls."""
+
+from __future__ import annotations
+
+import math
+import ssl
+import time
+from typing import Any
+
+import httpx2
+
+from .. import httpclient
+from ..errors import ProviderError
+from .common import iter_sse_data
+
+# The policy itself lives in ``minimalcode.httpclient`` so the web and vision
+# clients cannot drift from the protocol client. ``time`` stays imported here
+# because backoff sleeps are patched at this module by the retry tests.
+# Backoff runs 0.5, 1, 2, 4, then 8 seconds per attempt, so all 11 attempts span
+# roughly a minute: long enough to ride out a provider hiccup, short enough that
+# a dead endpoint still fails inside one agent turn.
+
+# A TLS session can be reset after the handshake but before the response headers
+# ("[SSL: UNEXPECTED_EOF_WHILE_READING]"), which httpcore surfaces as ReadError or
+# as a bare ssl.SSLError. One-shot handshakes (session create, challenge fetch) are
+# idempotent and cost nothing server-side, so they retry these; a streamed
+# generation does not, because retrying it could duplicate output and billing.
+HANDSHAKE_RETRY_ERRORS = (
+    ssl.SSLError,
+    httpx2.ReadError,
+    httpx2.WriteError,
+    httpx2.PoolTimeout,
+    ConnectionError,
+)
+
+
+class TransportMixin:
+    """Open a provider response with connection and HTTP-status retries."""
+
+    # Attributes supplied by OpenAICompatibleProvider.
+    _client: httpx2.Client
+    notify: Any
+    timeout_seconds: float
+
+    def _endpoint(self) -> str:
+        raise NotImplementedError
+
+    def _checked_lines(self, lines):
+        """Yield SSE lines while enforcing the whole-stream deadline."""
+        deadline = getattr(self, "_stream_deadline", None)
+        for line in lines:
+            if deadline is not None and time.monotonic() > deadline:
+                raise ProviderError(
+                    "provider stream exceeded timeout_seconds "
+                    f"({self.timeout_seconds:g}s)"
+                )
+            yield line
+
+    def _sse_events(self, lines):
+        """Yield one payload per SSE event, joined across `data:` lines."""
+        yield from iter_sse_data(self._checked_lines(lines))
+
+    def _sse_lines(self, response: httpx2.Response):
+        """Yield ``response.iter_lines()`` under the whole-stream deadline.
+
+        Protocols that read raw SSE lines rather than whole events still have to
+        inherit ``timeout_seconds``; calling ``iter_lines`` directly would let a
+        server that trickles one byte hold the turn open indefinitely.
+        """
+        yield from self._checked_lines(response.iter_lines())
+
+    def _read_body(self, response: httpx2.Response) -> bytes:
+        """Read a response body under the same whole-request deadline.
+
+        ``response.read()`` is bounded only by the per-socket read timeout, so a
+        server that keeps trickling a byte would hold a non-streaming request
+        open forever. Reading chunks and checking the deadline applies the
+        configured ``timeout_seconds`` to the whole body.
+        """
+        deadline = getattr(self, "_stream_deadline", None)
+        chunks: list[bytes] = []
+        try:
+            for chunk in response.iter_bytes():
+                if deadline is not None and time.monotonic() > deadline:
+                    raise ProviderError(
+                        "provider response exceeded timeout_seconds "
+                        f"({self.timeout_seconds:g}s)"
+                    )
+                chunks.append(chunk)
+        except (httpx2.HTTPError, OSError) as exc:
+            raise ProviderError(f"provider response read failed: {exc}") from exc
+        return b"".join(chunks)
+
+    def _post_json(self, url: str, payload: dict[str, Any]) -> bytes:
+        """POST a small idempotent JSON request and return its body.
+
+        Used for provider handshakes (session creation, challenge fetch) that sit
+        outside the single wire-protocol call. Routed through ``_open_url`` so they
+        inherit the connection retry policy, and given the extra TLS-reset
+        retryable set because a dropped handshake costs nothing to repeat.
+        """
+        response = self._open_url(url, payload, retry_extra=HANDSHAKE_RETRY_ERRORS)
+        try:
+            return self._read_body(response)
+        finally:
+            response.close()
+
+    def _open(self, payload: dict[str, Any]) -> httpx2.Response:
+        """POST ``payload`` to the protocol endpoint and return a response."""
+        return self._open_url(self._endpoint(), payload)
+
+    def _open_url(
+        self,
+        url: str,
+        payload: dict[str, Any],
+        *,
+        retry_extra: tuple[type[BaseException], ...] = (),
+    ) -> httpx2.Response:
+        """POST ``payload`` to ``url`` and return a successful response.
+
+        Retries cover connection failures that occurred before a response and
+        one 429/503 retry. Any other status is reported through ProviderError
+        with the bounded response body. ``retry_extra`` lets an idempotent
+        handshake treat TLS resets as retryable without loosening the policy for
+        a streamed generation.
+        """
+        # Keep the transport streamed even for a `stream: false` request: the
+        # parsers read either SSE lines or the whole body through `_read_body`,
+        # which is the only path that enforces the whole-generation deadline.
+        # Letting httpx2 read a non-streaming body internally would bypass it.
+        stream = True
+        last_error: ProviderError | None = None
+        # Retries share the request's time budget: without it a 1s timeout could
+        # still spend ~55s in connection backoff before giving up.
+        deadline = time.monotonic() + self.timeout_seconds
+        # Counts failures, not requests: the status retry below is charged outside
+        # this budget, so one call issues at most httpclient.CONNECTION_ATTEMPTS + 1 requests.
+        connection_failures = 0
+        status_retried = False
+        retryable_types = httpclient.RETRYABLE_CONNECTION_ERRORS + tuple(retry_extra)
+        while True:
+            try:
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    # The budget covers the attempts themselves, not only the
+                    # backoff: with a 20s connect timeout a second attempt could
+                    # overrun a 30s `timeout_seconds` and report much later.
+                    last_error = ProviderError(
+                        "provider request exceeded timeout_seconds "
+                        f"({self.timeout_seconds:g}s)"
+                    )
+                    break
+                request = self._client.build_request(
+                    "POST",
+                    url,
+                    json=payload,
+                    timeout=httpx2.Timeout(remaining, connect=min(20.0, remaining)),
+                )
+                response = self._client.send(request, stream=stream)
+            except (httpx2.HTTPError, httpx2.InvalidURL, ValueError, OSError) as exc:
+                last_error = ProviderError(f"provider network error: {exc}")
+                connection_failures += 1
+                if (
+                    connection_failures >= httpclient.CONNECTION_ATTEMPTS
+                    or not isinstance(exc, retryable_types)
+                ):
+                    break
+                delay = min(
+                    httpclient.connection_backoff(connection_failures - 1),
+                    deadline - time.monotonic(),
+                )
+                if delay <= 0:
+                    # The budget is spent; retrying now would exceed
+                    # `timeout_seconds` instead of reporting the failure.
+                    break
+                if self.notify is not None:
+                    self.notify(
+                        "[retry] provider did not respond; attempt "
+                        f"{connection_failures}/{httpclient.CONNECTION_ATTEMPTS - 1} "
+                        f"in {delay:g}s ({exc})"
+                    )
+                time.sleep(delay)
+                continue
+            if 200 <= response.status_code < 300:
+                # The per-read timeout only bounds a single socket read; this
+                # deadline bounds the whole streamed generation.
+                self._stream_deadline = time.monotonic() + self.timeout_seconds
+                return response
+
+            status_code = response.status_code
+            if 300 <= status_code < 400:
+                location = response.headers.get("Location", "")
+                response.close()
+                raise ProviderError(
+                    "provider redirected the request to "
+                    f"{location or 'another location'}; refusing to forward "
+                    "credentials and the request body"
+                )
+            # Headers must be read before the body is consumed and closed.
+            retry_after = response.headers.get("Retry-After", "1")
+            try:
+                body = response.read().decode("utf-8", errors="replace")[:2000]
+            except httpx2.HTTPError as exc:
+                body = ""
+                last_error = ProviderError(
+                    f"provider network error while reading response: {exc}",
+                    status_code=status_code,
+                )
+            else:
+                last_error = ProviderError(
+                    f"provider returned HTTP {status_code}: {body}",
+                    status_code=status_code,
+                    body=body,
+                )
+            finally:
+                response.close()
+            # Rate limits and unavailable responses get one retry, after honouring
+            # Retry-After; the flag is never reset, so a repeated 429/503 is
+            # reported instead of looping. Other statuses are final, because the
+            # provider may already have started (and billed) a generation.
+            if status_code not in {429, 503} or status_retried:
+                break
+            status_retried = True
+            try:
+                # Retry-After is seconds in practice. The floor keeps "0" from
+                # collapsing the single status retry into an immediate repeat;
+                # the ceiling stops a provider from parking the turn for minutes.
+                requested_delay = float(retry_after)
+                if not math.isfinite(requested_delay):
+                    requested_delay = 1.0
+                delay = min(max(requested_delay, 0.2), 5.0)
+            except ValueError:
+                delay = 1.0
+            if delay >= deadline - time.monotonic():
+                # Honouring Retry-After is not worth exceeding the caller's
+                # timeout; report the status instead of stalling.
+                break
+            time.sleep(delay)
+        if last_error is None:  # pragma: no cover - defensive
+            raise ProviderError("provider request failed")
+        raise last_error

minimalcode/provider/types.py

@@ -0,0 +1,54 @@
+"""Provider-neutral reply and tool-call model objects."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(slots=True)
+class ToolCall:
+    """One function call requested by the model."""
+
+    id: str
+    name: str
+    arguments: str
+
+
+@dataclass(slots=True)
+class ProviderReply:
+    """Normalized result of one provider turn."""
+
+    content: str = ""
+    tool_calls: list[ToolCall] = field(default_factory=list)
+    finish_reason: str | None = None
+    usage: dict[str, int] = field(default_factory=dict)
+    # Anthropic thinking blocks must be replayed with their signatures while a
+    # tool call is open. Other protocols ignore this field, and the Chat payload
+    # builder strips it before sending.
+    provider_content: list[dict[str, Any]] | None = None
+
+    def as_message(self) -> dict[str, Any]:
+        """Render the reply as an assistant message for the conversation log."""
+        message: dict[str, Any] = {
+            "role": "assistant",
+            # A Chat assistant message without tool calls must carry a string;
+            # null content is rejected by strict OpenAI-compatible servers and
+            # would poison every later request in the session.
+            "content": self.content if self.content or not self.tool_calls else None,
+        }
+        if self.provider_content:
+            message["provider_content"] = self.provider_content
+        if self.tool_calls:
+            message["tool_calls"] = [
+                {
+                    "id": call.id,
+                    "type": "function",
+                    "function": {
+                        "name": call.name,
+                        "arguments": call.arguments,
+                    },
+                }
+                for call in self.tool_calls
+            ]
+        return message

minimalcode/sandbox/__init__.py

@@ -0,0 +1,64 @@
+"""Fail-closed sandbox policy, backends, and bounded execution.
+
+The package is split into policy, backend, execution, and runner modules.
+``__all__`` is the complete public surface; private helpers moved out of the
+pre-0.7 single ``sandbox`` module are imported from the submodule that defines
+them (``minimalcode.sandbox.execution``, ``.backends``, ``.policy``,
+``.network``, ``.sensitive``).
+"""
+
+from __future__ import annotations
+
+from .audit import SandboxAudit, SandboxDenial
+from .backends import (
+    BwrapBackend,
+    CommandRequest,
+    DockerBackend,
+    FirejailBackend,
+    LocalSandboxBackend,
+    MacSandboxExecBackend,
+    SandboxBackend,
+    SandboxUnavailable,
+    available_backends,
+    backend_for,
+)
+from .execution import truncate_text
+from .network import host_allowed, network_signals
+from .policy import NETWORK_POLICIES, SANDBOX_BACKENDS, SandboxPolicy
+from .runner import SandboxRunner
+from .sensitive import (
+    SAFE_ENV_TEMPLATES,
+    SENSITIVE_DIRECTORY_NAMES,
+    SENSITIVE_FILE_NAMES,
+    SENSITIVE_SEARCH_GLOBS,
+    SENSITIVE_SUFFIXES,
+    is_sensitive_path,
+)
+
+__all__ = [
+    "NETWORK_POLICIES",
+    "SAFE_ENV_TEMPLATES",
+    "SANDBOX_BACKENDS",
+    "SENSITIVE_DIRECTORY_NAMES",
+    "SENSITIVE_FILE_NAMES",
+    "SENSITIVE_SEARCH_GLOBS",
+    "SENSITIVE_SUFFIXES",
+    "BwrapBackend",
+    "CommandRequest",
+    "DockerBackend",
+    "FirejailBackend",
+    "LocalSandboxBackend",
+    "MacSandboxExecBackend",
+    "SandboxAudit",
+    "SandboxBackend",
+    "SandboxDenial",
+    "SandboxPolicy",
+    "SandboxRunner",
+    "SandboxUnavailable",
+    "available_backends",
+    "backend_for",
+    "host_allowed",
+    "is_sensitive_path",
+    "network_signals",
+    "truncate_text",
+]

minimalcode/sandbox/audit.py

@@ -0,0 +1,120 @@
+"""Bounded audit records for sandbox denials and operational notes."""
+
+from __future__ import annotations
+
+import re
+import threading
+from collections import deque
+from dataclasses import dataclass
+from typing import Any
+
+# A command that was refused on network grounds is the one most likely to
+# carry a credential: `curl -H "Authorization: Bearer ..."` is exactly what the
+# policy stops. Records must not become a second plaintext copy of the key.
+_SECRET_PATTERNS = (
+    re.compile(r"(?i)\b(bearer\s+)[A-Za-z0-9._~+/=-]+"),
+    re.compile(r"(?i)\b(authorization\s*:\s*)(?!bearer)[^\s'\"]+"),
+    re.compile(r"(?i)\b(x-api-key\s*:\s*)[^\s'\"]+"),
+    re.compile(
+        # `access_token=...` and `client-secret: ...` are as sensitive as a
+        # bare `token=...`, so allow a name prefix before the keyword.
+        r"(?i)\b([\w-]*(?:api[_-]?key|token|password|passwd|secret)\s*[=:]\s*)"
+        r"[^\s'\"]+"
+    ),
+    re.compile(r"(?i)(--(?:password|token|api-key|header)\s+)[^\s'\"]+"),
+    re.compile(
+        # The value may be attached (`-uadmin:pw`), `=`-joined
+        # (`--user=admin:pw`), quoted (`-b "session=abc"`), or a separate
+        # argument. The quoted branch matters: `[^\s'\"]` could never match a
+        # value that starts with a quote, so the credential was recorded in
+        # plaintext.
+        r"(?i)(\s(?:-u|--user|--password|-b|--cookie|-H|--header)"
+        r"(?:\s+|=\s*|(?=[^-])))"
+        r"(?:['\"][^'\"]*['\"]|[^\s'\"]+)"
+    ),
+    # `mysql -phunter2` only attaches its password; a bare `-p` flag such as
+    # `docker run -p 8080:80` must keep its argument readable.
+    re.compile(r"(?i)(\s-p(?=[^\s-]))[^\s'\"]+"),
+    # `https://user:password@host` puts the credential in the URL itself, which
+    # none of the option-shaped patterns above reach.
+    re.compile(r"(?i)(://[^/\s:@'\"]+:)[^/\s@'\"]+(?=@)"),
+    # `net use \\server\share /user:name password` passes the secret as a
+    # separate, positionally paired argument.
+    re.compile(r"(?i)(/user:\S+\s+)[^\s'\"]+"),
+)
+_MAX_TARGET_CHARS = 500
+
+
+def redact_command(text: str) -> str:
+    """Mask credential-shaped values and bound the recorded text."""
+    redacted = str(text)
+    for pattern in _SECRET_PATTERNS:
+        redacted = pattern.sub(lambda match: match.group(1) + "<redacted>", redacted)
+    if len(redacted) > _MAX_TARGET_CHARS:
+        redacted = redacted[:_MAX_TARGET_CHARS] + " ...<truncated>"
+    return redacted
+
+
+@dataclass(frozen=True, slots=True)
+class SandboxDenial:
+    """One refused operation and the policy reason behind it."""
+
+    operation: str
+    target: str
+    reason: str
+
+    def as_dict(self) -> dict[str, str]:
+        """Return a JSON-serializable representation."""
+        return {
+            "operation": self.operation,
+            "target": self.target,
+            "reason": self.reason,
+        }
+
+
+class SandboxAudit:
+    """Bounded in-memory audit trail of refused operations and notes."""
+
+    def __init__(self, max_entries: int = 200) -> None:
+        # Both deques are bounded and share one lock so summary snapshots are
+        # internally consistent without retaining unbounded history.
+        self._lock = threading.Lock()
+        self._denials: deque[SandboxDenial] = deque(maxlen=max_entries)
+        self._notes: deque[str] = deque(maxlen=max_entries)
+        self.total_denials = 0
+
+    def deny(self, operation: str, target: str, reason: str) -> None:
+        """Record one denial and increment the lifetime counter."""
+        with self._lock:
+            self._denials.append(
+                SandboxDenial(
+                    operation=str(operation),
+                    target=redact_command(str(target)),
+                    reason=str(reason),
+                )
+            )
+            self.total_denials += 1
+
+    def note(self, message: str) -> None:
+        """Record an operational note such as a retry or timeout."""
+        with self._lock:
+            self._notes.append(redact_command(str(message)))
+
+    def denials(self) -> list[SandboxDenial]:
+        """Return a snapshot of the recent denial ring buffer."""
+        with self._lock:
+            return list(self._denials)
+
+    def notes(self) -> list[str]:
+        """Return a snapshot of the recent note ring buffer."""
+        with self._lock:
+            return list(self._notes)
+
+    def summary(self) -> dict[str, Any]:
+        """Return counters and recent history for CLI/status output."""
+        with self._lock:
+            return {
+                "total_denials": self.total_denials,
+                "recent": [denial.as_dict() for denial in self._denials],
+                "notes": list(self._notes),
+            }