turnloop 0.1.0__py3-none-any.whl

turnloop/providers/anthropic.py

@@ -0,0 +1,318 @@
+"""Anthropic Messages API adapter.
+
+The internal message model was designed to match this API, so conversion is
+close to identity. What is *not* identity is prompt caching: cache breakpoints
+have to be placed deliberately, and placing them wrong silently costs money
+rather than failing loudly.
+
+Breakpoint strategy (max 4 allowed by the API, we use 3):
+  1. last system segment      — stable across the whole session
+  2. last tool definition     — stable unless the tool set changes
+  3. last block of the second-to-last user turn — the moving frontier, so the
+     current turn's new content is the only uncached part
+
+Placing one on the *last* user turn instead would invalidate on every request,
+which is the classic way to pay cache-write prices for zero cache reads.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import AsyncIterator
+
+import httpx
+
+from turnloop.core.events import (
+    MessageDone,
+    StreamEvent,
+    TextDelta,
+    ThinkingDelta,
+    ToolUseArgsDelta,
+    ToolUseStart,
+)
+from turnloop.core.messages import (
+    ContentBlock,
+    Message,
+    TextBlock,
+    ThinkingBlock,
+    ToolResultBlock,
+    ToolUseBlock,
+    Usage,
+)
+from turnloop.providers.base import (
+    CompletionRequest,
+    ErrorClass,
+    Provider,
+    classify_error,
+    raise_for_class,
+)
+from turnloop.providers.openai_compat import _brief, _parse_args
+from turnloop.providers.sse import iter_sse
+
+API_VERSION = "2023-06-01"
+
+_STOP_REASONS = {
+    "end_turn": "end_turn",
+    "tool_use": "tool_use",
+    "max_tokens": "max_tokens",
+    "stop_sequence": "stop_sequence",
+    "refusal": "refusal",
+}
+
+_EPHEMERAL = {"cache_control": {"type": "ephemeral"}}
+
+
+class AnthropicProvider(Provider):
+    kind = "anthropic"
+
+    def __init__(self, *args, beta_headers: list[str] | None = None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.base_url = self.base_url or "https://api.anthropic.com/v1"
+        self.beta_headers = beta_headers or []
+
+    def headers(self) -> dict[str, str]:
+        h = super().headers()
+        h["anthropic-version"] = API_VERSION
+        if self.api_key:
+            h["x-api-key"] = self.api_key
+        if self.beta_headers:
+            h["anthropic-beta"] = ",".join(self.beta_headers)
+        return h
+
+    # --- request construction ---------------------------------------------
+
+    def build_payload(self, req: CompletionRequest) -> dict:
+        payload: dict = {
+            "model": self.model,
+            "max_tokens": req.max_tokens or self.caps.max_output,
+            "messages": self.to_wire_messages(req.messages),
+            "stream": True,
+        }
+        if req.system:
+            payload["system"] = self._system_blocks(req.system)
+        if req.tools:
+            tools = [t.to_anthropic() for t in req.tools]
+            if self.caps.supports_prompt_caching and tools:
+                tools[-1] = {**tools[-1], **_EPHEMERAL}
+            payload["tools"] = tools
+            if self.caps.supports_tool_choice and req.tool_choice != "auto":
+                payload["tool_choice"] = {
+                    "none": {"type": "none"},
+                    "required": {"type": "any"},
+                }[req.tool_choice]
+
+        if req.thinking_tokens and self.caps.native_thinking:
+            payload["thinking"] = {"type": "enabled", "budget_tokens": req.thinking_tokens}
+            # Extended thinking requires an unconstrained sampler.
+            payload.pop("temperature", None)
+        elif req.temperature is not None and self.caps.supports_temperature:
+            payload["temperature"] = req.temperature
+
+        if req.stop_sequences:
+            payload["stop_sequences"] = req.stop_sequences
+        return payload
+
+    def _system_blocks(self, segments: list[str]) -> list[dict]:
+        blocks: list[dict] = [{"type": "text", "text": s} for s in segments if s.strip()]
+        if blocks and self.caps.supports_prompt_caching:
+            blocks[-1] = {**blocks[-1], **_EPHEMERAL}
+        return blocks
+
+    def to_wire_messages(self, messages: list[Message]) -> list[dict]:
+        wire: list[dict] = []
+        for msg in messages:
+            blocks = [b for b in (self._block_to_wire(b) for b in msg.content) if b]
+            if not blocks:
+                continue
+            wire.append({"role": msg.role, "content": blocks})
+
+        if self.caps.supports_prompt_caching:
+            self._mark_cache_frontier(wire)
+        return wire
+
+    @staticmethod
+    def _block_to_wire(block: ContentBlock) -> dict | None:
+        if isinstance(block, TextBlock):
+            return {"type": "text", "text": block.text} if block.text else None
+        if isinstance(block, ThinkingBlock):
+            # Only signed thinking may be replayed; unsigned reasoning from
+            # another provider would be rejected as an invalid signature.
+            if not block.resendable:
+                return None
+            return {"type": "thinking", "thinking": block.text, "signature": block.signature}
+        if isinstance(block, ToolUseBlock):
+            return {"type": "tool_use", "id": block.id, "name": block.name, "input": block.args}
+        if isinstance(block, ToolResultBlock):
+            return {
+                "type": "tool_result",
+                "tool_use_id": block.tool_use_id,
+                "content": block.content or "(no output)",
+                "is_error": block.is_error,
+            }
+        return None
+
+    @staticmethod
+    def _mark_cache_frontier(wire: list[dict]) -> None:
+        """Cache up to the second-to-last user turn — the stable prefix."""
+        user_indices = [i for i, m in enumerate(wire) if m["role"] == "user"]
+        if len(user_indices) < 2:
+            return
+        target = wire[user_indices[-2]]
+        if target["content"]:
+            target["content"][-1] = {**target["content"][-1], **_EPHEMERAL}
+
+    # --- streaming --------------------------------------------------------
+
+    async def _stream_once(self, req: CompletionRequest) -> AsyncIterator[StreamEvent]:
+        url = f"{self.base_url}/messages"
+        payload = self.build_payload(req)
+        bytes_seen = 0
+        acc = _Accumulator()
+
+        try:
+            async with self.client.stream(
+                "POST", url, json=payload, headers=self.headers()
+            ) as resp:
+                if resp.status_code >= 400:
+                    body = (await resp.aread()).decode("utf-8", errors="replace")
+                    raise_for_class(
+                        classify_error(None, resp),
+                        f"{self.name}: HTTP {resp.status_code} {_brief(body)}",
+                        status=resp.status_code,
+                        body=body,
+                    )
+                async for frame in iter_sse(resp.aiter_lines()):
+                    bytes_seen += len(frame.data)
+                    chunk = frame.json()
+                    if chunk is None:
+                        continue
+                    if chunk.get("type") == "error":
+                        err = chunk.get("error", {})
+                        # An overloaded_error mid-stream is transient; anything else
+                        # arriving as a stream error will fail the same way on retry.
+                        in_stream: ErrorClass = (
+                            "retryable" if err.get("type") == "overloaded_error" else "fatal"
+                        )
+                        raise_for_class(in_stream, f"{self.name}: {err.get('message', err)}")
+                    for event in acc.feed(chunk):
+                        yield event
+        except (httpx.HTTPError, httpx.StreamError) as exc:
+            kind = classify_error(exc, None, bytes_seen)
+            raise_for_class(
+                "fatal" if kind == "fatal" else "retryable",
+                f"{self.name}: {type(exc).__name__}: {exc}",
+            )
+
+        yield acc.finish(self.name, self.model)
+
+
+class _Accumulator:
+    """Assembles Anthropic's block-oriented event stream."""
+
+    def __init__(self):
+        self.blocks: dict[int, dict] = {}
+        self.stop_reason: str | None = None
+        self.usage = Usage()
+
+    def feed(self, chunk: dict) -> list[StreamEvent]:
+        kind = chunk.get("type")
+        events: list[StreamEvent] = []
+
+        if kind == "message_start":
+            self.usage = _usage_from((chunk.get("message") or {}).get("usage") or {})
+
+        elif kind == "content_block_start":
+            index = chunk["index"]
+            block = chunk.get("content_block") or {}
+            btype = block.get("type")
+            self.blocks[index] = {"type": btype, "text": "", "args": "", **block}
+            if btype == "tool_use":
+                events.append(
+                    ToolUseStart(index=index, id=block.get("id", ""), name=block.get("name", ""))
+                )
+
+        elif kind == "content_block_delta":
+            index = chunk["index"]
+            delta = chunk.get("delta") or {}
+            slot = self.blocks.setdefault(index, {"type": "text", "text": "", "args": ""})
+            dtype = delta.get("type")
+            if dtype == "text_delta":
+                slot["text"] += delta.get("text", "")
+                events.append(TextDelta(delta.get("text", "")))
+            elif dtype == "thinking_delta":
+                slot["text"] += delta.get("thinking", "")
+                events.append(ThinkingDelta(delta.get("thinking", "")))
+            elif dtype == "signature_delta":
+                slot["signature"] = slot.get("signature", "") + delta.get("signature", "")
+            elif dtype == "input_json_delta":
+                fragment = delta.get("partial_json", "")
+                slot["args"] += fragment
+                if fragment:
+                    events.append(ToolUseArgsDelta(index=index, fragment=fragment))
+
+        elif kind == "message_delta":
+            delta = chunk.get("delta") or {}
+            if sr := delta.get("stop_reason"):
+                self.stop_reason = _STOP_REASONS.get(sr, "end_turn")
+            if usage := chunk.get("usage"):
+                merged = _usage_from(usage)
+                # message_delta reports cumulative output tokens only.
+                self.usage = Usage(
+                    input_tokens=self.usage.input_tokens or merged.input_tokens,
+                    output_tokens=merged.output_tokens or self.usage.output_tokens,
+                    cache_read_tokens=self.usage.cache_read_tokens or merged.cache_read_tokens,
+                    cache_write_tokens=self.usage.cache_write_tokens or merged.cache_write_tokens,
+                )
+
+        return events
+
+    def finish(self, provider: str, model: str) -> MessageDone:
+        content: list[ContentBlock] = []
+        for index in sorted(self.blocks):
+            slot = self.blocks[index]
+            btype = slot.get("type")
+            if btype == "text" and slot["text"]:
+                content.append(TextBlock(text=slot["text"]))
+            elif btype == "thinking" and slot["text"]:
+                content.append(
+                    ThinkingBlock(
+                        text=slot["text"],
+                        signature=slot.get("signature"),
+                        provider=f"{provider}:thinking",
+                    )
+                )
+            elif btype == "redacted_thinking":
+                content.append(
+                    ThinkingBlock(text="[redacted]", provider=f"{provider}:redacted")
+                )
+            elif btype == "tool_use":
+                content.append(
+                    ToolUseBlock(
+                        id=slot.get("id", ""),
+                        name=slot.get("name", ""),
+                        args=slot.get("input") or _parse_args(slot.get("args", "")),
+                    )
+                )
+
+        stop = self.stop_reason or "end_turn"
+        msg = Message(
+            role="assistant",
+            content=content,
+            usage=self.usage,
+            stop_reason=stop,  # type: ignore[arg-type]
+            meta={"provider": provider, "model": model},
+        )
+        return MessageDone(message=msg, usage=self.usage, stop_reason=stop)
+
+
+def _usage_from(u: dict) -> Usage:
+    return Usage(
+        input_tokens=u.get("input_tokens", 0) or 0,
+        output_tokens=u.get("output_tokens", 0) or 0,
+        cache_read_tokens=u.get("cache_read_input_tokens", 0) or 0,
+        cache_write_tokens=u.get("cache_creation_input_tokens", 0) or 0,
+    )
+
+
+def _dumps(obj) -> str:  # pragma: no cover - debugging aid
+    return json.dumps(obj, indent=2, default=str)

turnloop/providers/base.py

@@ -0,0 +1,329 @@
+"""Provider abstraction.
+
+Design note. There is one seam in this project between "the agent loop" and "an
+LLM", and it is `Provider.stream()`. Everything provider-specific — wire format,
+auth, caching, reasoning fields, error taxonomy — lives behind it. Nothing above
+it may import a provider module directly; the registry resolves names.
+
+Capability degradation is deliberately *not* handled here. Adapters report what
+they can do via `Capabilities`; the loop decides how to cope. Otherwise every
+adapter grows its own copy of the fallback logic.
+"""
+
+from __future__ import annotations
+
+import time
+from abc import ABC, abstractmethod
+from collections.abc import AsyncIterator
+from dataclasses import dataclass, field
+from typing import Literal
+
+import httpx
+from pydantic import BaseModel
+
+from turnloop.core.events import ProviderStatus, StreamEvent
+from turnloop.core.messages import Message, ToolSpec
+from turnloop.core.tokens import TokenEstimator
+from turnloop.errors import (
+    ColdBootTimeout,
+    FatalProviderError,
+    RetryableProviderError,
+)
+
+ErrorClass = Literal["cold_boot", "retryable", "fatal"]
+
+
+class Capabilities(BaseModel):
+    """What a model/endpoint can do. Consulted by the loop, not by adapters."""
+
+    max_context: int = 200_000
+    max_output: int = 8_192
+
+    supports_prompt_caching: bool = False
+    supports_parallel_tool_calls: bool = True
+    supports_reasoning_field: bool = False  # separate reasoning_content on the wire
+    native_thinking: bool = False  # request a thinking budget (Anthropic)
+    supports_system_role: bool = True
+    supports_tool_choice: bool = True
+    supports_temperature: bool = True
+
+    # Per-million-token pricing. Zero means "unknown/free".
+    price_in_per_mtok: float = 0.0
+    price_out_per_mtok: float = 0.0
+    price_cache_read_per_mtok: float = 0.0
+    price_cache_write_per_mtok: float = 0.0
+
+    # Self-hosted endpoints bill wall clock, not tokens. GLM-5.2 on 4xH200 is
+    # $18.16/hr, which dwarfs any token cost and must be surfaced differently.
+    cost_per_hour: float = 0.0
+
+    # vLLM's --max-num-seqs, or an API's concurrency ceiling. The loop builds one
+    # CapacityLimiter from this, shared process-wide, so subagent fan-out cannot
+    # queue behind itself.
+    max_concurrent_requests: int = 8
+
+    def cost_for(self, input_tokens: int, output_tokens: int, cache_read: int = 0,
+                 cache_write: int = 0) -> float:
+        m = 1_000_000
+        return (
+            input_tokens * self.price_in_per_mtok / m
+            + output_tokens * self.price_out_per_mtok / m
+            + cache_read * self.price_cache_read_per_mtok / m
+            + cache_write * self.price_cache_write_per_mtok / m
+        )
+
+
+@dataclass(slots=True)
+class CompletionRequest:
+    messages: list[Message]
+    system: list[str] = field(default_factory=list)  # segments, cacheable in order
+    tools: list[ToolSpec] = field(default_factory=list)
+    max_tokens: int | None = None
+    temperature: float | None = 0.0
+    thinking_tokens: int | None = None  # only honored when caps.native_thinking
+    tool_choice: Literal["auto", "none", "required"] = "auto"
+    stop_sequences: list[str] = field(default_factory=list)
+
+
+def classify_error(exc: BaseException | None, response: httpx.Response | None,
+                   bytes_received: int = 0, body: str | None = None) -> ErrorClass:
+    """Decide whether a failure means "still booting", "try again", or "give up".
+
+    Getting this wrong is expensive in both directions: treat a 400 as a cold
+    boot and you retry a malformed tool schema for 55 minutes; treat a cold boot
+    as fatal and the GLM endpoint looks permanently broken for its first ~29
+    minutes.
+
+    The discriminator for timeouts is whether any bytes arrived. Zero bytes means
+    the server never answered — consistent with vLLM not yet bound to its port.
+    A timeout mid-stream is a different failure and is merely retryable.
+    """
+    if response is not None:
+        status = response.status_code
+        if status in (502, 503, 504):
+            return "cold_boot"  # Modal's edge answers before the container does
+        if status in (408, 409, 425, 429, 500, 529):
+            return "retryable"
+        # A per-minute token quota can surface as 413 rather than 429 — Groq's free
+        # tier does exactly this. The status alone says "your request is too big
+        # forever"; the body says "too big *this minute*", which is worth waiting out.
+        if body and "rate_limit" in body and status in (400, 413):
+            return "retryable"
+        if 400 <= status < 500:
+            return "fatal"
+        if status >= 500:
+            return "retryable"
+        return "retryable"
+
+    if isinstance(exc, httpx.ConnectError | httpx.ConnectTimeout):
+        return "cold_boot"
+    if isinstance(exc, httpx.ReadTimeout | httpx.PoolTimeout):
+        return "cold_boot" if bytes_received == 0 else "retryable"
+    if isinstance(exc, httpx.RemoteProtocolError | httpx.ReadError | httpx.WriteError):
+        return "retryable"
+    if isinstance(exc, httpx.TimeoutException):
+        return "cold_boot" if bytes_received == 0 else "retryable"
+    return "fatal"
+
+
+def raise_for_class(kind: ErrorClass, message: str, *, status: int | None = None,
+                    body: str | None = None, retry_after: float | None = None) -> None:
+    if kind == "fatal":
+        raise FatalProviderError(message, status=status, body=body, retry_after=retry_after)
+    raise RetryableProviderError(message, status=status, body=body, retry_after=retry_after)
+
+
+def parse_retry_after(response: httpx.Response | None) -> float | None:
+    """Seconds from a `Retry-After` header, when the server sent one."""
+    if response is None:
+        return None
+    raw = response.headers.get("retry-after") or response.headers.get("x-ratelimit-reset-tokens")
+    if not raw:
+        return None
+    try:
+        return max(0.0, float(str(raw).rstrip("s")))
+    except ValueError:
+        return None  # HTTP-date form; the default backoff is good enough
+
+
+class Provider(ABC):
+    """Base class for every adapter.
+
+    Subclasses implement `_stream_once` (one attempt, no retries) and the loop
+    around it — health gating, cold-boot patience, backoff — is inherited.
+    """
+
+    kind: str = "abstract"
+
+    def __init__(self, *, name: str, model: str, caps: Capabilities,
+                 base_url: str | None = None, api_key: str = "", headers: dict | None = None,
+                 timeout_s: float | None = None, health_url: str | None = None,
+                 cold_boot_budget_s: float = 55 * 60, cold_boot_poll_s: float = 10.0,
+                 max_retries: int = 3):
+        self.name = name
+        self.model = model
+        self.caps = caps
+        self.base_url = (base_url or "").rstrip("/")
+        self.api_key = api_key
+        self.extra_headers = headers or {}
+        self.timeout_s = timeout_s
+        self.health_url = health_url
+        self.cold_boot_budget_s = cold_boot_budget_s
+        self.cold_boot_poll_s = cold_boot_poll_s
+        self.max_retries = max_retries
+
+        self.estimator = TokenEstimator()
+        self._client: httpx.AsyncClient | None = None
+        self._healthy_until: float = 0.0
+        # Wall clock for cost_per_hour providers, from first successful request.
+        self.first_request_at: float | None = None
+        self.last_request_at: float | None = None
+
+    # --- HTTP plumbing -----------------------------------------------------
+
+    def _timeout(self) -> httpx.Timeout:
+        """Short connect, unbounded read.
+
+        Connect timing is our cold-boot signal, so it stays tight. Read must be
+        unbounded: time-to-first-token on a 4xH200 serving a 744B model can be
+        minutes, and this is exactly where the openai SDK's 600s default breaks.
+        """
+        if self.timeout_s is None:
+            return httpx.Timeout(connect=10.0, read=None, write=30.0, pool=None)
+        return httpx.Timeout(self.timeout_s, connect=min(10.0, self.timeout_s))
+
+    @property
+    def client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            self._client = httpx.AsyncClient(timeout=self._timeout(), follow_redirects=True)
+        return self._client
+
+    async def aclose(self) -> None:
+        if self._client is not None:
+            await self._client.aclose()
+            self._client = None
+
+    def headers(self) -> dict[str, str]:
+        return {"content-type": "application/json", **self.extra_headers}
+
+    # --- readiness ---------------------------------------------------------
+
+    async def preflight(self) -> AsyncIterator[ProviderStatus]:
+        """Poll a health endpoint until it answers, reporting progress.
+
+        Only self-hosted endpoints configure `health_url`. For hosted APIs this
+        yields nothing and returns immediately.
+        """
+        if not self.health_url:
+            return
+        if time.monotonic() < self._healthy_until:
+            return
+
+        started = time.monotonic()
+        attempt = 0
+        while True:
+            elapsed = time.monotonic() - started
+            try:
+                resp = await self.client.get(
+                    self.health_url, timeout=httpx.Timeout(15.0, connect=10.0)
+                )
+                if resp.status_code < 400:
+                    # Trust health for a while; re-checking every turn wastes a
+                    # round trip and, on Modal, counts as billable traffic.
+                    self._healthy_until = time.monotonic() + 120
+                    yield ProviderStatus("endpoint ready", phase="ready", elapsed_s=elapsed)
+                    return
+            except Exception:  # noqa: BLE001 - any transport failure means "not up"
+                pass
+
+            if elapsed > self.cold_boot_budget_s:
+                raise ColdBootTimeout(self._timeout_message(elapsed))
+
+            attempt += 1
+            yield ProviderStatus(self._boot_hint(elapsed), phase="cold_boot", elapsed_s=elapsed)
+            await _sleep(self.cold_boot_poll_s)
+
+    def _boot_hint(self, elapsed: float) -> str:
+        mins, secs = divmod(int(elapsed), 60)
+        budget_mins = self.cold_boot_budget_s / 60
+        base = f"cold boot: {mins}m{secs:02d}s elapsed, budget {budget_mins:.0f}m"
+        if self.caps.cost_per_hour:
+            base += " (first boot ~29 min, ~13 min with a warm compile cache)"
+        return base
+
+    def _timeout_message(self, elapsed: float) -> str:
+        """An error a tired human can act on, not just "it timed out".
+
+        The endpoint accepted every connection and never answered — which is
+        exactly what a stopped Modal app also looks like from here (see
+        `classify_error`'s docstring for why the two cannot be told apart on
+        the wire). Say that plainly instead of implying the wait was simply
+        too short.
+        """
+        msg = (
+            f"{self.name}: endpoint accepted connections for "
+            f"{self.cold_boot_budget_s / 60:.0f} min but never became healthy. "
+            "This is indistinguishable from a stopped deployment — Modal's edge "
+            "accepts the TCP connection either way."
+        )
+        if self.caps.cost_per_hour:
+            msg += " If this is Modal-hosted, verify the app is actually deployed (not stopped)."
+        return msg
+
+    # --- the streaming loop -----------------------------------------------
+
+    @abstractmethod
+    def _stream_once(self, req: CompletionRequest) -> AsyncIterator[StreamEvent]:
+        """One attempt. Raise Fatal/Retryable errors; do not retry internally."""
+        raise NotImplementedError
+
+    async def stream(self, req: CompletionRequest) -> AsyncIterator[StreamEvent]:
+        """Health-gate, then attempt with backoff, surfacing status as events."""
+        async for status in self.preflight():
+            yield status
+
+        attempt = 0
+        while True:
+            try:
+                async for event in self._stream_once(req):
+                    yield event
+                self.last_request_at = time.monotonic()
+                if self.first_request_at is None:
+                    self.first_request_at = self.last_request_at
+                return
+            except FatalProviderError:
+                raise
+            except RetryableProviderError as exc:
+                attempt += 1
+                if attempt > self.max_retries:
+                    raise
+                delay = exc.retry_after if exc.retry_after else min(2 ** attempt, 30)
+                delay = min(delay, 90)  # never park a turn for longer than this
+                yield ProviderStatus(
+                    f"{exc} — retry {attempt}/{self.max_retries} in {delay}s", phase="retrying"
+                )
+                await _sleep(delay)
+            except ColdBootTimeout:
+                raise
+
+    # --- accounting -------------------------------------------------------
+
+    def gpu_seconds(self) -> float | None:
+        """Wall clock a self-hosted endpoint has been in use this session."""
+        if not self.caps.cost_per_hour or self.first_request_at is None:
+            return None
+        return time.monotonic() - self.first_request_at
+
+    def estimate_tokens(self, messages: list[Message], system: list[str],
+                        tools: list[ToolSpec]) -> int:
+        return self.estimator.estimate_request(messages, system, tools)
+
+    def __repr__(self) -> str:  # pragma: no cover
+        return f"<{type(self).__name__} {self.name}:{self.model}>"
+
+
+async def _sleep(seconds: float) -> None:
+    # Indirected so tests can patch a single symbol instead of anyio internals.
+    import anyio
+
+    await anyio.sleep(seconds)