stcode 0.0.1a0__py3-none-any.whl

stcode/core/providers/__init__.py

@@ -0,0 +1,31 @@
+"""
+Providers — LLM adapters, unified request/event types, the registry, and the
+difficulty-routing gateway built on them.
+
+A thin re-export hub; each concern is its own leaf module (`base.py`, `<provider>.py`,
+`registry.py`, `gateway.py`) so nothing outside needs the internal layout.
+"""
+
+from stcode.core.providers.anthropic_claude import AnthropicProvider
+from stcode.core.providers.base import BaseModelProvider
+from stcode.core.providers.gateway import LLMGateway
+from stcode.core.providers.google_gemini import GoogleGenAIProvider
+from stcode.core.providers.openai_gpt import OpenAIProvider
+from stcode.core.providers.registry import (
+    DEFAULT_MODELS,
+    PROVIDERS,
+    default_model_for,
+    get_provider,
+)
+
+__all__ = [
+    "DEFAULT_MODELS",
+    "PROVIDERS",
+    "AnthropicProvider",
+    "BaseModelProvider",
+    "GoogleGenAIProvider",
+    "LLMGateway",
+    "OpenAIProvider",
+    "default_model_for",
+    "get_provider",
+]

stcode/core/providers/anthropic_claude.py

@@ -0,0 +1,243 @@
+"""
+Anthropic LLM provider.
+
+**Prompt caching lives here and nowhere else.** It is the single biggest cost lever, and
+the one optimisation that has to be applied at the provider boundary: only Anthropic
+asks for it explicitly. OpenAI caches long prefixes on its own, and Gemini needs a
+separate cached-content resource, which is a different feature. See the `prompt caching`
+section at the bottom for where the three markers go and why.
+"""
+
+from __future__ import annotations
+
+from typing import Any, AsyncGenerator
+
+import anthropic
+
+from stcode.core.providers.base import BaseModelProvider
+from stcode.core.providers.types import (
+    Message,
+    MessageStop,
+    ReasoningDelta,
+    ReasoningEffort,
+    StopReason,
+    StreamEvent,
+    TextBlock,
+    TextDelta,
+    ToolCallDelta,
+    ToolCallEnd,
+    ToolCallStart,
+    ToolDefinition,
+    ToolResultBlock,
+    ToolUseBlock,
+    Usage,
+)
+
+DEFAULT_MODEL = "claude-opus-5"
+
+# Anthropic's output_config.effort has no "none"/"minimal" rung — "none" instead
+# turns thinking off outright (handled separately in stream()), and "minimal"
+# has no closer match than "low".
+_EFFORT_TO_ANTHROPIC: dict[str, str] = {
+    "minimal": "low",
+    "low": "low",
+    "medium": "medium",
+    "high": "high",
+    "xhigh": "xhigh",
+    "max": "max",
+}
+
+
+class AnthropicProvider(BaseModelProvider):
+    def __init__(self, api_key: str | None = None, base_url: str | None = None) -> None:
+        super().__init__(api_key, base_url)
+        self._client = anthropic.AsyncAnthropic(api_key=api_key, base_url=base_url)
+
+    async def list_models(self) -> list[str]:
+        return [model.id async for model in self._client.models.list()]
+
+    async def aclose(self) -> None:
+        await self._client.close()
+
+    async def stream(
+        self,
+        messages: list[Message],
+        *,
+        model: str = DEFAULT_MODEL,
+        system: str | None = None,
+        tools: list[ToolDefinition] | None = None,
+        max_tokens: int = 8192,
+        reasoning_effort: ReasoningEffort | None = None,
+        temperature: float | None = None,
+        top_p: float | None = None,
+        stop: list[str] | None = None,
+        parallel_tool_calls: bool | None = None,
+    ) -> AsyncGenerator[StreamEvent, None]:
+        """`reasoning_effort="none"` sends `thinking: {"type": "disabled"}`; any
+        other value turns on adaptive thinking and sets `output_config.effort`
+        (Anthropic's `budget_tokens` token-ceiling style is a 400 on current
+        models, so this provider never uses it). Leaving `reasoning_effort`
+        unset omits `thinking` entirely, matching current models' own default
+        (adaptive thinking already on).
+
+        `temperature`/`top_p` are passed straight through — on Claude Opus 5,
+        Sonnet 5, Fable 5, and Opus 4.7/4.8, the API itself returns 400 if
+        either is combined with adaptive thinking, i.e. whenever
+        `reasoning_effort` is anything but `"none"`.
+        """
+        kwargs: dict[str, Any] = {}
+        if system:
+            kwargs["system"] = _mark_last([{"type": "text", "text": system}])
+        if tools:
+            kwargs["tools"] = _mark_last([self._to_tool(tool) for tool in tools])
+            if parallel_tool_calls is not None:
+                kwargs["tool_choice"] = {
+                    "type": "auto",
+                    "disable_parallel_tool_use": not parallel_tool_calls,
+                }
+        if reasoning_effort == "none":
+            kwargs["thinking"] = {"type": "disabled"}
+        elif reasoning_effort is not None:
+            kwargs["thinking"] = {"type": "adaptive"}
+            kwargs["output_config"] = {"effort": _EFFORT_TO_ANTHROPIC[reasoning_effort]}
+        if temperature is not None:
+            kwargs["temperature"] = temperature
+        if top_p is not None:
+            kwargs["top_p"] = top_p
+        if stop is not None:
+            kwargs["stop_sequences"] = stop
+
+        # index -> tool_use block id, for pairing content_block_delta/_stop back to a call
+        pending_calls: dict[int, str] = {}
+        usage = Usage()
+        stop_reason: StopReason = "end_turn"
+
+        async with self._client.messages.stream(
+            model=model,
+            max_tokens=max_tokens,
+            messages=_mark_last_block([self._to_message(message) for message in messages]),
+            **kwargs,
+        ) as stream:
+            async for event in stream:
+                if event.type == "message_start":
+                    started = event.message.usage
+                    usage.input_tokens = started.input_tokens
+                    # Present only when caching is in play, and the whole point of it:
+                    # a non-zero read on turn 2 is the evidence the prefix held.
+                    usage.cache_creation_input_tokens = (
+                        getattr(started, "cache_creation_input_tokens", None) or 0
+                    )
+                    usage.cache_read_input_tokens = (
+                        getattr(started, "cache_read_input_tokens", None) or 0
+                    )
+                elif event.type == "content_block_start":
+                    if event.content_block.type == "tool_use":
+                        pending_calls[event.index] = event.content_block.id
+                        yield ToolCallStart(id=event.content_block.id, name=event.content_block.name)
+                elif event.type == "content_block_delta":
+                    if event.delta.type == "text_delta":
+                        yield TextDelta(text=event.delta.text)
+                    elif event.delta.type == "thinking_delta":
+                        yield ReasoningDelta(text=event.delta.thinking)
+                    elif event.delta.type == "input_json_delta":
+                        call_id = pending_calls.get(event.index)
+                        if call_id is not None:
+                            yield ToolCallDelta(id=call_id, partial_json=event.delta.partial_json)
+                elif event.type == "content_block_stop":
+                    call_id = pending_calls.pop(event.index, None)
+                    if call_id is not None:
+                        block = event.content_block
+                        args = block.input if isinstance(getattr(block, "input", None), dict) else {}
+                        yield ToolCallEnd(id=call_id, name=block.name, input=args)
+                elif event.type == "message_delta":
+                    usage.output_tokens = event.usage.output_tokens
+                    if event.delta.stop_reason is not None:
+                        stop_reason = self._map_stop_reason(event.delta.stop_reason)
+
+        yield MessageStop(stop_reason=stop_reason, usage=usage)
+
+    @staticmethod
+    def _to_tool(tool: ToolDefinition) -> dict[str, Any]:
+        return {
+            "name": tool.name,
+            "description": tool.description,
+            "input_schema": tool.input_schema,
+        }
+
+    @staticmethod
+    def _to_message(message: Message) -> dict[str, Any]:
+        if isinstance(message.content, str):
+            return {"role": message.role, "content": message.content}
+
+        content: list[dict[str, Any]] = []
+        for block in message.content:
+            if isinstance(block, TextBlock):
+                content.append({"type": "text", "text": block.text})
+            elif isinstance(block, ToolUseBlock):
+                content.append(
+                    {"type": "tool_use", "id": block.id, "name": block.name, "input": block.input}
+                )
+            elif isinstance(block, ToolResultBlock):
+                content.append(
+                    {
+                        "type": "tool_result",
+                        "tool_use_id": block.tool_use_id,
+                        "content": block.content,
+                        "is_error": block.is_error,
+                    }
+                )
+        return {"role": message.role, "content": content}
+
+    @staticmethod
+    def _map_stop_reason(stop_reason: str) -> StopReason:
+        if stop_reason in ("end_turn", "tool_use", "max_tokens", "stop_sequence", "pause_turn", "refusal"):
+            return stop_reason  # type: ignore[return-value]
+        return "end_turn"
+
+
+# ---- prompt caching --------------------------------------------------------------
+#
+# Anthropic caches the request *prefix*, in the order tools -> system -> messages, up
+# to each breakpoint. Three breakpoints, and each earns its place:
+#
+#   last tool     `registry.definitions()` sorts by name, so the tool block is
+#                 byte-stable for a session and survives a turn where the system
+#                 prompt moved underneath it.
+#   last system   Stable too, except on turns where the todo list changed —
+#                 `environment_section` is deliberately last for exactly this reason.
+#   last message  The one that actually matters in an agent loop. After ten tool calls
+#                 the history dwarfs both of the above, so without a breakpoint here
+#                 "turn 2 is ~10x cheaper" would be a rounding error rather than a
+#                 fact. Each turn's breakpoint is the next turn's cache hit.
+#
+# A write costs 1.25x and a read 0.1x, so this pays for itself on the second request
+# and every one after. A single-shot call pays the premium for nothing — the deliberate
+# trade, because this project is an agent loop.
+
+CACHE_CONTROL = {"type": "ephemeral"}
+
+
+def _mark_last(blocks: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Put a cache breakpoint on the final entry of `blocks`."""
+    if blocks:
+        blocks[-1] = {**blocks[-1], "cache_control": CACHE_CONTROL}
+    return blocks
+
+
+def _mark_last_block(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Put a cache breakpoint on the last content block of the last message.
+
+    A string `content` is promoted to a one-block list first: `cache_control` is a
+    property of a block, and there is nowhere to hang it on a bare string.
+    """
+    if not messages:
+        return messages
+    last = dict(messages[-1])
+    content = last["content"]
+    if isinstance(content, str):
+        content = [{"type": "text", "text": content}]
+    if not content:
+        return messages
+    last["content"] = [*content[:-1], {**content[-1], "cache_control": CACHE_CONTROL}]
+    messages[-1] = last
+    return messages

stcode/core/providers/base.py

@@ -0,0 +1,73 @@
+"""
+Base LLM Provider Class
+"""
+
+from abc import ABC, abstractmethod
+from typing import AsyncGenerator
+
+from stcode.core.providers.types import Message, ReasoningEffort, StreamEvent, ToolDefinition
+
+
+class BaseModelProvider(ABC):
+    """Base LLM Provider Class — external LLM provider communication async methods."""
+
+    def __init__(self, api_key: str | None = None, base_url: str | None = None) -> None:
+        """`base_url` lets a subclass talk to any endpoint that speaks its provider's
+        wire protocol — an OpenAI-compatible router, a self-hosted Anthropic-compatible
+        gateway, a proxy — not just the vendor's own API.
+        """
+        self.api_key = api_key
+        self.base_url = base_url
+
+    async def aclose(self) -> None:
+        """Release the underlying SDK client's network resources (connection pool,
+        open sockets). Subclasses whose client holds one should override this;
+        callers should always await it — via `async with provider:` or a
+        try/finally — once done streaming, or the client's transport can outlive
+        the event loop and raise on interpreter shutdown.
+        """
+
+    async def __aenter__(self) -> "BaseModelProvider":
+        return self
+
+    async def __aexit__(self, *_exc_info: object) -> None:
+        await self.aclose()
+
+    @abstractmethod
+    async def list_models(self) -> list[str]:
+        """List all possible models in provider."""
+
+    @abstractmethod
+    def stream(
+        self,
+        messages: list[Message],
+        *,
+        model: str,
+        system: str | None = None,
+        tools: list[ToolDefinition] | None = None,
+        max_tokens: int = 8192,
+        reasoning_effort: ReasoningEffort | None = None,
+        temperature: float | None = None,
+        top_p: float | None = None,
+        stop: list[str] | None = None,
+        parallel_tool_calls: bool | None = None,
+    ) -> AsyncGenerator[StreamEvent, None]:
+        """Stream a completion as a sequence of unified StreamEvents.
+
+        `messages` holds only user/assistant turns — the system prompt is
+        passed separately since every provider treats it as a distinct field.
+
+        `reasoning_effort` is the superset of every provider's effort scale
+        (`"none"` through `"max"`); each provider maps it onto its own control
+        (OpenAI's `reasoning_effort`, Anthropic's adaptive `thinking` +
+        `output_config.effort`, Gemini's `thinking_config.thinking_level`) and
+        clamps or drops values its API doesn't support. Leave it `None` to let
+        the provider apply its own default rather than forcing a level.
+
+        `temperature`, `top_p`, `stop`, and `parallel_tool_calls` are passed
+        through to each provider's equivalent field as-is — this base class
+        does not resolve cross-field conflicts a provider's own API rejects
+        (e.g. Anthropic returns 400 for `temperature`/`top_p` alongside
+        adaptive thinking on current-generation models); see each provider's
+        `stream()` docstring for its specific caveats.
+        """

stcode/core/providers/gateway.py

@@ -0,0 +1,267 @@
+"""
+LLM Gateway — model orchestration: difficulty-based routing and retry.
+
+This is a thin dispatch layer: given a difficulty tier (or an explicit provider/model
+override), it resolves which provider+model+credentials to use, retries transient
+failures, and streams back unified StreamEvents. It deliberately does not run an agent
+loop (no tool execution, no dynamic tool sets, no human-in-the-loop gating) — see
+`LLMGateway`'s docstring for why.
+
+It takes a `ModelConfig` and asks it for routes and credentials; it never touches paths,
+TOML or the environment itself — `core/configs.py` owns all three.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import random
+from typing import Any, AsyncGenerator
+
+import anthropic
+import openai
+from google.genai import errors as genai_errors
+
+from stcode.core.common import trace
+from stcode.core.configs import Difficulty, ModelConfig
+from stcode.core.providers.base import BaseModelProvider
+from stcode.core.providers.registry import default_model_for, get_provider
+from stcode.core.providers.types import (
+    Message,
+    MessageStop,
+    ReasoningEffort,
+    StreamEvent,
+    TextDelta,
+    ToolCallEnd,
+    ToolDefinition,
+)
+
+logger = logging.getLogger("stcode.providers.gateway")
+
+
+# Transient/retryable errors per SDK — network hiccups, rate limits, and 5xx. Anything
+# else (bad request, auth, not found, content policy) is a caller/config bug and should
+# surface immediately rather than being retried.
+_RETRYABLE_ANTHROPIC: tuple[type[Exception], ...] = (
+    anthropic.APIConnectionError,  # also covers APITimeoutError, a subclass
+    anthropic.RateLimitError,
+    anthropic.InternalServerError,
+    anthropic.OverloadedError,
+)
+_RETRYABLE_OPENAI: tuple[type[Exception], ...] = (
+    openai.APIConnectionError,  # also covers APITimeoutError, a subclass
+    openai.RateLimitError,
+    openai.InternalServerError,
+)
+
+
+def _is_retryable(exc: Exception) -> bool:
+    if isinstance(exc, _RETRYABLE_ANTHROPIC + _RETRYABLE_OPENAI):
+        return True
+    if isinstance(exc, genai_errors.ServerError):
+        return True
+    # google-genai has no dedicated RateLimitError — 429 surfaces as a generic ClientError.
+    if isinstance(exc, genai_errors.ClientError) and exc.code == 429:
+        return True
+    return False
+
+
+class LLMGateway:
+    """Routes completions to a provider+model by difficulty tier, retrying transient
+    failures.
+
+    Deliberately excludes the agent loop. Tool execution, turn-by-turn tool sets and
+    human-in-the-loop approval all vary per call and live *above* this layer; baking a
+    loop in would mean either re-exposing every provider knob through the gateway or
+    hard-coding one harness's shape into a thin "send this, stream that back" API.
+    """
+
+    def __init__(self, config: ModelConfig) -> None:
+        self._config = config
+        self._provider_instances: dict[tuple[str, str | None, str | None], BaseModelProvider] = {}
+        self._limits: dict[tuple[str, int], asyncio.Semaphore] = {}
+        """One semaphore per `(provider, cap)`, shared by everything holding this
+        gateway — which is the point. Keyed by the cap as well as the name so lowering
+        it in `reconfigure` makes a new, smaller one rather than a stale wide one."""
+
+    async def reconfigure(self, config: ModelConfig) -> None:
+        """Replace this gateway's configuration **in place**, keeping its identity.
+
+        In place because every agent in the daemon holds a reference to this object: a
+        new key typed into `/model` has to reach the session already running. The
+        cached clients are closed, since each was built around the key it replaces.
+        """
+        await self._close_clients()
+        self._config = config
+
+    async def aclose(self) -> None:
+        """Close every cached provider client. They are reused across `stream()` calls
+        and retries, so call this only when tearing the gateway down."""
+        await self._close_clients()
+
+    async def _close_clients(self) -> None:
+        for instance in self._provider_instances.values():
+            await instance.aclose()
+        self._provider_instances.clear()
+
+    async def __aenter__(self) -> "LLMGateway":
+        return self
+
+    async def __aexit__(self, *_exc_info: object) -> None:
+        await self.aclose()
+
+    def _get_provider(self, name: str) -> BaseModelProvider:
+        """A client for one `[model.providers]` entry, cached by its credentials."""
+        entry = self._config.providers[name]
+        key, base_url = entry.key(), entry.url()
+        cache_key = (name, key, base_url)
+        if cache_key not in self._provider_instances:
+            self._provider_instances[cache_key] = get_provider(
+                entry.provider, api_key=key, base_url=base_url
+            )
+        return self._provider_instances[cache_key]
+
+    def resolve(
+        self, difficulty: Difficulty, provider: str | None = None, model: str | None = None
+    ) -> tuple[str, str]:
+        """`(provider entry, model)` for one call.
+
+        A `provider` override (a session's `/model`) replaces the tier's route; a
+        `model` override alone keeps the route's provider. An empty model means the
+        library's own default.
+        """
+        if provider:
+            name, routed = provider, ""
+        else:
+            name, routed = self._config.route(difficulty)
+        entry = self._config.providers.get(name)
+        if entry is None:
+            raise ValueError(
+                f"Provider {name!r} not configured. Configured: {sorted(self._config.providers)}"
+            )
+        return name, model or routed or entry.model or default_model_for(entry.provider)
+
+    def _limiter(self, provider: str) -> "asyncio.Semaphore | contextlib.AbstractAsyncContextManager[Any]":
+        """The in-flight cap for one provider, or a no-op when it has none.
+
+        Built lazily rather than in `__init__`: a gateway is routinely constructed
+        outside a running loop (the daemon builds one before it binds), and a semaphore
+        is only ever awaited from inside one.
+        """
+        entry = self._config.providers.get(provider)
+        cap = entry.max_concurrent if entry is not None else 0
+        if cap <= 0:
+            return contextlib.nullcontext()
+        key = (provider, cap)
+        limiter = self._limits.get(key)
+        if limiter is None:
+            limiter = asyncio.Semaphore(cap)
+            self._limits[key] = limiter
+        return limiter
+
+    async def list_models(self, provider: str) -> list[str]:
+        return await self._get_provider(self.resolve("medium", provider)[0]).list_models()
+
+    async def stream(
+        self,
+        messages: list[Message],
+        *,
+        difficulty: Difficulty = "medium",
+        provider: str | None = None,
+        model: str | None = None,
+        system: str | None = None,
+        tools: list[ToolDefinition] | None = None,
+        max_tokens: int = 8192,
+        reasoning_effort: ReasoningEffort | None = None,
+        temperature: float | None = None,
+        top_p: float | None = None,
+        stop: list[str] | None = None,
+        parallel_tool_calls: bool | None = None,
+    ) -> AsyncGenerator[StreamEvent, None]:
+        """Stream one completion, routed by `difficulty` unless overridden. → `resolve`
+
+        `reasoning_effort`, `temperature`, `top_p`, `stop` and `parallel_tool_calls`
+        pass straight through; see `BaseModelProvider.stream` for the unified semantics.
+        """
+        provider_name, model_name = self.resolve(difficulty, provider, model)
+        provider_instance = self._get_provider(provider_name)
+
+        # `chat <model>`, the GenAI convention's name for an inference span. Platforms
+        # read the `gen_ai.*` attributes off it and show a generation with its tokens;
+        # anything named our own way shows as an anonymous box.
+        attributes = {
+            "gen_ai.operation.name": "chat",
+            "gen_ai.provider.name": provider_name,
+            "gen_ai.request.model": model_name,
+            "gen_ai.request.max_tokens": max_tokens,
+            "stcode.difficulty": difficulty,
+        }
+        # Held for the whole completion, not just the request: the point is how many
+        # streams the endpoint is serving at once. Every caller wraps its iteration in
+        # `aclosing`, so abandoning a stream still releases this.
+        async with self._limiter(provider_name):
+            with trace.span(f"chat {model_name}", kind="client", attributes=attributes) as recorder:
+                capture = trace.records_content()
+                output: list[dict[str, Any]] = []
+                if capture:
+                    inputs = [message.model_dump(mode="json") for message in messages]
+                    if system:
+                        inputs.insert(0, {"role": "system", "content": system})
+                    recorder.set(**{"gen_ai.input.messages": inputs})
+                async for event in self._stream_with_retry(
+                    provider_instance,
+                    messages,
+                    recorder,
+                    model=model_name,
+                    system=system,
+                    tools=tools,
+                    max_tokens=max_tokens,
+                    reasoning_effort=reasoning_effort,
+                    temperature=temperature,
+                    top_p=top_p,
+                    stop=stop,
+                    parallel_tool_calls=parallel_tool_calls,
+                ):
+                    if capture and isinstance(event, (TextDelta, ToolCallEnd)):
+                        output.append(event.model_dump(mode="json"))
+                    yield event
+                if capture:
+                    recorder.set(**{"gen_ai.output.messages": output})
+
+    async def _stream_with_retry(
+        self,
+        provider_instance: BaseModelProvider,
+        messages: list[Message],
+        recorder: "trace.Recorder",
+        **kwargs: Any,
+    ) -> AsyncGenerator[StreamEvent, None]:
+        """The retry loop. Split out only so the span above can wrap it whole."""
+        retry = self._config.retry
+        for attempt in range(1, retry.max_attempts + 1):
+            started = False
+            try:
+                async for event in provider_instance.stream(messages, **kwargs):
+                    started = True
+                    if isinstance(event, MessageStop):
+                        recorder.set(
+                            **{
+                                "gen_ai.response.finish_reasons": [event.stop_reason],
+                                "gen_ai.usage.input_tokens": event.usage.input_tokens,
+                                "gen_ai.usage.output_tokens": event.usage.output_tokens,
+                                "gen_ai.usage.cache_read_input_tokens": (
+                                    event.usage.cache_read_input_tokens
+                                ),
+                            }
+                        )
+                    yield event
+                return
+            except Exception as exc:
+                # Once anything is yielded the caller has partial content, and retrying
+                # would duplicate it. Only pre-first-token failures are retryable.
+                if started or attempt == retry.max_attempts or not _is_retryable(exc):
+                    raise
+                delay = min(retry.base_delay * (2 ** (attempt - 1)), retry.max_delay)
+                if retry.jitter:
+                    delay += random.uniform(0, delay * 0.1)
+                await asyncio.sleep(delay)