otto-cli-agent 0.1.0__py3-none-any.whl

agent/router/llm_provider/base.py

@@ -0,0 +1,507 @@
+"""Provider abstraction for Otto.
+
+One rule governs this module: the base class holds only what *every* vendor
+genuinely does. Anything a subset of vendors offer is a `Capability` plus an
+optional `Protocol` -- never an inherited method that raises.
+
+Subclasses implement exactly three things:
+
+    _build_client()   construct the vendor SDK client
+    _fetch_models()   one network call, normalised into `ModelInfo`
+    chat_model()      return a configured LangChain `BaseChatModel`
+
+Everything else is inherited. `chat_model` returning a LangChain model rather
+than a bespoke `chat()` method is deliberate: streaming, tool binding, message
+history, Langfuse callbacks and LangGraph node compatibility then come for
+free and behave identically across vendors.
+"""
+
+from __future__ import annotations
+
+import os
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import StrEnum
+from collections.abc import Mapping
+from typing import Any, Protocol, runtime_checkable
+
+from langchain_core.language_models import BaseChatModel
+
+from agent.router.llm_provider.temperature import looks_like_temperature_rejection
+from agent.router.llm_provider.retired import (
+    is_serviceable,
+    looks_retired,
+    note_retired,
+)
+
+__all__ = [
+    "Capability",
+    "ModelInfo",
+    "ProviderStatus",
+    "HealthReport",
+    "ProviderError",
+    "AuthError",
+    "ModelNotFound",
+    "CapabilityNotSupported",
+    "ProviderUnavailable",
+    "Completion",
+    "BaseProvider",
+    "SupportsFIM",
+    "SupportsEdit",
+    "SupportsEmbeddings",
+]
+
+
+# --------------------------------------------------------------------------
+# Capabilities
+# --------------------------------------------------------------------------
+
+
+class Capability(StrEnum):
+    """A thing a provider or a specific model can do.
+
+    Used in two distinct places, and they mean different things:
+
+      * `BaseProvider.capabilities` -- what the *vendor* offers.
+      * `ModelInfo.capabilities`    -- what *that model* offers.
+
+    Inception supports FIM; a given Inception model may not. Route on the
+    per-model set, gate the CLI on the per-provider one.
+    """
+
+    CHAT = "chat"
+    TOOLS = "tools"
+    VISION = "vision"
+    REASONING = "reasoning"
+    STRUCTURED_OUTPUT = "structured_output"
+    FIM = "fim"
+    EDIT = "edit"
+    EMBEDDINGS = "embeddings"
+
+
+# --------------------------------------------------------------------------
+# Value types
+# --------------------------------------------------------------------------
+
+
+@dataclass(frozen=True, slots=True)
+class ModelInfo:
+    """One model, normalised across vendors.
+
+    `raw` keeps the untouched vendor payload so provider-specific code can
+    reach fields the common shape drops, without another network call. It is
+    excluded from `repr` because vendor payloads are large.
+    """
+
+    id: str
+    provider: str
+    display_name: str | None = None
+    capabilities: frozenset[Capability] = frozenset()
+    context_window: int | None = None
+    max_output_tokens: int | None = None
+    raw: dict[str, Any] = field(default_factory=dict, repr=False, compare=False)
+
+    @property
+    def spec(self) -> str:
+        """Fully-qualified reference, e.g. ``inception:mercury-2``."""
+        return f"{self.provider}:{self.id}"
+
+    def supports(self, capability: Capability) -> bool:
+        return capability in self.capabilities
+
+    def __str__(self) -> str:
+        return self.spec
+
+
+@dataclass(frozen=True, slots=True)
+class Completion:
+    """A raw-SDK completion: the text, plus the usage the endpoint reported.
+
+    FIM and edit bypass LangChain, so nothing collects their token counts for
+    us. Returning a bare `str` threw them away -- and a call with no usage has
+    no cost, which silently defeats the whole point of a cost-driven router.
+
+    `usage` uses Langfuse's key names (input / output / total, plus optional
+    cached_input and reasoning) so it can be handed to `usage_details` without
+    translation.
+    """
+
+    text: str
+    usage: Mapping[str, int] | None = None
+
+    def __str__(self) -> str:
+        return self.text
+
+
+class ProviderStatus(StrEnum):
+    """Outcome of `BaseProvider.health_check`."""
+
+    OK = "ok"
+    NO_KEY = "no key"
+    AUTH_FAILED = "auth failed"
+    UNREACHABLE = "unreachable"
+    ERROR = "error"
+
+
+@dataclass(frozen=True, slots=True)
+class HealthReport:
+    """Per-provider result for the `doctor` / `models` commands.
+
+    Deliberately a value, not an exception: checking four providers should
+    report four outcomes, never abort on the first failure.
+    """
+
+    provider: str
+    status: ProviderStatus
+    model_count: int = 0
+    detail: str = ""
+
+    @property
+    def ok(self) -> bool:
+        return self.status is ProviderStatus.OK
+
+
+# --------------------------------------------------------------------------
+# Errors
+# --------------------------------------------------------------------------
+
+
+class ProviderError(Exception):
+    """Base for every provider failure.
+
+    Subclasses MUST translate vendor SDK exceptions into these before they
+    escape `_fetch_models`. That translation is the whole reason `health_check`
+    can stay vendor-agnostic -- it is the one piece of per-provider work that
+    is easy to forget.
+    """
+
+
+class AuthError(ProviderError):
+    """Key missing, malformed, or rejected by the vendor."""
+
+
+class ProviderUnavailable(ProviderError):
+    """Network failure, timeout, or vendor-side outage."""
+
+
+class ModelNotFound(ProviderError):
+    """Requested model id is not offered by this provider."""
+
+
+class CapabilityNotSupported(ProviderError):
+    """Provider or model cannot do the requested thing."""
+
+
+# --------------------------------------------------------------------------
+# Base provider
+# --------------------------------------------------------------------------
+
+
+#: Vendor SDK root modules whose exceptions can reach Otto unwrapped.
+#:
+#: A provider subclass translates its own SDK's errors (the invariant a few
+#: lines above), but the CHAT path does not go through the subclass at all --
+#: it goes through a LangChain class Otto does not own. `ChatOpenAI`,
+#: `ChatAnthropic` and `ChatGoogleGenerativeAI` raise raw `openai.RateLimitError`,
+#: `anthropic.APIStatusError`, google-genai errors and bare `httpx` errors
+#: straight out of `.stream()`.
+#:
+#: That is load-bearing because agent/pipeline/nodes.py's `_run_role` and
+#: `evaluator` catch only ProviderError around their tool loop. Before this,
+#: every chat model was Inception's own `ChatInception`, which translates; the
+#: moment a route points at another vendor, a 429 mid-solver stops being a
+#: clean edge back to the overseer and unwinds the entire graph instead.
+_VENDOR_SDK_ROOTS = frozenset({"openai", "anthropic", "google", "httpx", "httpx2"})
+
+#: Exception class-name fragments that mean "the vendor was unreachable", as
+#: opposed to "the vendor said no". Matched on the class name because the four
+#: SDKs above share no base class to check against.
+_UNAVAILABLE_MARKERS = ("Connection", "Timeout", "APIStatus", "ServiceUnavailable")
+
+
+def translate_unknown(exc: Exception, *, provider: str = "", model_id: str = "") -> ProviderError:
+    """Last-resort translation for an exception raised by a chat model Otto
+    does not own.
+
+    Deliberately duck-typed rather than importing the vendor SDKs: this module
+    is imported by agent/router/mapping.py's validation path, which a test runs
+    in a subprocess with every API key stripped, and which must stay free of
+    heavy vendor imports.
+    """
+    status = getattr(exc, "status_code", None) or getattr(exc, "code", None)
+    name = type(exc).__name__
+    detail = f"{name}: {exc}"
+
+    # Before the retirement check, because Anthropic phrases a refused
+    # temperature as "`temperature` is deprecated for this model" -- which
+    # reads as a retirement to the marker list below. The model is alive; it
+    # just takes no temperature, and nodes.py's _call has already learned
+    # that and retried. Treating it as retired would hide a live model for
+    # the rest of the run.
+    if looks_like_temperature_rejection(exc):
+        return ProviderError(detail)
+
+    # A vendor saying a model is permanently gone is worth remembering: the
+    # catalogue still lists it, so without this the next node in the same run
+    # resolves the same dead id and fails the same way.
+    if provider and model_id and looks_retired(exc):
+        note_retired(provider, model_id, detail)
+        return ModelNotFound(detail)
+
+    if status in (401, 403):
+        return AuthError(detail)
+    if status == 404:
+        return ModelNotFound(detail)
+    if status == 429 or (isinstance(status, int) and status >= 500):
+        return ProviderUnavailable(detail)
+    if any(marker in name for marker in _UNAVAILABLE_MARKERS):
+        return ProviderUnavailable(detail)
+    return ProviderError(detail)
+
+
+class BaseProvider(ABC):
+    """A single LLM vendor.
+
+    Subclasses set the class attributes below and implement the three
+    abstract methods. Nothing here performs I/O at construction time except
+    building the SDK client -- model discovery is lazy, so constructing a
+    provider is cheap and safe.
+    """
+
+    #: Short lowercase identifier used in model specs, e.g. ``"inception"``.
+    name: str = ""
+    #: Environment variable holding this vendor's API key.
+    env_var: str = ""
+    #: Optional default endpoint, for OpenAI-compatible vendors.
+    default_base_url: str | None = None
+    #: What the vendor offers. Narrow this per subclass.
+    capabilities: frozenset[Capability] = frozenset({Capability.CHAT})
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        *,
+        base_url: str | None = None,
+    ) -> None:
+        if not self.name or not self.env_var:
+            raise TypeError(
+                f"{type(self).__name__} must set both 'name' and 'env_var'"
+            )
+
+        key = api_key or os.environ.get(self.env_var)
+        if not key:
+            raise AuthError(f"{self.name}: {self.env_var} is not set")
+
+        self._api_key: str = key
+        self._base_url: str | None = base_url or self.default_base_url
+        self._models: list[ModelInfo] | None = None
+        self._client: Any = self._build_client()
+
+    # -- subclass responsibilities ----------------------------------------
+
+    @abstractmethod
+    def _build_client(self) -> Any:
+        """Construct and return the vendor SDK client.
+
+        Use `self._api_key` and `self._base_url` explicitly rather than
+        letting the SDK read the environment on its own -- an explicitly
+        passed key is testable and cannot be shadowed by a stale export.
+        """
+
+    @abstractmethod
+    def _fetch_models(self) -> list[ModelInfo]:
+        """Fetch every model this provider offers, as `ModelInfo`.
+
+        Called at most once per instance unless `refresh=True`. Set each
+        model's `capabilities` here -- it is the only place with enough
+        vendor context to know them.
+
+        Must raise `AuthError` on a rejected key, `ProviderUnavailable` on a
+        network or vendor fault, and `ProviderError` for anything else.
+        Letting a raw SDK exception escape breaks `health_check`.
+        """
+
+    @abstractmethod
+    def chat_model(self, model_id: str, **kwargs: Any) -> BaseChatModel:
+        """Return a LangChain chat model bound to `model_id`.
+
+        Do not implement chat by hand. Returning a `BaseChatModel` is what
+        keeps every provider interchangeable inside a LangGraph node and
+        visible to Langfuse.
+        """
+
+    # -- configuration ----------------------------------------------------
+
+    @classmethod
+    def is_configured(cls) -> bool:
+        """True if the key is present, without constructing the provider.
+
+        `__init__` raises on a missing key, so a `doctor` command that wants
+        to report rather than crash needs this cheap check first.
+        """
+        return bool(os.environ.get(cls.env_var))
+
+    @property
+    def masked_key(self) -> str:
+        """Key rendered for display. Never log or print `_api_key` itself."""
+        tail = self._api_key[-4:] if len(self._api_key) >= 4 else ""
+        return f"{'*' * 8}{tail}"
+
+    # -- model discovery --------------------------------------------------
+
+    def list_models(
+        self,
+        capability: Capability | None = None,
+        *,
+        refresh: bool = False,
+    ) -> list[ModelInfo]:
+        """Models offered, optionally narrowed to one capability.
+
+        Cached after the first call; pass `refresh=True` to re-fetch.
+        """
+        if self._models is None or refresh:
+            # Filtered here rather than at the routing layer so EVERY consumer
+            # -- routing, `otto models`, the doctor, health reports -- sees the
+            # same catalogue. A retired id that reaches Router._select resolves
+            # cleanly and then fails mid-run, which is the failure this exists
+            # to prevent (agent/router/llm_provider/retired.py).
+            self._models = [
+                m for m in self._fetch_models() if is_serviceable(self.name, m.id)
+            ]
+        if capability is None:
+            return list(self._models)
+        return [m for m in self._models if capability in m.capabilities]
+
+    def model_ids(self, capability: Capability | None = None) -> list[str]:
+        return [m.id for m in self.list_models(capability)]
+
+    def resolve_model(self, model_id: str) -> ModelInfo:
+        """Look up a model by id, raising `ModelNotFound` on a typo.
+
+        Call this before `chat_model` so a bad id fails with a useful list
+        instead of a vendor 404 halfway through a graph run.
+        """
+        for model in self.list_models():
+            if model.id == model_id:
+                return model
+        raise ModelNotFound(
+            f"{model_id!r} is not offered by {self.name}. "
+            f"Available: {', '.join(self.model_ids()) or '<none>'}"
+        )
+
+    # -- capability gating ------------------------------------------------
+
+    def supports(self, capability: Capability) -> bool:
+        return capability in self.capabilities
+
+    def require(self, capability: Capability) -> None:
+        """Guard clause for optional capabilities.
+
+        Prefer this over `isinstance(p, SupportsFIM)` at call sites -- it
+        yields an error naming the provider and the capability.
+        """
+        if not self.supports(capability):
+            raise CapabilityNotSupported(
+                f"{self.name} does not support {capability}"
+            )
+
+    # -- health -----------------------------------------------------------
+
+    def health_check(self) -> HealthReport:
+        """Verify the key actually works by listing models.
+
+        Env-var presence proves nothing -- only a real call distinguishes a
+        valid key from a revoked one. Never raises.
+        """
+        try:
+            models = self.list_models(refresh=True)
+        except AuthError as exc:
+            return HealthReport(self.name, ProviderStatus.AUTH_FAILED, detail=str(exc))
+        except ProviderUnavailable as exc:
+            return HealthReport(self.name, ProviderStatus.UNREACHABLE, detail=str(exc))
+        except ProviderError as exc:
+            return HealthReport(self.name, ProviderStatus.ERROR, detail=str(exc))
+        except Exception as exc:  # a subclass forgot to translate
+            return HealthReport(
+                self.name,
+                ProviderStatus.ERROR,
+                detail=f"untranslated {type(exc).__name__}: {exc}",
+            )
+        return HealthReport(self.name, ProviderStatus.OK, model_count=len(models))
+
+    @classmethod
+    def check(cls) -> HealthReport:
+        """Health check that tolerates a missing key and never raises."""
+        if not cls.is_configured():
+            return HealthReport(
+                cls.name or cls.__name__,
+                ProviderStatus.NO_KEY,
+                detail=f"{cls.env_var} is not set",
+            )
+        try:
+            return cls().health_check()
+        except ProviderError as exc:
+            return HealthReport(cls.name, ProviderStatus.ERROR, detail=str(exc))
+
+    # -- misc -------------------------------------------------------------
+
+    def __repr__(self) -> str:
+        loaded = "?" if self._models is None else str(len(self._models))
+        return (
+            f"<{type(self).__name__} name={self.name!r} "
+            f"models={loaded} caps={sorted(c.value for c in self.capabilities)}>"
+        )
+
+
+# --------------------------------------------------------------------------
+# Optional capabilities
+# --------------------------------------------------------------------------
+#
+# These are Protocols, not base-class methods, on purpose. A base method that
+# raises NotImplementedError lies in its own signature: the caller cannot tell
+# whether it will work without calling it. A Protocol lets both a static type
+# checker and `isinstance` answer the question up front.
+
+
+@runtime_checkable
+class SupportsFIM(Protocol):
+    """Fill-in-the-middle completion. Not an OpenAI-standard endpoint."""
+
+    def fim(
+        self,
+        model_id: str,
+        prefix: str,
+        suffix: str = "",
+        **kwargs: Any,
+    ) -> Completion: ...
+
+
+@runtime_checkable
+class SupportsEdit(Protocol):
+    """Next-edit prediction over a code region.
+
+    Note the absence of an `instruction` argument. Edit endpoints of this kind
+    infer the change from surrounding context -- the file, recently viewed
+    snippets, recent diffs, and cursor position -- rather than from a natural
+    language instruction. Providers that want an instruction should pass it
+    through `**kwargs` or expose a separate method.
+    """
+
+    def code_edit(
+        self,
+        model_id: str,
+        code_to_edit: str,
+        *,
+        current_file: str = "",
+        **kwargs: Any,
+    ) -> Completion: ...
+
+
+@runtime_checkable
+class SupportsEmbeddings(Protocol):
+    def embed(
+        self,
+        model_id: str,
+        texts: list[str],
+        **kwargs: Any,
+    ) -> list[list[float]]: ...

agent/router/llm_provider/custom.py

@@ -0,0 +1,152 @@
+"""Custom OpenAI-compatible endpoints: OpenRouter, a remote vLLM, Ollama, LM
+Studio -- anything that answers `GET /v1/models` and `POST /v1/chat/completions`.
+
+One factory, `openai_compatible(name)`, turns a NAME into a provider class
+that reads `<NAME>_API_KEY` and `<NAME>_BASE_URL` from the environment and
+otherwise behaves exactly like `OpenAIProvider` -- because it IS one, a
+subclass built with `type()`. The names live in ~/.otto/routes.json
+(agent/router/overrides.py registers each at startup); the key and URL rows
+live in the repo `.env` beside the built-in vendors' keys.
+
+Capability classification is a heuristic and says so, the same way
+`openai_provider._classify` does: a generic `/v1/models` returns ids and
+nothing else. The floor is CHAT+TOOLS; REASONING and VISION are guessed from
+well-known family names; a person can pin the truth per model in routes.json
+(`endpoints.<name>.capabilities`), which wins over the guess.
+"""
+from __future__ import annotations
+
+import os
+import re
+from collections.abc import Iterable, Mapping
+from typing import TYPE_CHECKING
+
+from agent.router.llm_provider.base import (
+    AuthError,
+    BaseProvider,
+    Capability,
+    HealthReport,
+    ProviderStatus,
+)
+
+if TYPE_CHECKING:  # pragma: no cover
+    from agent.router.llm_provider.openai_provider import OpenAIProvider
+
+__all__ = [
+    "NAME_RE", "BUILTIN", "validate_name", "env_prefix", "key_var", "url_var",
+    "classify_generic", "openai_compatible",
+]
+
+#: Lower-case, starts with a letter, up to 32 characters. The name becomes an
+#: environment-variable prefix (upper-cased, hyphens to underscores) and the
+#: provider half of a `provider:model` spec, so it has to be safe in both.
+NAME_RE = re.compile(r"^[a-z][a-z0-9_-]{1,31}$")
+BUILTIN = ("inception", "anthropic", "openai", "gemini")
+_RESERVED = BUILTIN + ("otto", "custom", "any", "none")
+
+
+def validate_name(name: str) -> None:
+    if not isinstance(name, str) or not NAME_RE.match(name):
+        raise ValueError(
+            f"{name!r} is not a usable endpoint name: lower-case letters, digits, "
+            f"'-' or '_', starting with a letter, at most 32 characters"
+        )
+    if name in _RESERVED:
+        raise ValueError(f"{name!r} is reserved")
+
+
+def env_prefix(name: str) -> str:
+    return name.upper().replace("-", "_")
+
+
+def key_var(name: str) -> str:
+    return f"{env_prefix(name)}_API_KEY"
+
+
+def url_var(name: str) -> str:
+    return f"{env_prefix(name)}_BASE_URL"
+
+
+_NON_CHAT = ("whisper", "tts", "dall-e", "moderation", "sora", "davinci", "babbage")
+_REASONING = re.compile(r"^(o\d|gpt-5|deepseek-r|qwq|.*-r1|.*think|.*reason)")
+_VISION_MARKERS = ("vision", "-vl", "llava", "gpt-4o", "gpt-4.1", "gpt-5",
+                   "claude", "gemini", "pixtral", "gemma-3")
+
+
+def classify_generic(model_id: str) -> frozenset[Capability]:
+    """Best-effort capabilities for an id from an unknown OpenAI-compatible
+    server. Heuristic -- see the module docstring."""
+    lowered = (model_id or "").lower()
+    vendor, _, tail = lowered.rpartition("/")     # OpenRouter's "vendor/model"
+    if "embed" in tail:
+        return frozenset({Capability.EMBEDDINGS})
+    if any(marker in tail for marker in _NON_CHAT):
+        return frozenset()
+    caps = {Capability.CHAT, Capability.TOOLS}
+    if _REASONING.match(tail) or vendor in ("anthropic", "google"):
+        caps.add(Capability.REASONING)
+    if any(marker in tail for marker in _VISION_MARKERS):
+        caps.add(Capability.VISION)
+    return frozenset(caps)
+
+
+def openai_compatible(
+    name: str,
+    *,
+    capability_overrides: Mapping[str, Iterable[str]] | None = None,
+) -> "type[OpenAIProvider]":
+    """A provider class for the endpoint called `name`.
+
+    The `openai` SDK import is inside the function on purpose: this module is
+    imported at startup by agent/router/overrides.py, and a routes.json that
+    names an endpoint must not make a machine without the SDK fail to start.
+    """
+    validate_name(name)
+    from agent.router.llm_provider.openai_provider import OpenAIProvider
+
+    overrides: dict[str, frozenset[Capability]] = {
+        model_id: frozenset(Capability(c) for c in caps)
+        for model_id, caps in (capability_overrides or {}).items()
+    }
+    key_name, url_name = key_var(name), url_var(name)
+
+    def __init__(self, api_key: str | None = None, *, base_url: str | None = None) -> None:
+        url = base_url or os.environ.get(url_name)
+        if not url:
+            # Without this an endpoint with a key but no URL would quietly
+            # send that key to api.openai.com.
+            raise AuthError(f"{name}: {url_name} is not set")
+        OpenAIProvider.__init__(self, api_key, base_url=url.rstrip("/"))
+
+    @classmethod
+    def is_configured(cls) -> bool:
+        # Servers that ignore keys (Ollama, a local vLLM) still need SOME
+        # value here -- the setup screen says so; "x" is fine.
+        return bool(os.environ.get(key_name)) and bool(os.environ.get(url_name))
+
+    @classmethod
+    def check(cls) -> HealthReport:
+        missing = [v for v in (key_name, url_name) if not os.environ.get(v)]
+        if missing:
+            return HealthReport(name, ProviderStatus.NO_KEY,
+                                detail=f"{' and '.join(missing)} not set")
+        return BaseProvider.check.__func__(cls)
+
+    def _classify(self, model_id: str) -> frozenset[Capability]:
+        return overrides.get(model_id) or classify_generic(model_id)
+
+    namespace = {
+        "__module__": __name__,
+        "__doc__": f"OpenAI-compatible endpoint {name!r} ({url_name}).",
+        "name": name,
+        "env_var": key_name,
+        "base_url_var": url_name,
+        "default_base_url": None,
+        "capabilities": frozenset({Capability.CHAT, Capability.TOOLS,
+                                   Capability.VISION, Capability.REASONING}),
+        "__init__": __init__,
+        "is_configured": is_configured,
+        "check": check,
+        "_classify": _classify,
+    }
+    return type(f"CustomProvider_{env_prefix(name)}", (OpenAIProvider,), namespace)