thinkless 0.2.0__py3-none-any.whl

thinkless/llm/anthropic.py

@@ -0,0 +1,126 @@
+"""Claude through the official Anthropic SDK."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Sequence
+from typing import Any
+
+from ..decision import Usage
+from .base import LLM, Completion, Message
+
+__all__ = ["AnthropicLLM"]
+
+FALLBACK_BETA = "server-side-fallback-2026-07-01"
+
+# Model families for which server-side refusal fallbacks are documented.
+FALLBACK_FAMILIES = ("claude-opus-5", "claude-fable-5", "claude-mythos-5")
+
+
+class AnthropicLLM(LLM):
+    """Claude models via the Messages API.
+
+    Args:
+        model: Claude model id. Defaults to ``claude-opus-5``. For
+            high-volume reply generation, ``claude-sonnet-5`` or
+            ``claude-haiku-4-5`` are the cheaper options.
+        api_key: Defaults to the SDK's credential resolution
+            (``ANTHROPIC_API_KEY``, ``ANTHROPIC_AUTH_TOKEN`` or an
+            ``ant auth login`` profile).
+        effort: Optional ``output_config.effort`` (``low`` to ``max``).
+            ``low`` suits short replies and decision questions. Supported on
+            Claude Opus 4.6 and later, Sonnet 5 and Fable; Haiku 4.5 and
+            Sonnet 4.5 reject it, so leave it unset for those.
+        fallbacks: Server-side refusal fallbacks (``fallbacks="default"``):
+            if a safety classifier declines the request, the API re-runs it
+            on the recommended fallback model in the same call. ``None`` (the
+            default) turns them on for the model families that document them
+            (Opus 5 and later, Fable 5, Mythos 5) and off otherwise. Available
+            on the Claude API and Claude Platform on AWS; pass ``False`` when
+            routing through Bedrock, Vertex AI or Foundry.
+        client: A preconfigured ``anthropic.Anthropic`` client (tests, proxies,
+            custom base URL).
+
+    Sampling parameters are not sent: current Claude models reject
+    ``temperature`` and control depth through ``effort`` instead.
+
+    Requires the ``anthropic`` extra.
+    """
+
+    provider = "anthropic"
+
+    def __init__(
+        self,
+        model: str = "claude-opus-5",
+        *,
+        api_key: str | None = None,
+        effort: str | None = None,
+        fallbacks: bool | None = None,
+        timeout: float = 120.0,
+        max_retries: int = 2,
+        client: Any | None = None,
+    ) -> None:
+        self.model = model
+        self._effort = effort
+        self._fallbacks = model.startswith(FALLBACK_FAMILIES) if fallbacks is None else fallbacks
+        if client is not None:
+            self._client = client
+            return
+        try:
+            import anthropic
+        except ImportError as exc:  # pragma: no cover - depends on the extra
+            raise ImportError(
+                'AnthropicLLM needs the anthropic extra: pip install "thinkless[anthropic]"'
+            ) from exc
+        kwargs: dict[str, Any] = {"timeout": timeout, "max_retries": max_retries}
+        if api_key:
+            kwargs["api_key"] = api_key
+        self._client = anthropic.Anthropic(**kwargs)
+
+    def complete(
+        self,
+        messages: Sequence[Message],
+        *,
+        system: str | None = None,
+        max_tokens: int = 1024,
+        temperature: float | None = None,
+        json_mode: bool = False,
+    ) -> Completion:
+        kwargs: dict[str, Any] = {
+            "model": self.model,
+            "max_tokens": max_tokens,
+            "messages": [dict(m) for m in messages],
+        }
+        if system:
+            kwargs["system"] = system
+        if self._effort:
+            kwargs["output_config"] = {"effort": self._effort}
+        started = time.perf_counter()
+        if self._fallbacks:
+            response = self._client.beta.messages.create(
+                betas=[FALLBACK_BETA], fallbacks="default", **kwargs
+            )
+        else:
+            response = self._client.messages.create(**kwargs)
+        latency = (time.perf_counter() - started) * 1000.0
+
+        # A refusal arrives as HTTP 200 with stop_reason "refusal"; content may
+        # be empty or partial, so it is never treated as an answer.
+        if response.stop_reason == "refusal":
+            text = ""
+        else:
+            text = "".join(
+                block.text for block in response.content if getattr(block, "type", None) == "text"
+            )
+        usage = response.usage
+        return Completion(
+            text=text,
+            model=getattr(response, "model", None) or self.model,
+            usage=Usage(
+                input_tokens=getattr(usage, "input_tokens", 0) or 0,
+                output_tokens=getattr(usage, "output_tokens", 0) or 0,
+            ),
+            latency_ms=latency,
+            stop_reason=response.stop_reason,
+            raw={"request_id": getattr(response, "_request_id", None)},
+        )

thinkless/llm/base.py

@@ -0,0 +1,82 @@
+"""The reasoning plane interface."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Sequence
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..decision import Usage
+
+__all__ = ["LLM", "Completion", "Message", "as_messages"]
+
+Message = dict[str, str]
+
+
+def as_messages(prompt: str | Sequence[Message]) -> list[Message]:
+    """Accept a plain prompt or a list of ``{"role", "content"}`` messages."""
+    if isinstance(prompt, str):
+        return [{"role": "user", "content": prompt}]
+    return [dict(m) for m in prompt]
+
+
+class Completion(BaseModel):
+    """The result of one generation.
+
+    ``cost_usd`` is the cost the backend reported for this call (OpenRouter
+    does). When it is ``None`` the engine estimates cost from the price table.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    text: str
+    model: str
+    usage: Usage = Field(default_factory=Usage)
+    latency_ms: float = 0.0
+    stop_reason: str | None = None
+    cost_usd: float | None = None
+    raw: dict[str, Any] | None = None
+
+
+class LLM(ABC):
+    """A generative model.
+
+    Implementations wrap one backend. ``provider`` identifies the backend in
+    traces and in the price table (``anthropic``, ``openai``, ``local``, ...).
+
+    Args of :meth:`complete`:
+        messages: Conversation turns, oldest first.
+        system: Optional system prompt.
+        max_tokens: Upper bound on generated tokens.
+        temperature: Sampling temperature. ``None`` leaves the backend
+            default; some hosted models reject the parameter entirely, and
+            their implementations ignore it.
+        json_mode: Ask the backend for a JSON object when it supports that.
+    """
+
+    provider: str = "llm"
+    model: str = "unknown"
+
+    @property
+    def name(self) -> str:
+        return f"{self.provider}:{self.model}"
+
+    @abstractmethod
+    def complete(
+        self,
+        messages: Sequence[Message],
+        *,
+        system: str | None = None,
+        max_tokens: int = 512,
+        temperature: float | None = None,
+        json_mode: bool = False,
+    ) -> Completion:
+        raise NotImplementedError
+
+    def warmup(self) -> None:  # noqa: B027 - optional hook
+        """Load weights or open connections ahead of the first call."""
+
+    def close(self) -> None:  # noqa: B027 - optional hook
+        """Release resources."""

thinkless/llm/factory.py

@@ -0,0 +1,91 @@
+"""Build an LLM from a short spec string, as used by the CLI."""
+
+from __future__ import annotations
+
+from .base import LLM
+
+__all__ = ["DEFAULT_LOCAL_MODEL", "from_spec"]
+
+DEFAULT_LOCAL_MODEL = "Qwen/Qwen3-1.7B"
+
+BACKENDS = ("local", "anthropic", "openrouter", "openai", "ollama", "vllm")
+
+
+REASONING = ("default", "off", "minimal", "low", "medium", "high")
+
+
+def from_spec(
+    spec: str,
+    *,
+    device: str = "auto",
+    base_url: str | None = None,
+    reasoning: str | None = None,
+) -> LLM:
+    """Create an LLM from ``backend[:model]``.
+
+    Examples:
+        ``local`` (Qwen3-1.7B in-process), ``local:Qwen/Qwen3-4B``,
+        ``anthropic`` (Claude Opus 5), ``anthropic:claude-haiku-4-5``,
+        ``openrouter:qwen/qwen3.7-flash``, ``openrouter:anthropic/claude-haiku-4.5``,
+        ``openai:<model>``, ``ollama:qwen3:8b``, ``vllm:<model>``.
+
+    Args:
+        spec: Backend name, optionally followed by a colon and a model id.
+        device: Device for the ``local`` backend.
+        base_url: Server URL for ``openai``, ``ollama`` and ``vllm``.
+        reasoning: ``off``, ``minimal``, ``low``, ``medium`` or ``high``;
+            ``None`` or ``default`` keeps the model's own default. Mapped to
+            OpenRouter's ``reasoning`` option, Anthropic's ``effort``, and the
+            thinking switch of local chat templates. Short decisions rarely
+            need reasoning, and hidden reasoning tokens are billed and count
+            against the output limit.
+    """
+    if reasoning not in (None, *REASONING):
+        raise ValueError(f"reasoning must be one of {', '.join(REASONING)}")
+    level = None if reasoning in (None, "default") else reasoning
+    backend, _, model = spec.partition(":")
+    backend = backend.strip().lower()
+    model = model.strip()
+    if backend == "local":
+        from .local import TransformersLLM
+
+        return TransformersLLM(
+            model or DEFAULT_LOCAL_MODEL, device=device, enable_thinking=level not in (None, "off")
+        )
+    if backend == "anthropic":
+        from .anthropic import AnthropicLLM
+
+        effort = {"minimal": "low", "off": None}.get(level or "", level)
+        return AnthropicLLM(model or "claude-opus-5", effort=effort)
+    if backend == "openrouter":
+        from .openrouter import OpenRouterLLM
+
+        if not model:
+            raise ValueError(
+                "the openrouter backend needs a model slug, for example "
+                "openrouter:qwen/qwen3.7-flash"
+            )
+        options = (
+            None if level is None else {"enabled": False} if level == "off" else {"effort": level}
+        )
+        return OpenRouterLLM(model, reasoning=options)
+    if backend in ("openai", "ollama", "vllm"):
+        from .openai_compat import OpenAICompatibleLLM
+
+        if backend == "openai":
+            if not model:
+                raise ValueError("the openai backend needs a model, for example openai:<model-id>")
+            return OpenAICompatibleLLM(model, base_url=base_url)
+        defaults = {
+            "ollama": ("qwen3:4b", "http://localhost:11434/v1"),
+            "vllm": ("", "http://localhost:8000/v1"),
+        }
+        default_model, default_url = defaults[backend]
+        if not (model or default_model):
+            raise ValueError(
+                f"the {backend} backend needs a model, for example {backend}:<model-id>"
+            )
+        return OpenAICompatibleLLM(
+            model or default_model, base_url=base_url or default_url, provider=backend
+        )
+    raise ValueError(f"unknown LLM backend {backend!r}; choose one of {', '.join(BACKENDS)}")

thinkless/llm/local.py

@@ -0,0 +1,168 @@
+"""Local generation with Hugging Face Transformers."""
+
+from __future__ import annotations
+
+import copy
+import re
+import threading
+import time
+from collections.abc import Sequence
+from typing import Any
+
+from .._hub import ensure_downloaded
+from ..decision import Usage
+from ..logs import get_logger
+from ..settings import resolve_device
+from .base import LLM, Completion, Message
+
+__all__ = ["TransformersLLM"]
+
+logger = get_logger("llm.local")
+
+_THINK = re.compile(r"<think>.*?</think>\s*", re.DOTALL)
+
+
+class TransformersLLM(LLM):
+    """Runs a chat model in-process with ``transformers``.
+
+    This is the zero-setup option: no server, no API key. For throughput, run
+    the same weights behind Ollama, vLLM or llama.cpp and use
+    :class:`~thinkless.llm.OpenAICompatibleLLM` instead; a serving engine is
+    several times faster than the plain ``generate`` loop used here.
+
+    Args:
+        model: Hugging Face model id or local path.
+        device: ``auto``, ``cpu``, ``cuda``, ``cuda:N`` or ``mps``.
+        dtype: ``auto`` picks bfloat16 on CUDA, float16 on MPS, float32 on CPU.
+        enable_thinking: Passed to chat templates that support a thinking
+            switch (Qwen3 and later). Off by default: decisions and short
+            replies do not benefit from long reasoning traces.
+
+    Requires the ``local-llm`` extra.
+    """
+
+    provider = "local"
+
+    def __init__(
+        self,
+        model: str = "Qwen/Qwen3-1.7B",
+        *,
+        device: str = "auto",
+        dtype: str = "auto",
+        enable_thinking: bool = False,
+    ) -> None:
+        self.model = model
+        self._device_request = device
+        self._dtype_request = dtype
+        self._enable_thinking = enable_thinking
+        self._lock = threading.Lock()
+        self._tokenizer: Any = None
+        self._model: Any = None
+        self.device: str | None = None
+
+    def warmup(self) -> None:
+        """Load the weights and generate one token, so the first request is not slow."""
+        self._load()
+        self.complete(
+            [{"role": "user", "content": "Reply with a short greeting for a customer."}],
+            max_tokens=16,
+        )
+
+    def _load(self) -> None:
+        if self._model is not None:
+            return
+        with self._lock:
+            if self._model is not None:
+                return
+            try:
+                import torch
+                from transformers import AutoModelForCausalLM, AutoTokenizer
+            except ImportError as exc:  # pragma: no cover - depends on the extra
+                raise ImportError(
+                    'TransformersLLM needs the local-llm extra: pip install "thinkless[local-llm]"'
+                ) from exc
+            device = resolve_device(self._device_request)
+            dtype: Any
+            if self._dtype_request != "auto":
+                dtype = getattr(torch, self._dtype_request)
+            elif device.startswith("cuda"):
+                dtype = torch.bfloat16
+            elif device == "mps":
+                dtype = torch.float16
+            else:
+                dtype = torch.float32
+            started = time.perf_counter()
+            ensure_downloaded(self.model)
+            tokenizer = AutoTokenizer.from_pretrained(self.model)
+            model: Any = AutoModelForCausalLM.from_pretrained(self.model, dtype=dtype)
+            model.to(device)
+            model.eval()
+            config: Any = copy.deepcopy(model.generation_config)
+            # Greedy decoding by default; clear sampling fields so transformers
+            # does not warn about unused flags on every call.
+            config.do_sample = False
+            config.temperature = None
+            config.top_p = None
+            config.top_k = None
+            if config.pad_token_id is None:
+                config.pad_token_id = tokenizer.pad_token_id or tokenizer.eos_token_id
+            self._generation_config = config
+            self._tokenizer = tokenizer
+            self._model = model
+            self.device = device
+            logger.info(
+                "loaded %s on %s in %.1f s", self.model, device, time.perf_counter() - started
+            )
+
+    def complete(
+        self,
+        messages: Sequence[Message],
+        *,
+        system: str | None = None,
+        max_tokens: int = 512,
+        temperature: float | None = None,
+        json_mode: bool = False,
+    ) -> Completion:
+        self._load()
+        import torch
+
+        chat: list[Message] = []
+        if system:
+            chat.append({"role": "system", "content": system})
+        chat.extend(messages)
+        started = time.perf_counter()
+        with self._lock:
+            inputs = self._tokenizer.apply_chat_template(
+                chat,
+                add_generation_prompt=True,
+                return_tensors="pt",
+                return_dict=True,
+                enable_thinking=self._enable_thinking,
+            ).to(self.device)
+            config: Any = copy.deepcopy(self._generation_config)
+            config.max_new_tokens = max_tokens
+            if temperature:
+                config.do_sample = True
+                config.temperature = temperature
+            with torch.inference_mode():
+                # use_model_defaults=False stops transformers from swapping our
+                # greedy settings for the checkpoint's sampling defaults.
+                output = self._model.generate(
+                    **inputs, generation_config=config, use_model_defaults=False
+                )
+            prompt_len = int(inputs["input_ids"].shape[1])
+            new_tokens = output[0][prompt_len:]
+            text = self._tokenizer.decode(new_tokens, skip_special_tokens=True)
+        text = _THINK.sub("", text).strip()
+        n_out = int(new_tokens.shape[0])
+        return Completion(
+            text=text,
+            model=self.model,
+            usage=Usage(input_tokens=prompt_len, output_tokens=n_out),
+            latency_ms=(time.perf_counter() - started) * 1000.0,
+            stop_reason="max_tokens" if n_out >= max_tokens else "end_turn",
+        )
+
+    def close(self) -> None:
+        self._model = None
+        self._tokenizer = None

thinkless/llm/openai_compat.py

@@ -0,0 +1,165 @@
+"""Any server that speaks the OpenAI Chat Completions API."""
+
+from __future__ import annotations
+
+import os
+import time
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from ..decision import Usage
+from ..logs import get_logger
+from .base import LLM, Completion, Message
+
+__all__ = ["OpenAICompatibleLLM", "reported_cost"]
+
+logger = get_logger("llm.openai")
+
+
+def reported_cost(usage: Any) -> float | None:
+    """The dollar cost a server reports in ``usage``, if it reports one.
+
+    OpenRouter adds ``usage.cost`` to every response. The OpenAI SDK keeps
+    fields it does not know in ``model_extra``, so both places are checked.
+    """
+    if usage is None:
+        return None
+    value = getattr(usage, "cost", None)
+    if value is None:
+        extra = getattr(usage, "model_extra", None) or {}
+        value = extra.get("cost")
+    try:
+        return None if value is None else float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+class OpenAICompatibleLLM(LLM):
+    """Chat Completions client for OpenAI and compatible servers.
+
+    Covers OpenAI itself plus Ollama, vLLM, LM Studio, llama.cpp server,
+    Groq, Together and any other server that implements
+    ``/v1/chat/completions``. For OpenRouter, use
+    :class:`~thinkless.llm.OpenRouterLLM`, which sets the endpoint, key and
+    reported cost for you.
+
+    Args:
+        model: Model id as the server knows it.
+        base_url: Server URL, for example ``http://localhost:11434/v1`` for
+            Ollama. ``None`` targets api.openai.com.
+        api_key: Defaults to ``OPENAI_API_KEY``. Local servers usually accept
+            any value.
+        provider: Name used in traces and for price lookup. Use ``ollama``,
+            ``vllm`` or ``lmstudio`` for local servers so their cost is 0.
+        token_param: ``max_completion_tokens`` (OpenAI's current name) or
+            ``max_tokens`` (what most compatible servers accept). Chosen from
+            ``base_url`` when omitted.
+        supports_json_mode: Whether to send ``response_format`` when a caller
+            asks for JSON. If the server rejects it, the client retries once
+            without it and stops sending it.
+        default_headers: Extra HTTP headers sent with every request.
+        extra_body: Extra fields merged into every request body, for
+            server-specific options.
+
+    Requires the ``openai`` extra.
+    """
+
+    def __init__(
+        self,
+        model: str,
+        *,
+        base_url: str | None = None,
+        api_key: str | None = None,
+        provider: str = "openai",
+        token_param: str | None = None,
+        supports_json_mode: bool = True,
+        default_headers: Mapping[str, str] | None = None,
+        extra_body: Mapping[str, Any] | None = None,
+        timeout: float = 120.0,
+        max_retries: int = 2,
+        client: Any | None = None,
+    ) -> None:
+        self.model = model
+        self.provider = provider
+        self._token_param = token_param or ("max_tokens" if base_url else "max_completion_tokens")
+        self._json_mode = supports_json_mode
+        self._extra_body = dict(extra_body or {})
+        if client is not None:
+            self._client = client
+            return
+        try:
+            from openai import OpenAI
+        except ImportError as exc:  # pragma: no cover - depends on the extra
+            raise ImportError(
+                'OpenAICompatibleLLM needs the openai extra: pip install "thinkless[openai]"'
+            ) from exc
+        key = api_key or os.environ.get("OPENAI_API_KEY") or ("unused" if base_url else None)
+        self._client = OpenAI(
+            base_url=base_url,
+            api_key=key,
+            timeout=timeout,
+            max_retries=max_retries,
+            default_headers=dict(default_headers) if default_headers else None,
+        )
+
+    def complete(
+        self,
+        messages: Sequence[Message],
+        *,
+        system: str | None = None,
+        max_tokens: int = 512,
+        temperature: float | None = None,
+        json_mode: bool = False,
+    ) -> Completion:
+        chat: list[dict[str, Any]] = []
+        if system:
+            chat.append({"role": "system", "content": system})
+        chat.extend(dict(m) for m in messages)
+        kwargs: dict[str, Any] = {
+            "model": self.model,
+            "messages": chat,
+            self._token_param: max_tokens,
+        }
+        if temperature is not None:
+            kwargs["temperature"] = temperature
+        if self._extra_body:
+            kwargs["extra_body"] = dict(self._extra_body)
+        use_json = json_mode and self._json_mode
+        if use_json:
+            kwargs["response_format"] = {"type": "json_object"}
+        started = time.perf_counter()
+        try:
+            response = self._client.chat.completions.create(**kwargs)
+        except Exception as exc:
+            if not (use_json and _rejects_response_format(exc)):
+                raise
+            logger.warning(
+                "%s rejected response_format; retrying without JSON mode for this client",
+                self.model,
+            )
+            self._json_mode = False
+            kwargs.pop("response_format", None)
+            response = self._client.chat.completions.create(**kwargs)
+        latency = (time.perf_counter() - started) * 1000.0
+        choice = response.choices[0]
+        usage = response.usage
+        return Completion(
+            text=choice.message.content or "",
+            model=getattr(response, "model", None) or self.model,
+            usage=Usage(
+                input_tokens=getattr(usage, "prompt_tokens", 0) or 0,
+                output_tokens=getattr(usage, "completion_tokens", 0) or 0,
+            ),
+            latency_ms=latency,
+            stop_reason=choice.finish_reason,
+            cost_usd=reported_cost(usage),
+        )
+
+
+def _rejects_response_format(exc: Exception) -> bool:
+    """True for a client error that names response_format (or JSON mode)."""
+    status = getattr(exc, "status_code", None)
+    if status is not None and not 400 <= int(status) < 500:
+        return False
+    text = str(exc).lower()
+    return "response_format" in text or "json_object" in text or "json mode" in text