thinkless 0.2.0__py3-none-any.whl

thinkless/llm/openrouter.py

@@ -0,0 +1,78 @@
+"""OpenRouter: hundreds of models from every major lab behind one API key."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from typing import Any
+
+from .openai_compat import OpenAICompatibleLLM
+
+__all__ = ["OPENROUTER_URL", "OpenRouterLLM"]
+
+OPENROUTER_URL = "https://openrouter.ai/api/v1"
+PROJECT_URL = "https://github.com/inboxpraveen/ThinkLess"
+
+
+class OpenRouterLLM(OpenAICompatibleLLM):
+    """Models served through OpenRouter.
+
+    OpenRouter routes one OpenAI-compatible API to models from Anthropic,
+    OpenAI, Google, Qwen, DeepSeek, Mistral, Meta and others, so a single key
+    is enough to compare reasoning models. It reports the billed cost of every
+    call, which ThinkLess records in traces instead of an estimate.
+
+    Args:
+        model: OpenRouter model slug, for example ``qwen/qwen3.7-flash`` or
+            ``anthropic/claude-haiku-4.5``.
+        api_key: Defaults to ``OPENROUTER_API_KEY``.
+        reasoning: OpenRouter's ``reasoning`` options, for example
+            ``{"effort": "low"}`` or ``{"enabled": False}``. Left unset, each
+            model uses its own default. Short decisions and replies rarely
+            need long reasoning, and reasoning tokens are billed as output.
+        provider_preferences: OpenRouter's ``provider`` routing options, for
+            example ``{"sort": "price"}`` or ``{"require_parameters": True}``.
+        app_name, app_url: Attribution headers OpenRouter shows in its
+            dashboards. Set either to ``None`` to omit it.
+        **kwargs: Passed to :class:`OpenAICompatibleLLM`.
+
+    Requires the ``openai`` extra.
+    """
+
+    def __init__(
+        self,
+        model: str,
+        *,
+        api_key: str | None = None,
+        reasoning: Mapping[str, Any] | None = None,
+        provider_preferences: Mapping[str, Any] | None = None,
+        app_name: str | None = "ThinkLess",
+        app_url: str | None = PROJECT_URL,
+        **kwargs: Any,
+    ) -> None:
+        key = api_key or os.environ.get("OPENROUTER_API_KEY")
+        if not key and kwargs.get("client") is None:
+            raise ValueError(
+                "OpenRouter needs an API key: set OPENROUTER_API_KEY (a .env file works "
+                "with the CLI) or pass api_key="
+            )
+        headers: dict[str, str] = {}
+        if app_url:
+            headers["HTTP-Referer"] = app_url
+        if app_name:
+            headers["X-Title"] = app_name
+        extra: dict[str, Any] = dict(kwargs.pop("extra_body", None) or {})
+        if reasoning is not None:
+            extra["reasoning"] = dict(reasoning)
+        if provider_preferences is not None:
+            extra["provider"] = dict(provider_preferences)
+        super().__init__(
+            model,
+            base_url=OPENROUTER_URL,
+            api_key=key,
+            provider="openrouter",
+            token_param=kwargs.pop("token_param", "max_tokens"),
+            default_headers=headers,
+            extra_body=extra,
+            **kwargs,
+        )

thinkless/llm/scripted.py

@@ -0,0 +1,75 @@
+"""A deterministic LLM for tests, examples and dry runs."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable, Sequence
+
+from ..decision import Usage
+from .base import LLM, Completion, Message
+
+__all__ = ["ScriptedLLM"]
+
+Responder = Callable[[Sequence[Message], "str | None"], str]
+
+
+def _approx_tokens(text: str) -> int:
+    # Roughly four characters per token for English text.
+    return max(1, len(text) // 4)
+
+
+class ScriptedLLM(LLM):
+    """Returns canned responses.
+
+    Args:
+        responses: Either a list of strings returned in order (the last one
+            repeats), or a callable ``(messages, system) -> str``.
+        model: Name reported in traces.
+        latency_ms: Artificial delay, to make dry-run traces look realistic.
+
+    Token usage is approximated from character counts.
+    """
+
+    provider = "scripted"
+
+    def __init__(
+        self,
+        responses: Sequence[str] | Responder,
+        *,
+        model: str = "scripted",
+        latency_ms: float = 0.0,
+    ) -> None:
+        self.model = model
+        self._responses = responses
+        self._index = 0
+        self._latency_ms = latency_ms
+        self.calls: list[tuple[list[Message], str | None]] = []
+
+    def complete(
+        self,
+        messages: Sequence[Message],
+        *,
+        system: str | None = None,
+        max_tokens: int = 512,
+        temperature: float | None = None,
+        json_mode: bool = False,
+    ) -> Completion:
+        started = time.perf_counter()
+        self.calls.append((list(messages), system))
+        if callable(self._responses):
+            text = self._responses(messages, system)
+        else:
+            if not self._responses:
+                raise ValueError("ScriptedLLM has no responses")
+            text = self._responses[min(self._index, len(self._responses) - 1)]
+            self._index += 1
+        if self._latency_ms:
+            time.sleep(self._latency_ms / 1000.0)
+        prompt = (system or "") + "".join(m.get("content", "") for m in messages)
+        return Completion(
+            text=text,
+            model=self.model,
+            usage=Usage(input_tokens=_approx_tokens(prompt), output_tokens=_approx_tokens(text)),
+            latency_ms=(time.perf_counter() - started) * 1000.0,
+            stop_reason="end_turn",
+        )

thinkless/logs.py

@@ -0,0 +1,68 @@
+"""Logging setup.
+
+The library only ever logs through the ``thinkless`` logger and installs a
+``NullHandler``, so it stays silent unless the application configures logging.
+:func:`configure_logging` is a convenience for scripts and the CLI.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+import time
+from typing import Any
+
+__all__ = ["JSONFormatter", "configure_logging", "get_logger"]
+
+logging.getLogger("thinkless").addHandler(logging.NullHandler())
+
+
+def get_logger(name: str) -> logging.Logger:
+    """A child of the ``thinkless`` logger."""
+    return logging.getLogger(name if name.startswith("thinkless") else f"thinkless.{name}")
+
+
+class JSONFormatter(logging.Formatter):
+    """One JSON object per line, for log shippers."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, Any] = {
+            "ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created))
+            + f".{int(record.msecs):03d}Z",
+            "level": record.levelname.lower(),
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+        if record.exc_info:
+            payload["exc_info"] = self.formatException(record.exc_info)
+        for key in ("trace_id", "span_id", "provider", "question"):
+            value = getattr(record, key, None)
+            if value is not None:
+                payload[key] = value
+        return json.dumps(payload, ensure_ascii=False)
+
+
+def configure_logging(level: str | int = "INFO", *, json_format: bool = False) -> None:
+    """Attach a handler to the ``thinkless`` logger.
+
+    Args:
+        level: Log level name or number.
+        json_format: Emit JSON lines instead of Rich-formatted text.
+    """
+    logger = logging.getLogger("thinkless")
+    for existing in list(logger.handlers):
+        if not isinstance(existing, logging.NullHandler):
+            logger.removeHandler(existing)
+    handler: logging.Handler
+    if json_format:
+        handler = logging.StreamHandler(sys.stderr)
+        handler.setFormatter(JSONFormatter())
+    else:
+        from rich.logging import RichHandler
+
+        handler = RichHandler(show_path=False, rich_tracebacks=True, markup=False)
+        handler.setFormatter(logging.Formatter("%(message)s"))
+    logger.addHandler(handler)
+    logger.setLevel(level if isinstance(level, int) else level.upper())
+    logger.propagate = False

thinkless/pricing.py

@@ -0,0 +1,100 @@
+"""Cost estimation from a price table."""
+
+from __future__ import annotations
+
+import os
+import sys
+from functools import lru_cache
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel
+
+from .decision import Usage
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:  # pragma: no cover
+    import tomli as tomllib
+
+__all__ = ["Price", "PriceTable", "default_prices"]
+
+LOCAL_PROVIDERS = frozenset(
+    {
+        "local",
+        "transformers",
+        "rules",
+        "laya",
+        "gliner",
+        "scripted",
+        "ollama",
+        "vllm",
+        "lmstudio",
+        "llamacpp",
+    }
+)
+
+
+class Price(BaseModel):
+    provider: str
+    model: str
+    input: float
+    output: float
+    match: str = "exact"
+    source: str | None = None
+
+    def matches(self, provider: str, model: str) -> bool:
+        if provider != self.provider:
+            return False
+        if self.match == "prefix":
+            return model.startswith(self.model)
+        return model == self.model
+
+
+class PriceTable:
+    """Looks up per-token prices and turns usage into dollars."""
+
+    def __init__(self, prices: list[Price], as_of: str | None = None) -> None:
+        # Longest model id first so a specific entry wins over a prefix.
+        self.prices = sorted(prices, key=lambda p: len(p.model), reverse=True)
+        self.as_of = as_of
+
+    @classmethod
+    def from_toml(cls, text: str) -> PriceTable:
+        data: dict[str, Any] = tomllib.loads(text)
+        return cls([Price(**entry) for entry in data.get("price", [])], data.get("as_of"))
+
+    @classmethod
+    def load(cls, path: str | Path | None = None) -> PriceTable:
+        """Load a price table from ``path``, ``$THINKLESS_PRICING`` or the bundled file."""
+        target = path or os.environ.get("THINKLESS_PRICING")
+        if target:
+            return cls.from_toml(Path(target).read_text(encoding="utf-8"))
+        bundled = resources.files("thinkless").joinpath("data/pricing.toml")
+        return cls.from_toml(bundled.read_text(encoding="utf-8"))
+
+    def find(self, provider: str, model: str) -> Price | None:
+        for price in self.prices:
+            if price.matches(provider, model):
+                return price
+        return None
+
+    def cost(self, provider: str, model: str | None, usage: Usage) -> tuple[float, bool]:
+        """Dollar cost of ``usage`` and whether the price was known.
+
+        Local providers always cost 0 and count as known.
+        """
+        if provider in LOCAL_PROVIDERS:
+            return 0.0, True
+        price = self.find(provider, model or "")
+        if price is None:
+            return 0.0, False
+        dollars = (usage.input_tokens * price.input + usage.output_tokens * price.output) / 1e6
+        return dollars, True
+
+
+@lru_cache(maxsize=1)
+def default_prices() -> PriceTable:
+    """The process-wide price table (cached)."""
+    return PriceTable.load()

thinkless/providers/__init__.py

@@ -0,0 +1,43 @@
+"""Decision providers.
+
+Local model providers are imported lazily, so the core package does not pull
+in torch or model libraries until one is used.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from .base import DecisionProvider, ProviderResult, State, render_state
+from .llm import LLMDecider
+from .rules import Rules
+from .systemone import SystemOne, SystemOneError
+
+if TYPE_CHECKING:
+    from .gliner import GLiNER
+    from .hf import HFClassifier
+    from .laya import Laya
+
+__all__ = [
+    "DecisionProvider",
+    "GLiNER",
+    "HFClassifier",
+    "LLMDecider",
+    "Laya",
+    "ProviderResult",
+    "Rules",
+    "State",
+    "SystemOne",
+    "SystemOneError",
+    "render_state",
+]
+
+_LAZY = {"GLiNER": ".gliner", "HFClassifier": ".hf", "Laya": ".laya"}
+
+
+def __getattr__(name: str) -> Any:
+    if name in _LAZY:
+        from importlib import import_module
+
+        return getattr(import_module(_LAZY[name], __name__), name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

thinkless/providers/base.py

@@ -0,0 +1,108 @@
+"""The decision provider interface."""
+
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from collections.abc import Mapping, Sequence
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..decision import Answer, Plane, Usage
+from ..questions import Kind, Question
+
+__all__ = ["DecisionProvider", "ProviderResult", "State", "render_state"]
+
+State = str | Mapping[str, Any] | Sequence[Any]
+"""What a question is asked about: text, a JSON-like object, or a list of either."""
+
+
+def render_state(state: State) -> str:
+    """Flatten a state into text for providers that only read strings.
+
+    Mappings become ``key: value`` lines (nested values as compact JSON) so
+    field names stay visible to the model.
+    """
+    if isinstance(state, str):
+        return state
+    if isinstance(state, Mapping):
+        lines = []
+        for key, value in state.items():
+            if value is None or value == "":
+                continue
+            text = (
+                value
+                if isinstance(value, str)
+                else json.dumps(value, ensure_ascii=False, default=str)
+            )
+            lines.append(f"{key}: {text}")
+        return "\n".join(lines)
+    if isinstance(state, Sequence):
+        return "\n".join(render_state(item) for item in state)
+    return str(state)
+
+
+class ProviderResult(BaseModel):
+    """What a provider returns for one batched call.
+
+    Attributes:
+        answers: One entry per question asked. ``None`` means the provider
+            abstained on that question.
+        model: The model that served the call, as reported by the backend.
+        usage: Tokens consumed by the call.
+        cost_usd: Cost reported by the backend for this call. When ``None``,
+            the engine estimates it from the price table.
+        meta: Structural details recorded on the trace span.
+        content: Prompts and raw completions; recorded only when content
+            capture is on.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    answers: dict[str, Answer | None]
+    model: str | None = None
+    usage: Usage = Field(default_factory=Usage)
+    cost_usd: float | None = None
+    meta: dict[str, Any] = Field(default_factory=dict)
+    content: dict[str, Any] = Field(default_factory=dict)
+
+
+class DecisionProvider(ABC):
+    """Answers typed questions.
+
+    Subclasses set:
+        name: Unique name within an engine, used in traces and allowlists.
+        plane: ``rule``, ``model`` or ``llm``.
+        kinds: Question kinds the provider can answer.
+        calibrated: Whether its probabilities are meaningful enough to
+            threshold. Prompted LLMs are not.
+        accepts_context: Whether ``answer`` takes a ``context`` keyword with
+            the decisions already settled in the same batch. The engine only
+            passes it to providers that set this.
+        price_key: Provider id used for price lookup.
+    """
+
+    name: str = "provider"
+    plane: ClassVar[Plane] = Plane.MODEL
+    kinds: ClassVar[frozenset[Kind]] = frozenset()
+    calibrated: ClassVar[bool] = True
+    accepts_context: ClassVar[bool] = False
+    price_key: str = "local"
+
+    def supports(self, question: Question) -> bool:
+        return question.kind in self.kinds and question.allows(self.name)
+
+    @abstractmethod
+    def answer(self, state: State, questions: Mapping[str, Question]) -> ProviderResult:
+        """Answer every question in one call where the backend allows it."""
+        raise NotImplementedError
+
+    def warmup(self) -> None:  # noqa: B027 - optional hook
+        """Load weights ahead of the first call."""
+
+    def close(self) -> None:  # noqa: B027 - optional hook
+        """Release resources."""
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(name={self.name!r})"

thinkless/providers/gliner.py

@@ -0,0 +1,194 @@
+"""GLiNER 2.5: schema-driven classification and extraction on small encoders."""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Mapping
+from typing import Any, ClassVar
+
+from .._hub import ensure_downloaded
+from ..confidence import normalize_distribution
+from ..decision import Answer, Plane
+from ..logs import get_logger
+from ..questions import Choice, Extract, Kind, Question
+from ..settings import resolve_device
+from .base import DecisionProvider, ProviderResult, State, render_state
+
+__all__ = ["GLiNER"]
+
+logger = get_logger("providers.gliner")
+
+# Representative length for warmup: GPU kernels are chosen per input shape.
+_WARMUP_TEXT = (
+    "Hello, I placed order 4471 last Tuesday and the tracking page has not changed since. "
+    "Could you tell me when it will arrive, or refund it if it is lost? Thanks for your help."
+)
+
+
+def _field_spec(name: str, description: str) -> str:
+    return f"{name}::str::{description.replace('::', ':')}"
+
+
+class GLiNER(DecisionProvider):
+    """Answers choice and extract questions with a GLiNER2 model.
+
+    Choice questions are scored per label (the model emits an independent
+    score for every option) and the scores are normalized into a distribution,
+    so confidence is comparable with other providers. All choice questions of
+    a call share one forward pass, and so do all extract questions.
+
+    Measured on an RTX 5060 laptop GPU with ``gliner2.5-base-v1``: about 16 ms
+    per classification and 38 ms per extraction; about 85 ms each on CPU.
+
+    Args:
+        model: ``fastino/gliner2.5-base-v1`` (English, 0.2B),
+            ``fastino/gliner2.5-small-v1`` (74M, fastest on CPU) or
+            ``fastino/gliner2.5-multi-v1`` (multilingual).
+        device: ``auto``, ``cpu``, ``cuda`` or ``mps``.
+        name: Provider name in traces.
+        extract_threshold: Minimum span confidence for an extracted field.
+
+    Requires the ``gliner`` extra. Weights download on first use.
+    """
+
+    plane: ClassVar[Plane] = Plane.MODEL
+    kinds: ClassVar[frozenset[Kind]] = frozenset({Kind.CHOICE, Kind.EXTRACT})
+
+    def __init__(
+        self,
+        model: str = "fastino/gliner2.5-base-v1",
+        *,
+        device: str = "auto",
+        name: str = "gliner",
+        extract_threshold: float = 0.5,
+    ) -> None:
+        self.name = name
+        self.model = model
+        self._device_request = device
+        self._extract_threshold = extract_threshold
+        self._model: Any = None
+        self._lock = threading.Lock()
+        self.device: str | None = None
+
+    def warmup(self) -> None:
+        """Load the weights and run representative inferences of each kind.
+
+        The first forward passes on a GPU are several times slower than the
+        rest while kernels initialize, and GLiNER's TorchScript parts only
+        specialize after a couple of runs; paying for that here keeps it off
+        the first requests.
+        """
+        model = self._load()
+        labels = {f"option {i}": f"a description of option number {i}" for i in range(8)}
+        for _ in range(2):
+            self._warm(model, labels)
+
+    def _warm(self, model: Any, labels: dict[str, str]) -> None:
+        with self._lock:
+            model.classify_text(
+                _WARMUP_TEXT, {"w": {"labels": labels, "multi_label": True, "cls_threshold": 0.0}}
+            )
+            model.extract_json(
+                _WARMUP_TEXT, {"w": ["id::str::the reference number", "day::str::the day"]}
+            )
+
+    def _load(self) -> Any:
+        if self._model is not None:
+            return self._model
+        with self._lock:
+            if self._model is None:
+                try:
+                    from gliner2 import AutoExtractor
+                except ImportError as exc:  # pragma: no cover - depends on the extra
+                    raise ImportError(
+                        'The GLiNER provider needs the gliner extra: pip install "thinkless[gliner]"'
+                    ) from exc
+                device = resolve_device(self._device_request)
+                started = time.perf_counter()
+                ensure_downloaded(self.model)
+                model = AutoExtractor.from_pretrained(self.model)
+                if device != "cpu":
+                    model = model.to(device)
+                self._model = model
+                self.device = device
+                logger.info(
+                    "loaded %s on %s in %.1f s", self.model, device, time.perf_counter() - started
+                )
+        return self._model
+
+    def answer(self, state: State, questions: Mapping[str, Question]) -> ProviderResult:
+        model = self._load()
+        text = render_state(state)
+        answers: dict[str, Answer | None] = {}
+
+        choices = {k: q for k, q in questions.items() if isinstance(q, Choice)}
+        extracts = {k: q for k, q in questions.items() if isinstance(q, Extract)}
+
+        if choices:
+            tasks: dict[str, Any] = {}
+            for key, question in choices.items():
+                has_descriptions = all(question.options.values())
+                labels: Any = dict(question.options) if has_descriptions else question.labels
+                tasks[key] = {"labels": labels, "multi_label": True, "cls_threshold": 0.0}
+            with self._lock:
+                result = model.classify_text(text, tasks, include_confidence=True)
+            for key, question in choices.items():
+                answers[key] = self._choice_answer(result.get(key), question)
+
+        if extracts:
+            schema = {
+                key: [_field_spec(name, desc) for name, desc in q.fields.items()]
+                for key, q in extracts.items()
+            }
+            with self._lock:
+                result = model.extract_json(
+                    text, schema, threshold=self._extract_threshold, include_confidence=True
+                )
+            for key, extract in extracts.items():
+                answers[key] = self._extract_answer(result.get(key), extract)
+
+        return ProviderResult(answers=answers, model=self.model, meta={"device": self.device})
+
+    @staticmethod
+    def _choice_answer(raw: Any, question: Choice) -> Answer | None:
+        if not raw:
+            return None
+        items = raw if isinstance(raw, list) else [raw]
+        scores = dict.fromkeys(question.labels, 0.0)
+        for item in items:
+            label = item.get("label") if isinstance(item, Mapping) else None
+            if label in scores:
+                scores[label] = float(item.get("confidence") or 0.0)
+        if not any(scores.values()):
+            return None
+        probabilities = normalize_distribution(scores)
+        best = max(probabilities, key=probabilities.__getitem__)
+        return Answer(value=best, probabilities=probabilities, raw={"scores": scores})
+
+    @staticmethod
+    def _extract_answer(raw: Any, question: Extract) -> Answer:
+        record: Mapping[str, Any] = {}
+        if isinstance(raw, list) and raw:
+            record = raw[0] if isinstance(raw[0], Mapping) else {}
+        elif isinstance(raw, Mapping):
+            record = raw
+        values: dict[str, Any] = {}
+        confidences: dict[str, float | None] = {}
+        for name in question.fields:
+            item = record.get(name)
+            if isinstance(item, Mapping):
+                values[name] = item.get("text")
+                confidences[name] = (
+                    float(item["confidence"]) if item.get("confidence") is not None else None
+                )
+            elif isinstance(item, list) and item and isinstance(item[0], Mapping):
+                values[name] = item[0].get("text")
+                confidences[name] = item[0].get("confidence")
+            else:
+                values[name] = item if isinstance(item, str) else None
+                confidences[name] = None
+        return Answer(value=values, fields=confidences, raw={"record": dict(record)})
+
+    def close(self) -> None:
+        self._model = None