dot-inference 2.0.2__py3-none-any.whl

dot_inference/models.yaml

@@ -0,0 +1,481 @@
+# Model registry — canonical model definitions with per-provider routing.
+#
+# ─── Custom models ────────────────────────────────────────────────────
+#
+# To add your own models (or override built-in ones) without editing this
+# file, create a YAML file with the same format and set:
+#
+#   export DOTI_CUSTOM_MODELS=/path/to/my_models.yaml
+#
+# Entries in your custom file are merged on top of this registry:
+#   - New model IDs are added.
+#   - Existing model IDs are fully replaced.
+#
+# Example custom file:
+#
+#   anthropic/claude-sonnet-4:
+#     supports_strict_json: true
+#     input_token_cost: 3.0e-6
+#     output_token_cost: 1.5e-5
+#     providers:
+#       openrouter:
+#         api_model_name: anthropic/claude-sonnet-4
+#
+# ─── Field reference ──────────────────────────────────────────────────
+#
+# Each top-level key is a canonical model ID used in settings (e.g.
+# DOTI_GENERATION__OPENAI__MODEL_NAME=openai/gpt-4o).
+#
+# Model-level fields (defined once, shared across providers):
+#   supports_strict_json: true/false
+#     - If true: uses server-side constrained decoding (strict: true in
+#       json_schema). Output is guaranteed valid JSON.
+#     - If false: uses client-side parsing. Best-effort, may fail on
+#       weaker models.
+#
+#   supports_reasoning: true/false
+#     - Whether the model supports the reasoning_effort parameter.
+#     - Can be overridden per provider (see below) when reasoning is
+#       available through one provider but not another.
+#
+#   reasoning_effort_options: list[str] (optional)
+#     - The reasoning_effort values this model actually accepts. Lets a
+#       client offer only the valid choices. Defaults to
+#       [low, medium, high] when supports_reasoning is true and this is
+#       omitted. Mistral reasoning models (Medium 3.5, Small) accept only
+#       [none, high]. Ignored when supports_reasoning is false.
+#
+#   default_reasoning_effort: str (optional)
+#     - The effort to use for this model when the caller doesn't specify
+#       one. Must be a member of reasoning_effort_options. Falls back to
+#       the global GenerationSettings default when omitted.
+#
+#   input_token_cost / output_token_cost:
+#     - Default per-token USD cost (direct-from-vendor pricing).
+#     - Can be overridden per provider (e.g. OpenRouter markup).
+#     - Inline "# $X/M" comments restate the cost per million tokens.
+#
+#   default_provider: str (optional)
+#     - Provider used when get_llm_client() is called with model= but no
+#       provider=. Falls back to DOTI_GENERATION__PROVIDER when unset.
+#     - An explicit provider= argument always wins.
+#
+#   context_window: int (optional)
+#     - Maximum tokens (input + output) the model accepts in one request.
+#     - Public spec value; used by callers to render fullness gauges and
+#       guard against over-long prompts. Omit when not declared.
+#
+# Per-provider fields (nested under "providers"):
+#   api_model_name: str
+#     - The model string sent to the provider's API.
+#
+#   input_token_cost / output_token_cost (optional):
+#     - Provider-specific cost override.
+#
+#   supports_reasoning: true/false (optional)
+#     - Per-provider override for reasoning support.
+#     - When omitted, inherits the model-level value.
+#     - Useful when the model supports reasoning via one provider (e.g.
+#       OpenRouter) but not another (e.g. native SDK lacks the param).
+#
+#   features (optional, provider-specific):
+#     response_healing: true/false
+#       - OpenRouter plugin that post-processes malformed JSON responses.
+#       - Only useful when supports_strict_json is false.
+#
+#     fallback_models: list of model names
+#       - Alternative models if the primary fails (OpenRouter "models").
+#
+#     provider_routing: dict
+#       - OpenRouter provider routing preferences.
+#       - Example: only: ["mistral"] routes exclusively through Mistral.
+
+# ─── OpenAI models ───────────────────────────────────────────────────
+
+openai/gpt-5:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 1.25e-6   # $1.25/M
+  output_token_cost: 1.e-5    # $10/M
+  context_window: 400000
+  providers:
+    openai:
+      api_model_name: gpt-5
+    openrouter:
+      api_model_name: openai/gpt-5
+      features:
+        provider_routing:
+          only: ["openai"]
+
+openai/gpt-5.1:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 1.25e-6   # $1.25/M
+  output_token_cost: 1.e-5    # $10/M
+  context_window: 400000
+  providers:
+    openai:
+      api_model_name: gpt-5.1
+    openrouter:
+      api_model_name: openai/gpt-5.1
+      features:
+        provider_routing:
+          only: ["openai"]
+
+openai/gpt-5.2:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 1.75e-6   # $1.75/M
+  output_token_cost: 1.4e-5   # $14/M
+  context_window: 400000
+  providers:
+    openai:
+      api_model_name: gpt-5.2
+    openrouter:
+      api_model_name: openai/gpt-5.2
+      features:
+        provider_routing:
+          only: ["openai"]
+
+openai/gpt-5-mini:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 2.5e-7    # $0.25/M
+  output_token_cost: 2.e-6    # $2/M
+  context_window: 400000
+  providers:
+    openai:
+      api_model_name: gpt-5-mini
+    openrouter:
+      api_model_name: openai/gpt-5-mini
+      features:
+        provider_routing:
+          only: ["openai"]
+
+openai/gpt-oss-20b:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 2e-8      # $0.02/M
+  output_token_cost: 1e-7     # $0.10/M
+  context_window: 128000
+  providers:
+    openrouter:
+      api_model_name: openai/gpt-oss-20b
+    local:
+      api_model_name: openai/gpt-oss-20b
+
+# Self-hosted alias — see openai/gpt-oss-120b-local below.
+openai/gpt-oss-20b-local:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 0
+  output_token_cost: 0
+  context_window: 128000
+  default_provider: local
+  providers:
+    local:
+      api_model_name: openai/gpt-oss-20b
+
+openai/gpt-5.4-nano:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 2e-7      # $0.20/M
+  output_token_cost: 1.25e-6  # $1.25/M
+  context_window: 400000
+  providers:
+    openai:
+      api_model_name: gpt-5.4-nano
+    openrouter:
+      api_model_name: openai/gpt-5.4-nano
+      features:
+        provider_routing:
+          only: ["openai"]
+
+openai/gpt-5.4-mini:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 7.5e-7    # $0.75/M
+  output_token_cost: 4.5e-6   # $4.50/M
+  context_window: 400000
+  providers:
+    openai:
+      api_model_name: gpt-5.4-mini
+    openrouter:
+      api_model_name: openai/gpt-5.4-mini
+      features:
+        provider_routing:
+          only: ["openai"]
+
+openai/gpt-5.4:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 2.5e-6    # $2.50/M
+  output_token_cost: 1.5e-5   # $15/M
+  context_window: 1050000
+  providers:
+    openai:
+      api_model_name: gpt-5.4
+    openrouter:
+      api_model_name: openai/gpt-5.4
+      features:
+        provider_routing:
+          only: ["openai"]
+
+openai/gpt-oss-120b:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 3.9e-8    # $0.039/M
+  output_token_cost: 1.9e-7   # $0.19/M
+  context_window: 128000
+  providers:
+    openrouter:
+      api_model_name: openai/gpt-oss-120b
+    local:
+      api_model_name: openai/gpt-oss-120b
+
+# Self-hosted alias: pinned to the `local` provider so it can
+# coexist with the OpenRouter route above as a distinct selectable model.
+# Requires DOTI_GENERATION__LOCAL__BASE_URL (get_llm_client errors otherwise);
+openai/gpt-oss-120b-local:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 0
+  output_token_cost: 0
+  context_window: 128000
+  default_provider: local
+  providers:
+    local:
+      api_model_name: openai/gpt-oss-120b
+
+# ─── Google models ───────────────────────────────────────────────────
+# Gemini is only served by Google on OpenRouter, so no provider pin is
+# needed. The canonical IDs are the friendly aliases the agent exposes;
+# api_model_name carries the dated OpenRouter slug (which drifts as Google
+# ships new previews — keep these mapped to a slug that actually resolves).
+
+google/gemini-3-flash:
+  supports_strict_json: true
+  input_token_cost: 5e-7      # $0.50/M
+  output_token_cost: 3e-6     # $3/M
+  context_window: 1000000
+  providers:
+    openrouter:
+      api_model_name: google/gemini-3-flash-preview
+
+# Friendly alias "gemini-3-pro": OpenRouter has no plain google/gemini-3-pro
+# text endpoint, so this resolves to the current Gemini 3-series pro preview.
+google/gemini-3-pro:
+  supports_strict_json: true
+  input_token_cost: 2e-6      # $2/M
+  output_token_cost: 1.2e-5   # $12/M
+  context_window: 1000000
+  providers:
+    openrouter:
+      api_model_name: google/gemini-3.1-pro-preview
+
+# Gemma 4 31B (v26.04, April 2026): open-weight (Apache 2.0) dense multimodal
+# model with native function calling, structured outputs, and a configurable
+# reasoning mode. Served across providers on OpenRouter.
+google/gemma-4-31b-it:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 1.2e-7    # $0.12/M
+  output_token_cost: 3.5e-7   # $0.35/M
+  context_window: 262144
+  providers:
+    openrouter:
+      api_model_name: google/gemma-4-31b-it
+
+# ─── Mistral models ─────────────────────────────────────────────────
+
+mistral/mistral-large-latest:
+  default_provider: mistral
+  supports_strict_json: true
+  input_token_cost: 2e-6      # $2/M
+  output_token_cost: 6e-6     # $6/M
+  context_window: 256000
+  providers:
+    mistral:
+      api_model_name: mistral-large-latest
+    openrouter:
+      api_model_name: mistralai/mistral-large-2512
+      input_token_cost: 5e-7    # $0.50/M
+      output_token_cost: 1.5e-6 # $1.50/M
+      features:
+        provider_routing:
+          only: ["mistral"]
+
+
+# Alias — always resolves to the latest Mistral Small release.
+# Use this when you want automatic upgrades as Mistral ships new versions.
+# Note: behavior may change silently when the alias is reassigned.
+#
+# Reasoning: the Mistral REST API supports reasoning_effort ("high"/"none")
+# for this model, but the Python SDK (mistralai v2.0.2) does not yet expose
+# the parameter.  Reasoning is disabled at the model level to keep behavior
+# consistent across providers.  Re-enable once the SDK adds support.
+mistral/mistral-small-latest:
+  default_provider: mistral
+  supports_strict_json: true
+  input_token_cost: 1.5e-7    # $0.15/M
+  output_token_cost: 6e-7     # $0.60/M
+  context_window: 256000
+  providers:
+    mistral:
+      api_model_name: mistral-small-latest
+    openrouter:
+      api_model_name: mistralai/mistral-small-2603
+      input_token_cost: 1.5e-7  # $0.15/M
+      output_token_cost: 6e-7   # $0.60/M
+      features:
+        provider_routing:
+          only: ["mistral"]
+
+# Pinned version of Mistral Small 4 (March 2026).
+# Kept alongside the alias for production stability: pipelines that require
+# reproducible behavior should reference this ID so they are unaffected when
+# mistral-small-latest is reassigned to a future release.
+# Same SDK limitation as the alias above — see comment there.
+mistral/mistral-small-2603:
+  default_provider: mistral
+  supports_strict_json: true
+  input_token_cost: 1.5e-7    # $0.15/M
+  output_token_cost: 6e-7     # $0.60/M
+  providers:
+    mistral:
+      api_model_name: mistral-small-2603
+    openrouter:
+      api_model_name: mistralai/mistral-small-2603
+      input_token_cost: 1.5e-7  # $0.15/M
+      output_token_cost: 6e-7   # $0.60/M
+      features:
+        provider_routing:
+          only: ["mistral"]
+
+mistral/mistral-medium-3.1:
+  default_provider: mistral
+  supports_strict_json: true
+  input_token_cost: 4e-7      # $0.40/M
+  output_token_cost: 2e-6     # $2/M
+  providers:
+    mistral:
+      api_model_name: mistral-medium-2508
+    openrouter:
+      api_model_name: mistralai/mistral-medium-3.1
+      input_token_cost: 4e-7    # $0.40/M
+      output_token_cost: 2e-6   # $2/M
+      features:
+        provider_routing:
+          only: ["mistral"]
+
+# Frontier-class multimodal model optimized for agentic and coding use cases
+# (v26.04, April 2026). Open weights under a Modified MIT license.
+mistral/mistral-medium-3.5:
+  default_provider: mistral
+  supports_strict_json: true
+  # Adjustable reasoning: La Plateforme accepts only "none" (minimal
+  # thinking, no thinking chunk) and "high" (full thinking chunk).
+  # Mistral recommends "high" for agentic & code use cases.
+  supports_reasoning: true
+  reasoning_effort_options: [none, high]
+  default_reasoning_effort: high
+  input_token_cost: 1.5e-6    # $1.50/M
+  output_token_cost: 7.5e-6   # $7.50/M
+  context_window: 256000
+  providers:
+    mistral:
+      api_model_name: mistral-medium-2604
+    openrouter:
+      api_model_name: mistralai/mistral-medium-3-5
+      input_token_cost: 1.5e-6  # $1.50/M
+      output_token_cost: 7.5e-6 # $7.50/M
+      features:
+        provider_routing:
+          only: ["mistral"]
+
+# ─── NVIDIA models ───────────────────────────────────────────────
+
+nvidia/nemotron-3-super-120b-a12b:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 9e-8      # $0.09/M
+  output_token_cost: 4.5e-7   # $0.45/M
+  context_window: 1000000
+  providers:
+    openrouter:
+      api_model_name: nvidia/nemotron-3-super-120b-a12b
+
+# ─── MiniMax models ───────────────────────────────────────────────
+
+minimax/minimax-m2.7:
+  supports_strict_json: false
+  supports_reasoning: true
+  input_token_cost: 3e-7      # $0.30/M
+  output_token_cost: 1.2e-6   # $1.20/M
+  providers:
+    openrouter:
+      api_model_name: minimax/minimax-m2.7
+
+# ─── Qwen models ───────────────────────────────────────────────────
+
+qwen/qwen3.5-9b:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 5e-8      # $0.05/M
+  output_token_cost: 1.5e-7   # $0.15/M
+  providers:
+    openrouter:
+      api_model_name: qwen/qwen3.5-9b
+
+qwen/qwen3.5-27b:
+  supports_strict_json: true
+  supports_reasoning: true
+  input_token_cost: 1.95e-7   # $0.195/M
+  output_token_cost: 1.56e-6  # $1.56/M
+  providers:
+    openrouter:
+      api_model_name: qwen/qwen3.5-27b
+
+# ─── Anthropic models ──────────────────────────────────────────────
+
+anthropic/claude-haiku-4.5:
+  supports_strict_json: true
+  supports_reasoning: true
+  requires_cache_control: true
+  input_token_cost: 1e-6      # $1/M
+  output_token_cost: 5e-6     # $5/M
+  context_window: 200000
+  providers:
+    openrouter:
+      api_model_name: anthropic/claude-haiku-4.5
+      features:
+        provider_routing:
+          only: ["anthropic"]
+
+anthropic/claude-sonnet-4.6:
+  supports_strict_json: true
+  supports_reasoning: true
+  requires_cache_control: true
+  input_token_cost: 3e-6      # $3/M
+  output_token_cost: 1.5e-5   # $15/M
+  context_window: 1000000
+  providers:
+    openrouter:
+      api_model_name: anthropic/claude-sonnet-4.6
+      features:
+        provider_routing:
+          only: ["anthropic"]
+
+anthropic/claude-opus-4.6:
+  supports_strict_json: true
+  supports_reasoning: true
+  requires_cache_control: true
+  input_token_cost: 5e-6      # $5/M
+  output_token_cost: 2.5e-5   # $25/M
+  context_window: 1000000
+  providers:
+    openrouter:
+      api_model_name: anthropic/claude-opus-4.6
+      features:
+        provider_routing:
+          only: ["anthropic"]

dot_inference/settings.py

@@ -0,0 +1,138 @@
+# SPDX-FileCopyrightText: Kannon For Deep Tech
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+"""Settings for dot-inference configuration via environment variables.
+
+All environment variables are prefixed with ``DOTI_`` and use ``__`` as
+nested delimiter.  Provider-specific settings (model, api_key, base_url)
+are grouped under the provider name::
+
+    DOTI_GENERATION__PROVIDER=openai
+    DOTI_GENERATION__OPENAI__MODEL_NAME=gpt-5
+    DOTI_GENERATION__OPENAI__API_KEY=sk-...
+    DOTI_GENERATION__LOCAL__MODEL_NAME=llama3.2:1b
+    DOTI_GENERATION__LOCAL__BASE_URL=http://localhost:11434/v1
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum, auto
+
+from pydantic import BaseModel, SecretStr, field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Provider(StrEnum):
+    """Supported LLM providers."""
+
+    MISTRAL = auto()
+    OPENAI = auto()
+    OPENROUTER = auto()
+    AZURE = auto()
+    LOCAL = auto()
+
+
+class ReasoningEffort(StrEnum):
+    """Reasoning effort levels for reasoning models (e.g. o-series, gpt-5's).
+
+    ``NONE`` is the Mistral-style "minimal thinking, no thinking chunk"
+    setting (Mistral Medium 3.5 / Small accept only ``none`` and ``high``);
+    it is distinct from passing ``None``, which means "leave unset / use the
+    instance default". The ``low``/``medium``/``high`` ladder is the
+    OpenAI/Anthropic convention. Which subset a given model accepts is
+    declared per-model — see :func:`allowed_reasoning_efforts`.
+    """
+
+    NONE = auto()
+    LOW = auto()
+    MEDIUM = auto()
+    HIGH = auto()
+
+
+class ProviderSettings(BaseModel):
+    """Per-provider settings (model_name, api_key, base_url, ...).
+
+    ``api_key`` is a :class:`~pydantic.SecretStr`, so it renders as
+    ``**********`` in ``repr()``, ``str()`` and ``model_dump()`` — a settings
+    object caught in a traceback or an error report never carries the key in
+    clear text. Read the real value with ``api_key.get_secret_value()``.
+    """
+
+    model_name: str | None = None
+    api_key: SecretStr | None = None
+    base_url: str | None = None
+
+    @field_validator("api_key", mode="before")
+    @classmethod
+    def _blank_key_is_no_key(cls, v: object) -> object:
+        """Treat ``API_KEY=`` (set but empty) as absent.
+
+        Without this, an empty env var would yield ``SecretStr('')`` — a
+        truthy object — and the "is a key configured?" checks would wrongly
+        report yes.
+        """
+        return None if isinstance(v, str) and not v.strip() else v
+
+
+class AzureProviderSettings(ProviderSettings):
+    """Azure-specific provider settings."""
+
+    api_version: str = "2024-12-01-preview"  # might be outdated, needs double checking
+
+
+class GenerationSettings(BaseModel):
+    """Generation-related settings."""
+
+    mock: bool = False
+    provider: Provider = Provider.OPENROUTER
+    temperature: float = 0
+    max_tokens: int | None = None
+    reasoning_effort: ReasoningEffort | None = ReasoningEffort.MEDIUM
+    calls_per_minute_limit: int = 0  # no limit by default
+    timeout: float = 600  # HTTP timeout in seconds
+
+    # Per-provider settings
+    mistral: ProviderSettings = ProviderSettings()
+    openai: ProviderSettings = ProviderSettings()
+    openrouter: ProviderSettings = ProviderSettings()
+    azure: AzureProviderSettings = AzureProviderSettings()
+    local: ProviderSettings = ProviderSettings()
+
+
+class EmbeddingsSettings(BaseModel):
+    """Embeddings-related settings."""
+
+    mock: bool = False
+    provider: Provider = Provider.OPENAI
+    batch_size: int = 32
+    calls_per_minute_limit: int = 0  # no limit set here atm
+    timeout: float = 300  # HTTP timeout in seconds
+
+    # Per-provider settings
+    mistral: ProviderSettings = ProviderSettings()
+    openai: ProviderSettings = ProviderSettings()
+    openrouter: ProviderSettings = ProviderSettings()
+    azure: AzureProviderSettings = AzureProviderSettings()
+    local: ProviderSettings = ProviderSettings()
+
+
+class InferenceSettings(BaseSettings):
+    """Root settings combining generation and embedding configuration.
+
+    Instantiate explicitly — there is no module-level singleton::
+
+        from dot_inference.settings import InferenceSettings
+
+        settings = InferenceSettings()  # reads from env
+        settings = InferenceSettings(_env_file="path/to/.env")  # also reads a .env file
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="DOTI_",
+        env_nested_delimiter="__",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    generation: GenerationSettings = GenerationSettings()
+    embedding: EmbeddingsSettings = EmbeddingsSettings()