debugerai 0.2.0__py3-none-any.whl

debugai/metrics.py

@@ -0,0 +1,139 @@
+"""Per-model metrics ledger — lightweight thread-safe counters for tokens,
+cost, latency, requests, and failures.
+
+    import debugai
+    debugai.metrics.snapshot()      # full dict
+    debugai.metrics.by_model        # per-model breakdown
+    debugai.metrics.reset()         # clear all counters
+"""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass, field
+
+
+@dataclass
+class _ModelStats:
+    requests: int = 0
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    total_tokens: int = 0
+    cost_usd: float = 0.0
+    failures: int = 0
+    cache_hits: int = 0
+    cache_misses: int = 0
+    _latencies: list[float] = field(default_factory=list)
+
+    def record(self, prompt: int, completion: int, cost: float,
+               latency_ms: float, failed: bool, from_cache: bool = False) -> None:
+        self.requests += 1
+        self.prompt_tokens += prompt
+        self.completion_tokens += completion
+        self.total_tokens += prompt + completion
+        self.cost_usd = round(self.cost_usd + cost, 8)
+        if not from_cache:
+            self._latencies.append(latency_ms)
+        if failed:
+            self.failures += 1
+        if from_cache:
+            self.cache_hits += 1
+        else:
+            self.cache_misses += 1
+
+    def _pct(self, p: float) -> float:
+        if not self._latencies:
+            return 0.0
+        s = sorted(self._latencies)
+        i = max(0, min(int(len(s) * p), len(s) - 1))
+        return round(s[i], 2)
+
+    def to_dict(self) -> dict:
+        return {
+            "requests": self.requests,
+            "prompt_tokens": self.prompt_tokens,
+            "completion_tokens": self.completion_tokens,
+            "total_tokens": self.total_tokens,
+            "cost_usd": round(self.cost_usd, 6),
+            "failures": self.failures,
+            "cache_hits": self.cache_hits,
+            "cache_misses": self.cache_misses,
+            "latency_p50_ms": self._pct(0.50),
+            "latency_p95_ms": self._pct(0.95),
+        }
+
+
+class MetricsLedger:
+    """Thread-safe per-model aggregate counters. Updated by the SDK worker
+    after each request; safe to read from any thread."""
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._models: dict[str, _ModelStats] = {}
+        self._global = _ModelStats()
+
+    # ── Recording (called by background worker) ─────────────────────────────
+    def record(self, model: str, prompt_tokens: int, completion_tokens: int,
+               cost_usd: float, latency_ms: float, failed: bool,
+               from_cache: bool = False) -> None:
+        with self._lock:
+            if model not in self._models:
+                self._models[model] = _ModelStats()
+            self._models[model].record(prompt_tokens, completion_tokens,
+                                        cost_usd, latency_ms, failed, from_cache)
+            self._global.record(prompt_tokens, completion_tokens,
+                                cost_usd, latency_ms, failed, from_cache)
+
+    # ── Read properties (safe from any thread) ──────────────────────────────
+    @property
+    def requests(self) -> int:
+        with self._lock:
+            return self._global.requests
+
+    @property
+    def failures(self) -> int:
+        with self._lock:
+            return self._global.failures
+
+    @property
+    def total_tokens(self) -> int:
+        with self._lock:
+            return self._global.total_tokens
+
+    @property
+    def cost_usd(self) -> float:
+        with self._lock:
+            return self._global.cost_usd
+
+    @property
+    def latency_p50(self) -> float:
+        with self._lock:
+            return self._global._pct(0.50)
+
+    @property
+    def latency_p95(self) -> float:
+        with self._lock:
+            return self._global._pct(0.95)
+
+    @property
+    def by_model(self) -> dict[str, dict]:
+        with self._lock:
+            return {m: s.to_dict() for m, s in self._models.items()}
+
+    def snapshot(self) -> dict:
+        """Return a complete, JSON-serialisable snapshot of all counters."""
+        with self._lock:
+            return {
+                **self._global.to_dict(),
+                "by_model": {m: s.to_dict() for m, s in self._models.items()},
+            }
+
+    def reset(self) -> None:
+        """Clear all counters (useful between test runs or reporting windows)."""
+        with self._lock:
+            self._models.clear()
+            self._global = _ModelStats()
+
+
+# Module-level singleton — `import debugai; debugai.metrics`
+metrics = MetricsLedger()

debugai/models.py

@@ -0,0 +1,92 @@
+"""Lazy-loaded small ML models (Architecture §8.2).
+
+These are NOT LLMs — they are tiny, fast, task-specific models that run on CPU:
+  - sentence-transformers/all-MiniLM-L6-v2  (embeddings, ~80MB)
+  - spaCy en_core_web_sm                    (NER, ~12MB)
+  - cross-encoder/nli-MiniLM2-L6-H768       (NLI, ~120MB)
+
+Each loader is a cached singleton so model weights load once per process. If a
+model is unavailable, the loader returns ``None`` and signal computations fall
+back to their deterministic pure-Python methods (per the doc's layered design).
+"""
+
+from __future__ import annotations
+
+import functools
+import logging
+import os
+
+log = logging.getLogger("debugai.models")
+
+EMBED_MODEL = "all-MiniLM-L6-v2"
+SPACY_MODEL = "en_core_web_sm"
+
+# Three NLI modes (set via env var):
+#
+#   default              → cross-encoder/nli-deberta-v3-base  local (~500 MB RAM)
+#                          Most accurate. Best for self-hosted VPS with ≥1 GB RAM.
+#
+#   DEBUGAI_LITE=1       → cross-encoder/nli-MiniLM2-L6-H768  local (~120 MB RAM)
+#                          Fits in free-tier PaaS (512 MB RAM).  Slightly more
+#                          false-positive contradictions but good enough.
+#
+#   DEBUGAI_NLI_API=1    → HuggingFace Inference API           zero local RAM
+#                          Sends (premise, hypothesis) to api-inference.huggingface.co.
+#                          Set HF_TOKEN for higher rate limits (free account works).
+#                          Best choice for Render / Railway free tier.
+_LITE    = bool(os.environ.get("DEBUGAI_LITE"))
+_NLI_API = bool(os.environ.get("DEBUGAI_NLI_API"))
+NLI_MODEL = ("cross-encoder/nli-MiniLM2-L6-H768" if _LITE
+             else "cross-encoder/nli-deberta-v3-base")
+NLI_HF_MODEL_ID = "cross-encoder/nli-deberta-v3-base"  # used by the API path
+
+
+@functools.lru_cache(maxsize=1)
+def embedder():
+    """SentenceTransformer for semantic cosine. None if unavailable."""
+    try:
+        from sentence_transformers import SentenceTransformer
+
+        log.info("loading embedding model %s", EMBED_MODEL)
+        return SentenceTransformer(EMBED_MODEL)
+    except Exception as e:  # pragma: no cover - environment dependent
+        log.warning("embedder unavailable (%s); using token-overlap fallback", e)
+        return None
+
+
+@functools.lru_cache(maxsize=1)
+def nli_model():
+    """CrossEncoder NLI model. Returns label-ordered logits. None if unavailable.
+
+    When DEBUGAI_NLI_API=1 this returns a special sentinel object that tells
+    compute_contradiction() to call the HuggingFace Inference API instead.
+    """
+    if _NLI_API:
+        return _HFNLISentinel()
+    try:
+        from sentence_transformers import CrossEncoder
+
+        log.info("loading NLI model %s", NLI_MODEL)
+        return CrossEncoder(NLI_MODEL)
+    except Exception as e:  # pragma: no cover - environment dependent
+        log.warning("NLI model unavailable (%s); contradiction set to 0.0", e)
+        return None
+
+
+class _HFNLISentinel:
+    """Marker returned by nli_model() when DEBUGAI_NLI_API=1.
+    compute_contradiction() detects this and calls the HF Inference API."""
+    is_hf_api = True
+
+
+@functools.lru_cache(maxsize=1)
+def ner():
+    """spaCy NER pipeline. None if unavailable (regex fallback used instead)."""
+    try:
+        import spacy
+
+        log.info("loading spaCy model %s", SPACY_MODEL)
+        return spacy.load(SPACY_MODEL)
+    except Exception as e:  # pragma: no cover - environment dependent
+        log.warning("spaCy model unavailable (%s); using regex NER fallback", e)
+        return None

debugai/providers.py

@@ -0,0 +1,179 @@
+"""Provider routing table — maps model name prefixes to (base_url, api_key_env,
+adapter_class). Called by ``debugai.completion()`` / ``debugai.acompletion()``.
+
+The key architectural insight: most modern providers speak the OpenAI REST API spec
+(same endpoint shape, same response format). A single ``_OpenAICompatAdapter`` covers:
+  - Google Gemini (via Google's official OpenAI-compat endpoint)
+  - Ollama + any local model (Qwen, Llama, Phi, DeepSeek…)
+  - Groq, Together AI, Mistral AI, OpenRouter, Azure OpenAI, LM Studio, vLLM
+  - Any custom server that accepts POST /v1/chat/completions
+
+Only Cohere requires a native adapter (different API shape).
+
+All routing is prefix-based on the model name. Users can extend or override the table:
+    from debugai import register_provider
+    register_provider(matches=lambda m: m.startswith("my-"), adapter=MyAdapter,
+                      client_factory=lambda cfg: MyClient(...))
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Callable
+
+if TYPE_CHECKING:
+    from debugai.config import DebugAIConfig
+
+
+@dataclass(frozen=True)
+class ProviderRoute:
+    prefix: str                     # model name prefix, e.g. "gemini-", "ollama/"
+    name: str                       # human name, e.g. "Google Gemini"
+    base_url: str | None            # None → use the SDK default
+    api_key_env: str | None         # env var name for the API key; None → no key (local)
+    adapter: str                    # "openai" | "openai_compat" | "anthropic" | "cohere"
+    notes: str = ""
+
+
+# ---------------------------------------------------------------------------
+# The routing table. Checked in order — first prefix match wins.
+# ---------------------------------------------------------------------------
+PROVIDER_ROUTES: list[ProviderRoute] = [
+    # ── OpenAI ──────────────────────────────────────────────────────────────
+    ProviderRoute("gpt-",          "OpenAI",        None,                                "OPENAI_API_KEY",     "openai"),
+    ProviderRoute("o1-",           "OpenAI",        None,                                "OPENAI_API_KEY",     "openai"),
+    ProviderRoute("o3-",           "OpenAI",        None,                                "OPENAI_API_KEY",     "openai"),
+    ProviderRoute("o4-",           "OpenAI",        None,                                "OPENAI_API_KEY",     "openai"),
+    ProviderRoute("text-",         "OpenAI",        None,                                "OPENAI_API_KEY",     "openai"),
+
+    # ── Anthropic ───────────────────────────────────────────────────────────
+    ProviderRoute("claude-",       "Anthropic",     None,                                "ANTHROPIC_API_KEY",  "anthropic"),
+
+    # ── Google Gemini (OpenAI-compat endpoint) ──────────────────────────────
+    ProviderRoute("gemini-",       "Google Gemini", "https://generativelanguage.googleapis.com/v1beta/openai/", "GEMINI_API_KEY", "openai_compat"),
+    ProviderRoute("google/",       "Google Gemini", "https://generativelanguage.googleapis.com/v1beta/openai/", "GEMINI_API_KEY", "openai_compat"),
+
+    # ── Groq (fast inference, OpenAI-compat) ────────────────────────────────
+    ProviderRoute("groq/",         "Groq",          "https://api.groq.com/openai/v1",    "GROQ_API_KEY",       "openai_compat"),
+
+    # ── Together AI (OpenAI-compat) ─────────────────────────────────────────
+    ProviderRoute("together/",     "Together AI",   "https://api.together.xyz/v1",       "TOGETHER_API_KEY",   "openai_compat"),
+
+    # ── Mistral AI (OpenAI-compat) ──────────────────────────────────────────
+    ProviderRoute("mistral/",      "Mistral AI",    "https://api.mistral.ai/v1",         "MISTRAL_API_KEY",    "openai_compat"),
+
+    # ── OpenRouter (multi-provider proxy, OpenAI-compat) ────────────────────
+    ProviderRoute("openrouter/",   "OpenRouter",    "https://openrouter.ai/api/v1",      "OPENROUTER_API_KEY", "openai_compat"),
+
+    # ── Azure OpenAI (OpenAI-compat + endpoint from env) ────────────────────
+    ProviderRoute("azure/",        "Azure OpenAI",  None,                                "AZURE_OPENAI_API_KEY","openai_compat",
+                  notes="Set AZURE_OPENAI_ENDPOINT. Model name: 'azure/<deployment-name>'"),
+
+    # ── Cohere (native adapter) ──────────────────────────────────────────────
+    ProviderRoute("cohere/",       "Cohere",        None,                                "COHERE_API_KEY",     "cohere"),
+    ProviderRoute("command-",      "Cohere",        None,                                "COHERE_API_KEY",     "cohere"),
+
+    # ── Ollama — local models via the Ollama OpenAI-compat server ───────────
+    # Any model served by Ollama: qwen2.5, llama3, phi3, deepseek-coder, codellama…
+    # Default: http://localhost:11434/v1  (override with OLLAMA_BASE_URL or config.ollama_base_url)
+    ProviderRoute("ollama/",       "Ollama (local)",  None,  None, "ollama",
+                  notes="Requires Ollama running. Set OLLAMA_BASE_URL if not localhost:11434."),
+    # Common local model families also route to Ollama when no prefix given:
+    ProviderRoute("qwen",          "Ollama (Qwen)",   None,  None, "ollama"),
+    ProviderRoute("llama",         "Ollama (Llama)",  None,  None, "ollama"),
+    ProviderRoute("phi",           "Ollama (Phi)",    None,  None, "ollama"),
+    ProviderRoute("deepseek",      "Ollama (DeepSeek)",None, None, "ollama"),
+    ProviderRoute("codellama",     "Ollama (CodeLlama)",None,None, "ollama"),
+    ProviderRoute("gemma",         "Ollama (Gemma)",  None,  None, "ollama"),
+    ProviderRoute("mixtral",       "Ollama (Mixtral)",None,  None, "ollama"),
+    ProviderRoute("vicuna",        "Ollama (Vicuna)", None,  None, "ollama"),
+]
+
+
+def route_for(model: str) -> ProviderRoute | None:
+    """Return the first matching route for a model name, or None."""
+    low = model.lower()
+    for route in PROVIDER_ROUTES:
+        if low.startswith(route.prefix.lower()):
+            return route
+    return None
+
+
+def _get_adapter_map():
+    """Deferred import to avoid circular imports at module load time."""
+    from debugai.sdk import (
+        _AnthropicAdapter, _CohereAdapter,
+        _OpenAIAdapter, _OpenAICompatAdapter,
+    )
+    return {
+        "openai": _OpenAIAdapter,
+        "anthropic": _AnthropicAdapter,
+        "openai_compat": _OpenAICompatAdapter,
+        "ollama": _OpenAICompatAdapter,
+        "cohere": _CohereAdapter,
+    }
+
+
+# Lazily evaluated so there's no circular import at load time.
+class _AdapterMapProxy(dict):
+    _loaded = False
+    def _ensure(self):
+        if not self._loaded:
+            self.update(_get_adapter_map())
+            self._loaded = True
+    def get(self, key, default=None):
+        self._ensure()
+        return super().get(key, default)
+    def __getitem__(self, key):
+        self._ensure()
+        return super().__getitem__(key)
+
+
+_ADAPTER_MAP = _AdapterMapProxy()
+
+
+def make_client(route: ProviderRoute, config: "DebugAIConfig") -> Any:
+    """Build the provider client from a route + config. All OpenAI-compat clients
+    are instantiated via the OpenAI SDK (no per-provider install needed)."""
+
+    if route.adapter == "anthropic":
+        from anthropic import Anthropic
+        return Anthropic(timeout=60.0, max_retries=2)
+
+    if route.adapter == "cohere":
+        try:
+            import cohere  # optional; graceful ImportError if not installed
+            return cohere.ClientV2(api_key=os.environ.get(route.api_key_env or "COHERE_API_KEY", ""))
+        except ImportError:
+            raise ImportError(
+                "Cohere models require the 'cohere' package: pip install cohere"
+            )
+
+    if route.adapter == "ollama":
+        from openai import OpenAI
+        base_url = (getattr(config, "ollama_base_url", None)
+                    or os.environ.get("OLLAMA_BASE_URL")
+                    or "http://localhost:11434/v1")
+        return OpenAI(base_url=base_url, api_key="ollama", timeout=120.0)
+
+    if route.adapter in ("openai_compat", "openai"):
+        from openai import OpenAI
+        base_url = route.base_url
+        api_key: str | None = None
+
+        if route.adapter == "openai_compat":
+            if route.prefix.startswith("azure/"):
+                base_url = (os.environ.get("AZURE_OPENAI_ENDPOINT", "")
+                            .rstrip("/") + "/openai")
+            api_key = (os.environ.get(route.api_key_env, "") if route.api_key_env
+                       else "no-key-needed")
+
+        return OpenAI(
+            base_url=base_url,
+            api_key=api_key or os.environ.get(route.api_key_env or "", ""),
+            timeout=60.0,
+            max_retries=2,
+        )
+
+    raise ValueError(f"Unknown adapter type {route.adapter!r} in routing table.")

debugai/schema.py

@@ -0,0 +1,66 @@
+"""SDK capture schema (Architecture §3).
+
+The unified payload every integration level produces. Only the Core IO group is
+strictly required; retrieval and runtime groups unlock RAG-specific and
+capacity signals respectively.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class CaptureRecord:
+    """Unified request payload: prompt + context + output + metadata.
+
+    Groups (§3.1):
+      Core IO          — minimum viable input (required).
+      Retrieval        — RAG-specific signals.
+      Metadata         — pipeline configuration.
+      Runtime          — auto-captured metrics.
+    """
+
+    # --- Core IO (required) ---
+    user_prompt: str
+    llm_output: str
+    system_prompt: str = ""
+    expected_output: str | None = None
+
+    # --- Retrieval context (RAG) ---
+    retrieved_chunks: list[str] = field(default_factory=list)
+    similarity_scores: list[float] = field(default_factory=list)
+    retrieval_query: str | None = None
+    chunk_sources: list[dict[str, Any]] = field(default_factory=list)
+
+    # --- Pipeline metadata ---
+    model_name: str | None = None
+    temperature: float | None = None
+    max_tokens: int | None = None
+    tool_calls: list[dict[str, Any]] = field(default_factory=list)
+
+    # --- Runtime metrics ---
+    latency_ms: int | None = None
+    token_usage: dict[str, int] = field(default_factory=dict)
+    timestamp: str | None = None
+    error_code: str | None = None
+
+    # --- Optional capacity hints (used by ratio signals) ---
+    context_window: int | None = None  # model's max context window in tokens
+
+    def __post_init__(self) -> None:
+        if not self.user_prompt:
+            raise ValueError("user_prompt is required (Core IO)")
+        if self.llm_output is None:
+            raise ValueError("llm_output is required (Core IO)")
+
+    @property
+    def context_text(self) -> str:
+        """Concatenated retrieved chunks — the 'grounding' the output should rest on."""
+        return "\n".join(self.retrieved_chunks)
+
+    @property
+    def full_prompt(self) -> str:
+        """System + user prompt combined (used for context-length accounting)."""
+        return (self.system_prompt + "\n" + self.user_prompt).strip()